databricks-sql-connector 3.0.0b1__tar.gz → 3.0.1__tar.gz

{databricks_sql_connector-3.0.0b1 → databricks_sql_connector-3.0.1}/CHANGELOG.md

@@ -1,11 +1,35 @@
 # Release History
 
-## 2.9.4 (Unreleased)
-
-- Other: Introduce SQLAlchemy dialect compliance test suite and enumerate all excluded tests
-- Add integration tests for Databricks UC Volumes ingestion queries
-- Add `_retry_max_redirects` config
+## 3.0.1 (2023-12-01)
+
+- Other: updated docstring comment about default parameterization approach (#287)
+- Other: added tests for reading complex types and revised docstrings and type hints (#293)
+- Fix: SQLAlchemy dialect raised DeprecationWarning due to `dbapi` classmethod (#294)
+- Fix: SQLAlchemy dialect could not reflect TIMESTAMP_NTZ columns (#296)
+
+## 3.0.0 (2023-11-17)
+
+- Remove support for Python 3.7
+- Add support for native parameterized SQL queries. Requires DBR 14.2 and above. See docs/parameters.md for more info.
+- Completely rewritten SQLAlchemy dialect
+  - Adds support for SQLAlchemy >= 2.0 and drops support for SQLAlchemy 1.x
+  - Full e2e test coverage of all supported features
+  - Detailed usage notes in `README.sqlalchemy.md`
+  - Adds support for:
+    - New types: `TIME`, `TIMESTAMP`, `TIMESTAMP_NTZ`, `TINYINT`
+    - `Numeric` type scale and precision, like `Numeric(10,2)`
+    - Reading and writing `PrimaryKeyConstraint` and `ForeignKeyConstraint`
+    - Reading and writing composite keys
+    - Reading and writing from views
+    - Writing `Identity` to tables (i.e. autoincrementing primary keys)
+    - `LIMIT` and `OFFSET` for paging through results
+    - Caching metadata calls
 - Enable cloud fetch by default. To disable, set `use_cloud_fetch=False` when building `databricks.sql.client`.
+- Add integration tests for Databricks UC Volumes ingestion queries
+- Retries:
+  - Add `_retry_max_redirects` config
+  - Set `_enable_v3_retries=True` and warn if users override it
+- Security: bump minimum pyarrow version to 14.0.1 (CVE-2023-47248)
 
 ## 2.9.3 (2023-08-24)
 

{databricks_sql_connector-3.0.0b1 → databricks_sql_connector-3.0.1}/PKG-INFO

@@ -1,29 +1,28 @@
 Metadata-Version: 2.1
 Name: databricks-sql-connector
-Version: 3.0.0b1
+Version: 3.0.1
 Summary: Databricks SQL Connector for Python
 License: Apache-2.0
 Author: Databricks
 Author-email: databricks-sql-connector-maintainers@databricks.com
-Requires-Python: >=3.7.1,<4.0.0
+Requires-Python: >=3.8.0,<4.0.0
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: alembic
 Provides-Extra: sqlalchemy
 Requires-Dist: alembic (>=1.0.11,<2.0.0) ; extra == "alembic"
 Requires-Dist: lz4 (>=4.0.2,<5.0.0)
-Requires-Dist: numpy (>=1.16.6) ; python_version >= "3.7" and python_version < "3.11"
+Requires-Dist: numpy (>=1.16.6) ; python_version >= "3.8" and python_version < "3.11"
 Requires-Dist: numpy (>=1.23.4) ; python_version >= "3.11"
 Requires-Dist: oauthlib (>=3.1.0,<4.0.0)
 Requires-Dist: openpyxl (>=3.0.10,<4.0.0)
-Requires-Dist: pandas (>=1.2.5,<1.4.0) ; python_version >= "3.7" and python_version < "3.8"
 Requires-Dist: pandas (>=1.2.5,<3.0.0) ; python_version >= "3.8"
-Requires-Dist: pyarrow (>=10.0.1) ; python_version >= "3.11"
-Requires-Dist: pyarrow (>=6.0.0) ; python_version >= "3.7" and python_version < "3.11"
+Requires-Dist: pyarrow (>=14.0.1,<15.0.0)
 Requires-Dist: requests (>=2.18.1,<3.0.0)
 Requires-Dist: sqlalchemy (>=2.0.21) ; extra == "sqlalchemy" or extra == "alembic"
 Requires-Dist: thrift (>=0.16.0,<0.17.0)
@@ -45,7 +44,7 @@ You are welcome to file an issue here for general use cases. You can also contac
 
 ## Requirements
 
-Python 3.7 or above is required.
+Python 3.8 or above is required.
 
 ## Documentation
 
@@ -81,8 +80,7 @@ connection = sql.connect(
   access_token=access_token)
 
 cursor = connection.cursor()
-
-cursor.execute('SELECT * FROM RANGE(10)')
+cursor.execute('SELECT :param `p`, * FROM RANGE(10)', {"param": "foo"})
 result = cursor.fetchall()
 for row in result:
   print(row)

{databricks_sql_connector-3.0.0b1 → databricks_sql_connector-3.0.1}/README.md

@@ -11,7 +11,7 @@ You are welcome to file an issue here for general use cases. You can also contac
 
 ## Requirements
 
-Python 3.7 or above is required.
+Python 3.8 or above is required.
 
 ## Documentation
 
@@ -47,8 +47,7 @@ connection = sql.connect(
   access_token=access_token)
 
 cursor = connection.cursor()
-
-cursor.execute('SELECT * FROM RANGE(10)')
+cursor.execute('SELECT :param `p`, * FROM RANGE(10)', {"param": "foo"})
 result = cursor.fetchall()
 for row in result:
   print(row)

{databricks_sql_connector-3.0.0b1 → databricks_sql_connector-3.0.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "databricks-sql-connector"
-version = "3.0.0b1"
+version = "3.0.1"
 description = "Databricks SQL Connector for Python"
 authors = ["Databricks <databricks-sql-connector-maintainers@databricks.com>"]
 license = "Apache-2.0"
@@ -9,21 +9,18 @@ packages = [{ include = "databricks", from = "src" }]
 include = ["CHANGELOG.md"]
 
 [tool.poetry.dependencies]
-python = "^3.7.1"
+python = "^3.8.0"
 thrift = "^0.16.0"
 pandas = [
-    { version = ">=1.2.5,<1.4.0", python = ">=3.7,<3.8" },
-    { version = ">=1.2.5,<3.0.0", python = ">=3.8" },
-]
-pyarrow = [
-    { version = ">=6.0.0", python = ">=3.7,<3.11" },
-    { version = ">=10.0.1", python = ">=3.11" },
+    { version = ">=1.2.5,<3.0.0", python = ">=3.8" }
 ]
+pyarrow = "^14.0.1"
+
 lz4 = "^4.0.2"
 requests = "^2.18.1"
 oauthlib = "^3.1.0"
 numpy = [
-    { version = ">=1.16.6", python = ">=3.7,<3.11" },
+    { version = ">=1.16.6", python = ">=3.8,<3.11" },
     { version = ">=1.23.4", python = ">=3.11" },
 ]
 sqlalchemy = { version = ">=2.0.21", optional = true }

{databricks_sql_connector-3.0.0b1 → databricks_sql_connector-3.0.1}/src/databricks/sql/__init__.py

@@ -6,9 +6,7 @@ from databricks.sql.exc import *
 apilevel = "2.0"
 threadsafety = 1  # Threads may share the module, but not connections.
 
-# Python extended format codes, e.g. ...WHERE name=%(name)s
-# Note that when we switch to ParameterApproach.NATIVE, paramstyle will be `named`
-paramstyle = "pyformat"
+paramstyle = "named"
 
 
 class DBAPITypeObject(object):
@@ -31,7 +29,7 @@ DATETIME = DBAPITypeObject("timestamp")
 DATE = DBAPITypeObject("date")
 ROWID = DBAPITypeObject()
 
-__version__ = "3.0.0b1"
+__version__ = "3.0.1"
 USER_AGENT_NAME = "PyDatabricksSqlConnector"
 
 # These two functions are pyhive legacy

{databricks_sql_connector-3.0.0b1 → databricks_sql_connector-3.0.1}/src/databricks/sql/auth/retry.py

@@ -30,6 +30,7 @@ class CommandType(Enum):
     EXECUTE_STATEMENT = "ExecuteStatement"
     CLOSE_SESSION = "CloseSession"
     CLOSE_OPERATION = "CloseOperation"
+    GET_OPERATION_STATUS = "GetOperationStatus"
     OTHER = "Other"
 
     @classmethod
@@ -314,9 +315,9 @@ class DatabricksRetryPolicy(Retry):
             2. The request received a 501 (Not Implemented) status code
                Because this request can never succeed.
             3. The request received a 404 (Not Found) code and the request CommandType
-               was CloseSession or CloseOperation. This code indicates that the session
-               or cursor was already closed. Further retries will always return the same
-               code.
+               was GetOperationStatus, CloseSession or CloseOperation. This code indicates
+               that the command, session or cursor was already closed. Further retries will
+               always return the same code.
             4. The request CommandType was ExecuteStatement and the HTTP code does not
                appear in the default status_forcelist or force_dangerous_codes list. By
                default, this means ExecuteStatement is only retried for codes 429 and 503.
@@ -343,6 +344,13 @@ class DatabricksRetryPolicy(Retry):
         if not self._is_method_retryable(method):  # type: ignore
             return False, "Only POST requests are retried"
 
+        # Request failed with 404 and was a GetOperationStatus. This is not recoverable. Don't retry.
+        if status_code == 404 and self.command_type == CommandType.GET_OPERATION_STATUS:
+            return (
+                False,
+                "GetOperationStatus received 404 code from Databricks. Operation was canceled.",
+            )
+
         # Request failed with 404 because CloseSession returns 404 if you repeat the request.
         if (
             status_code == 404

{databricks_sql_connector-3.0.0b1 → databricks_sql_connector-3.0.1}/src/databricks/sql/client.py

@@ -1,10 +1,11 @@
-from typing import Dict, Tuple, List, Optional, Any, Union
+from typing import Dict, Tuple, List, Optional, Any, Union, Sequence
 
 import pandas
 import pyarrow
 import requests
 import json
 import os
+import decimal
 
 from databricks.sql import __version__
 from databricks.sql import *
@@ -18,10 +19,21 @@ from databricks.sql.thrift_backend import ThriftBackend
 from databricks.sql.utils import (
     ExecuteResponse,
     ParamEscaper,
-    named_parameters_to_tsparkparams,
     inject_parameters,
+    transform_paramstyle,
+)
+from databricks.sql.parameters.native import (
+    DbsqlParameterBase,
+    TDbsqlParameter,
+    TParameterDict,
+    TParameterSequence,
+    TParameterCollection,
+    ParameterStructure,
+    dbsql_parameter_from_primitive,
     ParameterApproach,
 )
+
+
 from databricks.sql.types import Row
 from databricks.sql.auth.auth import get_python_sql_connector_auth_provider
 from databricks.sql.experimental.oauth_persistence import OAuthPersistence
@@ -49,6 +61,7 @@ class Connection:
         session_configuration: Dict[str, Any] = None,
         catalog: Optional[str] = None,
         schema: Optional[str] = None,
+        _use_arrow_native_complex_types: Optional[bool] = True,
         **kwargs,
     ) -> None:
         """
@@ -62,11 +75,13 @@ class Connection:
                 Http Bearer access token, e.g. Databricks Personal Access Token.
                 Unless if you use auth_type=`databricks-oauth` you need to pass `access_token.
                 Examples:
+                        ```
                          connection = sql.connect(
                             server_hostname='dbc-12345.staging.cloud.databricks.com',
                             http_path='sql/protocolv1/o/6789/12abc567',
                             access_token='dabpi12345678'
                          )
+                        ```
             :param http_headers: An optional list of (k, v) pairs that will be set as Http headers on every request
             :param session_configuration: An optional dictionary of Spark session parameters. Defaults to None.
                 Execute the SQL command `SET -v` to get a full list of available commands.
@@ -74,12 +89,12 @@ class Connection:
             :param schema: An optional initial schema to use. Requires DBR version 9.0+
 
         Other Parameters:
-            use_inline_params: `boolean`, optional (default is True)
+            use_inline_params: `boolean` | str, optional (default is False)
                 When True, parameterized calls to cursor.execute() will try to render parameter values inline with the
                 query text instead of using native bound parameters supported in DBR 14.1 and above. This connector will attempt to
-                sanitise parameterized inputs to prevent SQL injection. Before you can switch this to False, you must
-                update your queries to use the PEP-249 `named` paramstyle instead of the `pyformat` paramstyle used
-                in INLINE mode.
+                sanitise parameterized inputs to prevent SQL injection.  The inline parameter approach is maintained for
+                legacy purposes and will be deprecated in a future release. When this parameter is `True` you will see
+                a warning log message. To suppress this log message, set `use_inline_params="silent"`.
             auth_type: `str`, optional
                 `databricks-oauth` : to use oauth with fine-grained permission scopes, set to `databricks-oauth`.
                 This is currently in private preview for Databricks accounts on AWS.
@@ -127,6 +142,7 @@ class Connection:
                 own implementation of OAuthPersistence.
 
                 Examples:
+                ```
                         # for development only
                         from databricks.sql.experimental.oauth_persistence import DevOnlyFilePersistence
 
@@ -136,8 +152,14 @@ class Connection:
                             auth_type="databricks-oauth",
                             experimental_oauth_persistence=DevOnlyFilePersistence("~/dev-oauth.json")
                         )
-
-
+                ```
+            :param _use_arrow_native_complex_types: `bool`, optional
+                Controls whether a complex type field value is returned as a string or as a native Arrow type. Defaults to True.
+                When True:
+                    MAP is returned as List[Tuple[str, Any]]
+                    STRUCT is returned as Dict[str, Any]
+                    ARRAY is returned as numpy.ndarray
+                When False, complex types are returned as a strings. These are generally deserializable as JSON.
         """
 
         # Internal arguments in **kwargs:
@@ -168,9 +190,6 @@ class Connection:
         # _disable_pandas
         #  In case the deserialisation through pandas causes any issues, it can be disabled with
         #  this flag.
-        # _use_arrow_native_complex_types
-        # DBR will return native Arrow types for structs, arrays and maps instead of Arrow strings
-        # (True by default)
         # _use_arrow_native_decimals
         # Databricks runtime will return native Arrow types for decimals instead of Arrow strings
         # (True by default)
@@ -209,6 +228,7 @@ class Connection:
             http_path,
             (http_headers or []) + base_headers,
             auth_provider,
+            _use_arrow_native_complex_types=_use_arrow_native_complex_types,
             **kwargs,
         )
 
@@ -222,8 +242,36 @@ class Connection:
         logger.info("Successfully opened session " + str(self.get_session_id_hex()))
         self._cursors = []  # type: List[Cursor]
 
-        self._suppress_inline_warning = "use_inline_params" in kwargs
-        self.use_inline_params = kwargs.get("use_inline_params", True)
+        self.use_inline_params = self._set_use_inline_params_with_warning(
+            kwargs.get("use_inline_params", False)
+        )
+
+    def _set_use_inline_params_with_warning(self, value: Union[bool, str]):
+        """Valid values are True, False, and "silent"
+
+        False: Use native parameters
+        True: Use inline parameters and log a warning
+        "silent": Use inline parameters and don't log a warning
+        """
+
+        if value is False:
+            return False
+
+        if value not in [True, "silent"]:
+            raise ValueError(
+                f"Invalid value for use_inline_params: {value}. "
+                + 'Valid values are True, False, and "silent"'
+            )
+
+        if value is True:
+            logger.warning(
+                "Parameterised queries executed with this client will use the inline parameter approach."
+                "This approach will be deprecated in a future release. Consider using native parameters."
+                "Learn more: https://github.com/databricks/databricks-sql-python/tree/main/docs/parameters.md"
+                'To suppress this warning, set use_inline_params="silent"'
+            )
+
+        return value
 
     def __enter__(self):
         return self
@@ -377,7 +425,7 @@ class Cursor:
             raise Error("There is no active result set")
 
     def _determine_parameter_approach(
-        self, params: Optional[Union[List, Dict[str, Any]]] = None
+        self, params: Optional[TParameterCollection]
     ) -> ParameterApproach:
         """Encapsulates the logic for choosing whether to send parameters in native vs inline mode
 
@@ -394,34 +442,60 @@ class Cursor:
         if params is None:
             return ParameterApproach.NONE
 
-        server_supports_native_approach = (
-            self.connection.server_parameterized_queries_enabled(
-                self.connection.protocol_version
-            )
-        )
-
         if self.connection.use_inline_params:
-            if (
-                server_supports_native_approach
-                and not self.connection._suppress_inline_warning
-            ):
-                logger.warning(
-                    "This query will be executed with inline parameters."
-                    "Consider using native parameters."
-                    "Learn more: https://github.com/databricks/databricks-sql-python/tree/main/docs/parameters.md"
-                    "To suppress this warning, pass use_inline_params=True when creating the connection."
-                )
             return ParameterApproach.INLINE
 
-        elif server_supports_native_approach:
+        else:
             return ParameterApproach.NATIVE
+
+    def _all_dbsql_parameters_are_named(self, params: List[TDbsqlParameter]) -> bool:
+        """Return True if all members of the list have a non-null .name attribute"""
+        return all([i.name is not None for i in params])
+
+    def _normalize_tparametersequence(
+        self, params: TParameterSequence
+    ) -> List[TDbsqlParameter]:
+        """Retains the same order as the input list."""
+
+        output: List[TDbsqlParameter] = []
+        for p in params:
+            if isinstance(p, DbsqlParameterBase):
+                output.append(p)  # type: ignore
+            else:
+                output.append(dbsql_parameter_from_primitive(value=p))  # type: ignore
+
+        return output
+
+    def _normalize_tparameterdict(
+        self, params: TParameterDict
+    ) -> List[TDbsqlParameter]:
+        return [
+            dbsql_parameter_from_primitive(value=value, name=name)
+            for name, value in params.items()
+        ]
+
+    def _normalize_tparametercollection(
+        self, params: Optional[TParameterCollection]
+    ) -> List[TDbsqlParameter]:
+        if params is None:
+            return []
+        if isinstance(params, dict):
+            return self._normalize_tparameterdict(params)
+        if isinstance(params, Sequence):
+            return self._normalize_tparametersequence(list(params))
+
+    def _determine_parameter_structure(
+        self,
+        parameters: List[TDbsqlParameter],
+    ) -> ParameterStructure:
+        all_named = self._all_dbsql_parameters_are_named(parameters)
+        if all_named:
+            return ParameterStructure.NAMED
         else:
-            raise NotSupportedError(
-                "Parameterized operations are not supported by this server. DBR 14.1 is required."
-            )
+            return ParameterStructure.POSITIONAL
 
     def _prepare_inline_parameters(
-        self, stmt: str, params: Optional[Union[List, Dict[str, Any]]]
+        self, stmt: str, params: Optional[Union[Sequence, Dict[str, Any]]]
     ) -> Tuple[str, List]:
         """Return a statement and list of native parameters to be passed to thrift_backend for execution
 
@@ -446,7 +520,10 @@ class Cursor:
         return rendered_statement, NO_NATIVE_PARAMS
 
     def _prepare_native_parameters(
-        self, stmt: str, params: Optional[Union[List[Any], Dict[str, Any]]]
+        self,
+        stmt: str,
+        params: List[TDbsqlParameter],
+        param_structure: ParameterStructure,
     ) -> Tuple[str, List[TSparkParameter]]:
         """Return a statement and a list of native parameters to be passed to thrift_backend for execution
 
@@ -466,9 +543,12 @@ class Cursor:
         """
 
         stmt = stmt
-        params = named_parameters_to_tsparkparams(params)  # type: ignore
+        output = [
+            p.as_tspark_param(named=param_structure == ParameterStructure.NAMED)
+            for p in params
+        ]
 
-        return stmt, params
+        return stmt, output
 
     def _close_and_clear_active_result_set(self):
         try:
@@ -627,24 +707,27 @@ class Cursor:
     def execute(
         self,
         operation: str,
-        parameters: Optional[Union[List[Any], Dict[str, Any]]] = None,
+        parameters: Optional[TParameterCollection] = None,
     ) -> "Cursor":
         """
         Execute a query and wait for execution to complete.
 
         The parameterisation behaviour of this method depends on which parameter approach is used:
-            - With INLINE mode (default), parameters are rendered inline with the query text
-            - With NATIVE mode, parameters are sent to the server separately for binding
+            - With INLINE mode, parameters are rendered inline with the query text
+            - With NATIVE mode (default), parameters are sent to the server separately for binding
 
         This behaviour is controlled by the `use_inline_params` argument passed when building a connection.
 
-        The syntax for these approaches is different:
+        The paramstyle for these approaches is different:
 
-        If the connection was instantiated with use_inline_params=False, then parameters
-        should be given in PEP-249 `named` paramstyle like :param_name
+        If the connection was instantiated with use_inline_params=False (default), then parameters
+        should be given in PEP-249 `named` paramstyle like :param_name. Parameters passed by positionally
+        are indicated using a `?` in the query text.
 
-        If the connection was instantiated with use_inline_params=True (default), then parameters
-        should be given in PEP-249 `pyformat` paramstyle like %(param_name)s
+        If the connection was instantiated with use_inline_params=True, then parameters
+        should be given in PEP-249 `pyformat` paramstyle like %(param_name)s. Parameters passed by positionally
+        are indicated using a `%s` marker in the query. Note: this approach is not recommended as it can break
+        your SQL query syntax and will be removed in a future release.
 
         ```python
         inline_operation = "SELECT * FROM table WHERE field = %(some_value)s"
@@ -668,8 +751,13 @@ class Cursor:
                 operation, parameters
             )
         elif param_approach == ParameterApproach.NATIVE:
+            normalized_parameters = self._normalize_tparametercollection(parameters)
+            param_structure = self._determine_parameter_structure(normalized_parameters)
+            transformed_operation = transform_paramstyle(
+                operation, normalized_parameters, param_structure  # type: ignore
+            )
             prepared_operation, prepared_params = self._prepare_native_parameters(
-                operation, parameters
+                transformed_operation, normalized_parameters, param_structure
             )
 
         self._check_not_closed()

databricks_sql_connector-3.0.1/src/databricks/sql/parameters/__init__.py

@@ -0,0 +1,15 @@
+from databricks.sql.parameters.native import (
+    IntegerParameter,
+    StringParameter,
+    BigIntegerParameter,
+    BooleanParameter,
+    DateParameter,
+    DoubleParameter,
+    FloatParameter,
+    VoidParameter,
+    SmallIntParameter,
+    TimestampParameter,
+    TimestampNTZParameter,
+    TinyIntParameter,
+    DecimalParameter,
+)