databricks-sql-connector 3.0.0__tar.gz → 3.0.1__tar.gz

{databricks_sql_connector-3.0.0 → databricks_sql_connector-3.0.1}/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Release History
 
+## 3.0.1 (2023-12-01)
+
+- Other: updated docstring comment about default parameterization approach (#287)
+- Other: added tests for reading complex types and revised docstrings and type hints (#293)
+- Fix: SQLAlchemy dialect raised DeprecationWarning due to `dbapi` classmethod (#294)
+- Fix: SQLAlchemy dialect could not reflect TIMESTAMP_NTZ columns (#296)
+
 ## 3.0.0 (2023-11-17)
 
 - Remove support for Python 3.7

{databricks_sql_connector-3.0.0 → databricks_sql_connector-3.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: databricks-sql-connector
-Version: 3.0.0
+Version: 3.0.1
 Summary: Databricks SQL Connector for Python
 License: Apache-2.0
 Author: Databricks

{databricks_sql_connector-3.0.0 → databricks_sql_connector-3.0.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "databricks-sql-connector"
-version = "3.0.0"
+version = "3.0.1"
 description = "Databricks SQL Connector for Python"
 authors = ["Databricks <databricks-sql-connector-maintainers@databricks.com>"]
 license = "Apache-2.0"

{databricks_sql_connector-3.0.0 → databricks_sql_connector-3.0.1}/src/databricks/sql/__init__.py

@@ -29,7 +29,7 @@ DATETIME = DBAPITypeObject("timestamp")
 DATE = DBAPITypeObject("date")
 ROWID = DBAPITypeObject()
 
-__version__ = "3.0.0"
+__version__ = "3.0.1"
 USER_AGENT_NAME = "PyDatabricksSqlConnector"
 
 # These two functions are pyhive legacy

{databricks_sql_connector-3.0.0 → databricks_sql_connector-3.0.1}/src/databricks/sql/client.py

@@ -61,6 +61,7 @@ class Connection:
         session_configuration: Dict[str, Any] = None,
         catalog: Optional[str] = None,
         schema: Optional[str] = None,
+        _use_arrow_native_complex_types: Optional[bool] = True,
         **kwargs,
     ) -> None:
         """
@@ -152,8 +153,13 @@ class Connection:
                             experimental_oauth_persistence=DevOnlyFilePersistence("~/dev-oauth.json")
                         )
                 ```
-
-
+            :param _use_arrow_native_complex_types: `bool`, optional
+                Controls whether a complex type field value is returned as a string or as a native Arrow type. Defaults to True.
+                When True:
+                    MAP is returned as List[Tuple[str, Any]]
+                    STRUCT is returned as Dict[str, Any]
+                    ARRAY is returned as numpy.ndarray
+                When False, complex types are returned as a strings. These are generally deserializable as JSON.
         """
 
         # Internal arguments in **kwargs:
@@ -184,9 +190,6 @@ class Connection:
         # _disable_pandas
         #  In case the deserialisation through pandas causes any issues, it can be disabled with
         #  this flag.
-        # _use_arrow_native_complex_types
-        # DBR will return native Arrow types for structs, arrays and maps instead of Arrow strings
-        # (True by default)
         # _use_arrow_native_decimals
         # Databricks runtime will return native Arrow types for decimals instead of Arrow strings
         # (True by default)
@@ -225,6 +228,7 @@ class Connection:
             http_path,
             (http_headers or []) + base_headers,
             auth_provider,
+            _use_arrow_native_complex_types=_use_arrow_native_complex_types,
             **kwargs,
         )
 
@@ -709,18 +713,18 @@ class Cursor:
         Execute a query and wait for execution to complete.
 
         The parameterisation behaviour of this method depends on which parameter approach is used:
-            - With INLINE mode (default), parameters are rendered inline with the query text
-            - With NATIVE mode, parameters are sent to the server separately for binding
+            - With INLINE mode, parameters are rendered inline with the query text
+            - With NATIVE mode (default), parameters are sent to the server separately for binding
 
         This behaviour is controlled by the `use_inline_params` argument passed when building a connection.
 
         The paramstyle for these approaches is different:
 
-        If the connection was instantiated with use_inline_params=False, then parameters
+        If the connection was instantiated with use_inline_params=False (default), then parameters
         should be given in PEP-249 `named` paramstyle like :param_name. Parameters passed by positionally
         are indicated using a `?` in the query text.
 
-        If the connection was instantiated with use_inline_params=True (default), then parameters
+        If the connection was instantiated with use_inline_params=True, then parameters
         should be given in PEP-249 `pyformat` paramstyle like %(param_name)s. Parameters passed by positionally
         are indicated using a `%s` marker in the query. Note: this approach is not recommended as it can break
         your SQL query syntax and will be removed in a future release.

{databricks_sql_connector-3.0.0 → databricks_sql_connector-3.0.1}/src/databricks/sqlalchemy/_parse.py

@@ -5,6 +5,8 @@ import sqlalchemy
 from sqlalchemy.engine import CursorResult
 from sqlalchemy.engine.interfaces import ReflectedColumn
 
+from databricks.sqlalchemy import _types as type_overrides
+
 """
 This module contains helper functions that can parse the contents
 of metadata and exceptions received from DBR. These are mostly just
@@ -293,7 +295,8 @@ GET_COLUMNS_TYPE_MAP = {
     "struct": sqlalchemy.types.String,
     "uniontype": sqlalchemy.types.String,
     "decimal": sqlalchemy.types.Numeric,
-    "timestamp": sqlalchemy.types.DateTime,
+    "timestamp": type_overrides.TIMESTAMP,
+    "timestamp_ntz": type_overrides.TIMESTAMP_NTZ,
     "date": sqlalchemy.types.Date,
 }
 

{databricks_sql_connector-3.0.0 → databricks_sql_connector-3.0.1}/src/databricks/sqlalchemy/base.py

@@ -82,7 +82,7 @@ class DatabricksDialect(default.DefaultDialect):
     EMPTY_FK = EMPTY_INDEX = []
 
     @classmethod
-    def dbapi(cls):
+    def import_dbapi(cls):
         return sql
 
     def _force_paramstyle_to_native_mode(self):

{databricks_sql_connector-3.0.0 → databricks_sql_connector-3.0.1}/src/databricks/sqlalchemy/test_local/e2e/test_basic.py

@@ -1,21 +1,24 @@
-import os, datetime, decimal
-import pytest
+import datetime
+import decimal
+import os
+from typing import Tuple, Union
 from unittest import skipIf
+
+import pytest
 from sqlalchemy import (
-    create_engine,
-    select,
-    insert,
     Column,
     MetaData,
     Table,
     Text,
+    create_engine,
+    insert,
+    select,
     text,
 )
-from sqlalchemy.orm import Session, DeclarativeBase, Mapped, mapped_column
-from sqlalchemy.types import SMALLINT, Integer, BOOLEAN, String, DECIMAL, Date
 from sqlalchemy.engine import Engine
-
-from typing import Tuple, Union
+from sqlalchemy.engine.reflection import Inspector
+from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column
+from sqlalchemy.types import BOOLEAN, DECIMAL, Date, DateTime, Integer, String
 
 try:
     from sqlalchemy.orm import declarative_base
@@ -344,8 +347,6 @@ def test_dialect_type_mappings(db_engine, metadata_obj: MetaData):
 def test_inspector_smoke_test(samples_engine: Engine):
     """It does not appear that 3L namespace is supported here"""
 
-    from sqlalchemy.engine.reflection import Inspector
-
     schema, table = "nyctaxi", "trips"
 
     try:
@@ -431,3 +432,34 @@ def test_user_agent_adjustment(db_engine):
     c2.close()
 
     assert same_ua, f"User agents didn't match \n {ua1} \n {ua2}"
+
+
+@pytest.fixture
+def sample_table(metadata_obj: MetaData, db_engine: Engine):
+    """This fixture creates a sample table and cleans it up after the test is complete."""
+    from databricks.sqlalchemy._parse import GET_COLUMNS_TYPE_MAP
+
+    table_name = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s"))
+
+    args = [
+        Column(colname, coltype) for colname, coltype in GET_COLUMNS_TYPE_MAP.items()
+    ]
+
+    SampleTable = Table(table_name, metadata_obj, *args)
+
+    metadata_obj.create_all(db_engine)
+
+    yield table_name
+
+    metadata_obj.drop_all(db_engine)
+
+
+def test_get_columns(db_engine, sample_table: str):
+    """Created after PECO-1297 and Github Issue #295 to verify that get_columsn behaves like it should for all known SQLAlchemy types"""
+
+    inspector = Inspector.from_engine(db_engine)
+
+    # this raises an exception if `parse_column_info_from_tgetcolumnsresponse` fails a lookup
+    columns = inspector.get_columns(sample_table)
+
+    assert True