apache-airflow-providers-common-sql 1.26.0rc1__py3-none-any.whl → 1.27.0rc1__py3-none-any.whl

airflow/providers/common/sql/__init__.py

@@ -29,11 +29,11 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "1.26.0"
+__version__ = "1.27.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
-    "2.9.0"
+    "2.10.0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-common-sql:{__version__}` needs Apache Airflow 2.9.0+"
+        f"The package `apache-airflow-providers-common-sql:{__version__}` needs Apache Airflow 2.10.0+"
     )

airflow/providers/common/sql/hooks/sql.py

@@ -37,8 +37,8 @@ import sqlparse
 from deprecated import deprecated
 from methodtools import lru_cache
 from more_itertools import chunked
-from sqlalchemy import create_engine
-from sqlalchemy.engine import Inspector, make_url
+from sqlalchemy import create_engine, inspect
+from sqlalchemy.engine import make_url
 from sqlalchemy.exc import ArgumentError, NoSuchModuleError
 from typing_extensions import Literal
 
@@ -56,7 +56,7 @@ from airflow.utils.module_loading import import_string
 if TYPE_CHECKING:
     from pandas import DataFrame
     from polars import DataFrame as PolarsDataFrame
-    from sqlalchemy.engine import URL
+    from sqlalchemy.engine import URL, Engine, Inspector
 
     from airflow.models import Connection
     from airflow.providers.openlineage.extractors import OperatorLineage
@@ -307,7 +307,7 @@ class DbApiHook(BaseHook):
             msg = "`sqlalchemy_url` property should be implemented in the provider subclass."
         raise NotImplementedError(msg)
 
-    def get_sqlalchemy_engine(self, engine_kwargs=None):
+    def get_sqlalchemy_engine(self, engine_kwargs=None) -> Engine:
         """
         Get an sqlalchemy_engine object.
 
@@ -328,7 +328,7 @@ class DbApiHook(BaseHook):
 
     @property
     def inspector(self) -> Inspector:
-        return Inspector.from_engine(self.get_sqlalchemy_engine())
+        return inspect(self.get_sqlalchemy_engine())
 
     @cached_property
     def dialect_name(self) -> str:
@@ -890,7 +890,14 @@ class DbApiHook(BaseHook):
                         )
                         sql = self._generate_insert_sql(table, values[0], target_fields, replace, **kwargs)
                         self.log.debug("Generated sql: %s", sql)
-                        cur.executemany(sql, values)
+
+                        try:
+                            cur.executemany(sql, values)
+                        except Exception as e:
+                            self.log.error("Generated sql: %s", sql)
+                            self.log.error("Parameters: %s", values)
+                            raise e
+
                         conn.commit()
                         nb_rows += len(chunked_rows)
                         self.log.info("Loaded %s rows into %s so far", nb_rows, table)
@@ -899,7 +906,14 @@ class DbApiHook(BaseHook):
                         values = self._serialize_cells(row, conn)
                         sql = self._generate_insert_sql(table, values, target_fields, replace, **kwargs)
                         self.log.debug("Generated sql: %s", sql)
-                        cur.execute(sql, values)
+
+                        try:
+                            cur.execute(sql, values)
+                        except Exception as e:
+                            self.log.error("Generated sql: %s", sql)
+                            self.log.error("Parameters: %s", values)
+                            raise e
+
                         if commit_every and i % commit_every == 0:
                             conn.commit()
                             self.log.info("Loaded %s rows into %s so far", i, table)

airflow/providers/common/sql/hooks/sql.pyi

@@ -37,8 +37,10 @@ from functools import cached_property as cached_property
 from typing import Any, Callable, Protocol, TypeVar, overload
 
 from _typeshed import Incomplete as Incomplete
-from pandas import DataFrame as DataFrame
-from sqlalchemy.engine import URL as URL, Inspector as Inspector
+from pandas import DataFrame as PandasDataFrame
+from polars import DataFrame as PolarsDataFrame
+from sqlalchemy.engine import URL as URL, Engine as Engine, Inspector as Inspector
+from typing_extensions import Literal
 
 from airflow.hooks.base import BaseHook as BaseHook
 from airflow.models import Connection as Connection
@@ -93,7 +95,7 @@ class DbApiHook(BaseHook):
     def get_uri(self) -> str: ...
     @property
     def sqlalchemy_url(self) -> URL: ...
-    def get_sqlalchemy_engine(self, engine_kwargs: Incomplete | None = None): ...
+    def get_sqlalchemy_engine(self, engine_kwargs: Incomplete | None = None) -> Engine: ...
     @property
     def inspector(self) -> Inspector: ...
     @cached_property
@@ -105,10 +107,65 @@ class DbApiHook(BaseHook):
     def get_reserved_words(self, dialect_name: str) -> set[str]: ...
     def get_pandas_df(
         self, sql, parameters: list | tuple | Mapping[str, Any] | None = None, **kwargs
-    ) -> DataFrame: ...
+    ) -> PandasDataFrame: ...
     def get_pandas_df_by_chunks(
         self, sql, parameters: list | tuple | Mapping[str, Any] | None = None, *, chunksize: int, **kwargs
-    ) -> Generator[DataFrame, None, None]: ...
+    ) -> Generator[PandasDataFrame, None, None]: ...
+    @overload
+    def get_df(
+        self,
+        sql: str | list[str],
+        parameters: list | tuple | Mapping[str, Any] | None = None,
+        *,
+        df_type: Literal["pandas"] = "pandas",
+        **kwargs: Any,
+    ) -> PandasDataFrame: ...
+    @overload
+    def get_df(
+        self,
+        sql: str | list[str],
+        parameters: list | tuple | Mapping[str, Any] | None = None,
+        *,
+        df_type: Literal["polars"] = "polars",
+        **kwargs: Any,
+    ) -> PolarsDataFrame: ...
+    @overload
+    def get_df(  # fallback overload
+        self,
+        sql: str | list[str],
+        parameters: list | tuple | Mapping[str, Any] | None = None,
+        *,
+        df_type: Literal["pandas", "polars"] = "pandas",
+    ) -> PandasDataFrame | PolarsDataFrame: ...
+    @overload
+    def get_df_by_chunks(
+        self,
+        sql,
+        parameters: list | tuple | Mapping[str, Any] | None = None,
+        *,
+        chunksize: int,
+        df_type: Literal["pandas"] = "pandas",
+        **kwargs,
+    ) -> Generator[PandasDataFrame, None, None]: ...
+    @overload
+    def get_df_by_chunks(
+        self,
+        sql,
+        parameters: list | tuple | Mapping[str, Any] | None = None,
+        *,
+        chunksize: int,
+        df_type: Literal["polars"],
+        **kwargs,
+    ) -> Generator[PolarsDataFrame, None, None]: ...
+    @overload
+    def get_df_by_chunks(  # fallback overload
+        self,
+        sql,
+        parameters: list | tuple | Mapping[str, Any] | None = None,
+        *,
+        chunksize: int,
+        df_type: Literal["pandas", "polars"] = "pandas",
+    ) -> Generator[PandasDataFrame | PolarsDataFrame, None, None]: ...
     def get_records(
         self, sql: str | list[str], parameters: Iterable | Mapping[str, Any] | None = None
     ) -> Any: ...

airflow/providers/common/sql/operators/sql.py

@@ -161,7 +161,12 @@ class BaseSQLOperator(BaseOperator):
         :param hook_params: hook parameters
         :return: default hook for this connection
         """
+        hook_params = hook_params or {}
         connection = BaseHook.get_connection(conn_id)
+        conn_params = connection.extra_dejson
+        for conn_param in conn_params:
+            if conn_param not in hook_params:
+                hook_params[conn_param] = conn_params[conn_param]
         return connection.get_hook(hook_params=hook_params)
 
     @cached_property

{apache_airflow_providers_common_sql-1.26.0rc1.dist-info → apache_airflow_providers_common_sql-1.27.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-common-sql
-Version: 1.26.0rc1
+Version: 1.27.0rc1
 Summary: Provider package apache-airflow-providers-common-sql for Apache Airflow
 Keywords: airflow-provider,common.sql,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,7 +20,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow>=2.9.0rc0
+Requires-Dist: apache-airflow>=2.10.0rc1
 Requires-Dist: sqlparse>=0.5.1
 Requires-Dist: more-itertools>=9.0.0
 Requires-Dist: methodtools>=0.4.7
@@ -28,8 +28,8 @@ Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
 Requires-Dist: pandas>=2.1.2,<2.2 ; extra == "pandas"
 Requires-Dist: polars>=1.26.0 ; extra == "polars"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.26.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.26.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.27.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.27.0
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -63,7 +63,7 @@ Provides-Extra: polars
 
 Package ``apache-airflow-providers-common-sql``
 
-Release: ``1.26.0``
+Release: ``1.27.0``
 
 
 `Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
@@ -76,7 +76,7 @@ This is a provider package for ``common.sql`` provider. All classes for this pro
 are in ``airflow.providers.common.sql`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.26.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.27.0/>`_.
 
 Installation
 ------------
@@ -93,7 +93,7 @@ Requirements
 ==================  ==================
 PIP package         Version required
 ==================  ==================
-``apache-airflow``  ``>=2.9.0``
+``apache-airflow``  ``>=2.10.0``
 ``sqlparse``        ``>=0.5.1``
 ``more-itertools``  ``>=9.0.0``
 ``methodtools``     ``>=0.4.7``
@@ -119,5 +119,5 @@ Dependent package
 ==============================================================================================================  ===============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.26.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.27.0/changelog.html>`_.
 

{apache_airflow_providers_common_sql-1.26.0rc1.dist-info → apache_airflow_providers_common_sql-1.27.0rc1.dist-info}/RECORD

@@ -1,6 +1,6 @@
 airflow/providers/common/sql/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
 airflow/providers/common/sql/README_API.md,sha256=CxhaS8EedZ4dcbLUPC4-GLCMaY3OH96oHxXttUGU06E,5932
-airflow/providers/common/sql/__init__.py,sha256=Dq1VrbX8fQ-LThRC05hWOmMNtrEghP-rqLUo51LTr1Y,1498
+airflow/providers/common/sql/__init__.py,sha256=1fjlO-pK_nT17oF_-bhTA0xjWEMvmFYDYrXVDAfJy4M,1500
 airflow/providers/common/sql/get_provider_info.py,sha256=xCPXLKFA_1ilhGa0aB3E9ggdHtn9Do7Eb469begpZag,2767
 airflow/providers/common/sql/get_provider_info.pyi,sha256=0mydJPGQScnPpoa9-ohHVJFngFH6Lsk22KS243PE-gw,1596
 airflow/providers/common/sql/dialects/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
@@ -12,19 +12,19 @@ airflow/providers/common/sql/doc/adr/0003-introduce-notion-of-dialects-in-dbapih
 airflow/providers/common/sql/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/common/sql/hooks/handlers.py,sha256=XjvycIQsGpDrtg6RFACczybW_dER97RR6Z6B_S6jf6Y,3399
 airflow/providers/common/sql/hooks/handlers.pyi,sha256=3UDOBxvFi5dLzRlF2yCwlj8LuYgDFSKNLmCHhF_Qfik,1827
-airflow/providers/common/sql/hooks/sql.py,sha256=Gv3UftzA_uL6NTSx7Q1Ker-TYQs3DC6BC0doP8f0ZSQ,41833
-airflow/providers/common/sql/hooks/sql.pyi,sha256=WZTBUUEI5NaboGl6XVawW11MqPFZF9u71LBUS5XGTJ4,6764
+airflow/providers/common/sql/hooks/sql.py,sha256=1zgwVrP93mtwRGa4SmgMcj1Vhq82YSGfEkYT-2zD9ZA,42358
+airflow/providers/common/sql/hooks/sql.pyi,sha256=cQWV2jQ6IEnOp3Ajbg220AbXu2gn5aPQa5F77Wilg30,8565
 airflow/providers/common/sql/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/common/sql/operators/generic_transfer.py,sha256=EEKKMcOOLl0vyjXRHBI6UuoFLqpHpeHEj9zwMTrNAA4,8336
 airflow/providers/common/sql/operators/generic_transfer.pyi,sha256=rhuCB7KSm_NutW8m3BNQmaoiUPDXp1fTrSeoR0Jr4dU,3330
-airflow/providers/common/sql/operators/sql.py,sha256=WrLBJksK5XxMXipF2b5JBbeFF3xGhtM1varJI83vfuQ,49920
+airflow/providers/common/sql/operators/sql.py,sha256=DZNljvIwDgSwUpmCWZoe1O8xulY8BURCzvP78t9q--I,50157
 airflow/providers/common/sql/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/common/sql/sensors/sql.py,sha256=iYcE8vxLbya8GncOydjceAhu43uoGsN7wqjeYFQ-cak,5471
 airflow/providers/common/sql/sensors/sql.pyi,sha256=GiOk2qD0PO5HWISgTTdOJQLC9b2ItzvQr68adXIbjGQ,2530
 airflow/providers/common/sql/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/common/sql/triggers/sql.py,sha256=C6BEhJdypE_hrvrWU_jxJuOi5FbpQG4xJ0EYPn_fqR0,3665
 airflow/providers/common/sql/triggers/sql.pyi,sha256=7wVgfqUPJB7egsWwbZtwZV3TFm7DuKLclWetNInCM5w,1986
-apache_airflow_providers_common_sql-1.26.0rc1.dist-info/entry_points.txt,sha256=h8UXRp2crPuGmYVYRM5oe168qIh7g-4t2QQbVMizKjI,106
-apache_airflow_providers_common_sql-1.26.0rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-apache_airflow_providers_common_sql-1.26.0rc1.dist-info/METADATA,sha256=Uuum2BU4qXiPiYKIXsy7BgIMjv-Q3RTq_BsZCA2xWXI,5350
-apache_airflow_providers_common_sql-1.26.0rc1.dist-info/RECORD,,
+apache_airflow_providers_common_sql-1.27.0rc1.dist-info/entry_points.txt,sha256=h8UXRp2crPuGmYVYRM5oe168qIh7g-4t2QQbVMizKjI,106
+apache_airflow_providers_common_sql-1.27.0rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+apache_airflow_providers_common_sql-1.27.0rc1.dist-info/METADATA,sha256=jrfkGHOgkIXYbAiIF41X4s9qmxzIg-qJzzimugZDa9s,5352
+apache_airflow_providers_common_sql-1.27.0rc1.dist-info/RECORD,,