apache-airflow-providers-common-sql 1.18.0__py3-none-any.whl → 1.19.0__py3-none-any.whl

airflow/providers/common/sql/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "1.18.0"
+__version__ = "1.19.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.8.0"

airflow/providers/common/sql/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Common SQL",
         "description": "`Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__\n",
         "state": "ready",
-        "source-date-epoch": 1728485007,
+        "source-date-epoch": 1730012422,
         "versions": [
+            "1.19.0",
             "1.18.0",
             "1.17.1",
             "1.17.0",

airflow/providers/common/sql/hooks/sql.py

@@ -190,7 +190,7 @@ class DbApiHook(BaseHook):
         return getattr(self, self.conn_name_attr)
 
     @cached_property
-    def placeholder(self):
+    def placeholder(self) -> str:
         placeholder = self.connection_extra.get("placeholder")
         if placeholder:
             if placeholder in SQL_PLACEHOLDERS:
@@ -235,9 +235,11 @@ class DbApiHook(BaseHook):
         """
         return {k.lower(): v for k, v in self.connection_extra.items()}
 
-    def get_conn(self):
+    def get_conn(self) -> Any:
         """Return a connection object."""
         db = self.connection
+        if self.connector is None:
+            raise RuntimeError(f"{type(self).__name__} didn't have `self.connector` set!")
         return self.connector.connect(host=db.host, port=db.port, username=db.login, schema=db.schema)
 
     def get_uri(self) -> str:
@@ -570,7 +572,7 @@ class DbApiHook(BaseHook):
         """
         return getattr(conn, "autocommit", False) and self.supports_autocommit
 
-    def get_cursor(self):
+    def get_cursor(self) -> Any:
         """Return a cursor."""
         return self.get_conn().cursor()
 
@@ -618,6 +620,7 @@ class DbApiHook(BaseHook):
         replace=False,
         *,
         executemany=False,
+        fast_executemany=False,
         autocommit=False,
         **kwargs,
     ):
@@ -636,6 +639,8 @@ class DbApiHook(BaseHook):
         :param executemany: If True, all rows are inserted at once in
             chunks defined by the commit_every parameter. This only works if all rows
             have same number of column names, but leads to better performance.
+        :param fast_executemany: If True, the `fast_executemany` parameter will be set on the
+            cursor used by `executemany` which leads to better performance, if supported by driver.
         :param autocommit: What to set the connection's autocommit setting to
             before executing the query.
         """
@@ -644,6 +649,15 @@ class DbApiHook(BaseHook):
             conn.commit()
             with closing(conn.cursor()) as cur:
                 if self.supports_executemany or executemany:
+                    if fast_executemany:
+                        with contextlib.suppress(AttributeError):
+                            # Try to set the fast_executemany attribute
+                            cur.fast_executemany = True
+                            self.log.info(
+                                "Fast_executemany is enabled for conn_id '%s'!",
+                                self.get_conn_id(),
+                            )
+
                     for chunked_rows in chunked(rows, commit_every):
                         values = list(
                             map(

airflow/providers/common/sql/hooks/sql.pyi

@@ -31,19 +31,14 @@
 Definition of the public interface for airflow.providers.common.sql.hooks.sql
 isort:skip_file
 """
-from _typeshed import Incomplete
-from airflow.exceptions import (
-    AirflowException as AirflowException,
-    AirflowOptionalProviderFeatureException as AirflowOptionalProviderFeatureException,
-    AirflowProviderDeprecationWarning as AirflowProviderDeprecationWarning,
-)
+from _typeshed import Incomplete as Incomplete
 from airflow.hooks.base import BaseHook as BaseHook
 from airflow.models import Connection as Connection
 from airflow.providers.openlineage.extractors import OperatorLineage as OperatorLineage
 from airflow.providers.openlineage.sqlparser import DatabaseInfo as DatabaseInfo
 from functools import cached_property as cached_property
 from pandas import DataFrame as DataFrame
-from sqlalchemy.engine import Inspector, URL as URL
+from sqlalchemy.engine import Inspector as Inspector, URL as URL
 from typing import Any, Callable, Generator, Iterable, Mapping, Protocol, Sequence, TypeVar, overload
 
 T = TypeVar("T")
@@ -67,7 +62,7 @@ class DbApiHook(BaseHook):
     def __init__(self, *args, schema: str | None = None, log_sql: bool = True, **kwargs) -> None: ...
     def get_conn_id(self) -> str: ...
     @cached_property
-    def placeholder(self): ...
+    def placeholder(self) -> str: ...
     @property
     def connection(self) -> Connection: ...
     @connection.setter
@@ -76,7 +71,7 @@ class DbApiHook(BaseHook):
     def connection_extra(self) -> dict: ...
     @cached_property
     def connection_extra_lower(self) -> dict: ...
-    def get_conn(self): ...
+    def get_conn(self) -> Any: ...
     def get_uri(self) -> str: ...
     @property
     def sqlalchemy_url(self) -> URL: ...
@@ -123,7 +118,7 @@ class DbApiHook(BaseHook):
     ) -> tuple | list[tuple] | list[list[tuple] | tuple] | None: ...
     def set_autocommit(self, conn, autocommit) -> None: ...
     def get_autocommit(self, conn) -> bool: ...
-    def get_cursor(self): ...
+    def get_cursor(self) -> Any: ...
     def insert_rows(
         self,
         table,
@@ -138,7 +133,7 @@ class DbApiHook(BaseHook):
     ): ...
     def bulk_dump(self, table, tmp_file) -> None: ...
     def bulk_load(self, table, tmp_file) -> None: ...
-    def test_connection(self): ...
+    def test_connection(self) -> None: ...
     def get_openlineage_database_info(self, connection) -> DatabaseInfo | None: ...
     def get_openlineage_database_dialect(self, connection) -> str: ...
     def get_openlineage_default_schema(self) -> str | None: ...

airflow/providers/common/sql/operators/sql.py

@@ -145,13 +145,25 @@ class BaseSQLOperator(BaseOperator):
         self.hook_params = hook_params or {}
         self.retry_on_failure = retry_on_failure
 
+    @classmethod
+    # TODO: can be removed once Airflow min version for this provider is 3.0.0 or higher
+    def get_hook(cls, conn_id: str, hook_params: dict | None = None) -> BaseHook:
+        """
+        Return default hook for this connection id.
+
+        :param conn_id: connection id
+        :param hook_params: hook parameters
+        :return: default hook for this connection
+        """
+        connection = BaseHook.get_connection(conn_id)
+        return connection.get_hook(hook_params=hook_params)
+
     @cached_property
     def _hook(self):
         """Get DB Hook based on connection type."""
         conn_id = getattr(self, self.conn_id_field)
         self.log.debug("Get connection for %s", conn_id)
-        conn = BaseHook.get_connection(conn_id)
-        hook = conn.get_hook(hook_params=self.hook_params)
+        hook = self.get_hook(conn_id=conn_id, hook_params=self.hook_params)
         if not isinstance(hook, DbApiHook):
             raise AirflowException(
                 f"You are trying to use `common-sql` with {hook.__class__.__name__},"

airflow/providers/common/sql/operators/sql.pyi

@@ -31,25 +31,13 @@
 Definition of the public interface for airflow.providers.common.sql.operators.sql
 isort:skip_file
 """
-from _typeshed import Incomplete
-from airflow.exceptions import (
-    AirflowException as AirflowException,
-    AirflowFailException as AirflowFailException,
-)
-from airflow.hooks.base import BaseHook as BaseHook
+from _typeshed import Incomplete as Incomplete
 from airflow.models import BaseOperator as BaseOperator, SkipMixin as SkipMixin
-from airflow.providers.common.sql.hooks.sql import (
-    DbApiHook as DbApiHook,
-    fetch_all_handler as fetch_all_handler,
-    return_single_query_results as return_single_query_results,
-)
+from airflow.providers.common.sql.hooks.sql import DbApiHook as DbApiHook
 from airflow.providers.openlineage.extractors import OperatorLineage as OperatorLineage
 from airflow.utils.context import Context as Context
-from airflow.utils.helpers import merge_dicts as merge_dicts
-from functools import cached_property as cached_property
 from typing import Any, Callable, Iterable, Mapping, Sequence, SupportsAbs
 
-def _parse_boolean(val: str) -> str | bool: ...
 def parse_boolean(val: str) -> str | bool: ...
 
 class BaseSQLOperator(BaseOperator):
@@ -97,7 +85,7 @@ class SQLExecuteQueryOperator(BaseSQLOperator):
         show_return_value_in_logs: bool = False,
         **kwargs,
     ) -> None: ...
-    def execute(self, context): ...
+    def execute(self, context) -> None: ...
     def prepare_template(self) -> None: ...
     def get_openlineage_facets_on_start(self) -> OperatorLineage | None: ...
     def get_openlineage_facets_on_complete(self, task_instance) -> OperatorLineage | None: ...

airflow/providers/common/sql/sensors/sql.py

@@ -16,6 +16,7 @@
 # under the License.
 from __future__ import annotations
 
+from operator import itemgetter
 from typing import TYPE_CHECKING, Any, Callable, Mapping, Sequence
 
 from airflow.exceptions import AirflowException
@@ -46,10 +47,12 @@ class SqlSensor(BaseSensorOperator):
     :param sql: The SQL to run. To pass, it needs to return at least one cell
         that contains a non-zero / empty string value.
     :param parameters: The parameters to render the SQL query with (optional).
-    :param success: Success criteria for the sensor is a Callable that takes the first_cell's value
-        as the only argument, and returns a boolean (optional).
-    :param failure: Failure criteria for the sensor is a Callable that takes the first_cell's value
-        as the only argument and returns a boolean (optional).
+    :param success: Success criteria for the sensor is a Callable that takes the output
+        of selector as the only argument, and returns a boolean (optional).
+    :param failure: Failure criteria for the sensor is a Callable that takes the output
+        of selector as the only argument and returns a boolean (optional).
+    :param selector: Function which takes the resulting row and transforms it before
+        it is passed to success or failure (optional). Takes the first cell by default.
     :param fail_on_empty: Explicitly fail on no rows returned.
     :param hook_params: Extra config params to be passed to the underlying hook.
             Should match the desired hook constructor params.
@@ -67,6 +70,7 @@ class SqlSensor(BaseSensorOperator):
         parameters: Mapping[str, Any] | None = None,
         success: Callable[[Any], bool] | None = None,
         failure: Callable[[Any], bool] | None = None,
+        selector: Callable[[tuple[Any]], Any] = itemgetter(0),
         fail_on_empty: bool = False,
         hook_params: Mapping[str, Any] | None = None,
         **kwargs,
@@ -76,6 +80,7 @@ class SqlSensor(BaseSensorOperator):
         self.parameters = parameters
         self.success = success
         self.failure = failure
+        self.selector = selector
         self.fail_on_empty = fail_on_empty
         self.hook_params = hook_params
         super().__init__(**kwargs)
@@ -102,11 +107,11 @@ class SqlSensor(BaseSensorOperator):
             else:
                 return False
 
-        first_cell = records[0][0]
+        condition = self.selector(records[0])
         if self.failure is not None:
             if callable(self.failure):
-                if self.failure(first_cell):
-                    message = f"Failure criteria met. self.failure({first_cell}) returned True"
+                if self.failure(condition):
+                    message = f"Failure criteria met. self.failure({condition}) returned True"
                     raise AirflowException(message)
             else:
                 message = f"self.failure is present, but not callable -> {self.failure}"
@@ -114,8 +119,8 @@ class SqlSensor(BaseSensorOperator):
 
         if self.success is not None:
             if callable(self.success):
-                return self.success(first_cell)
+                return self.success(condition)
             else:
                 message = f"self.success is present, but not callable -> {self.success}"
                 raise AirflowException(message)
-        return bool(first_cell)
+        return bool(condition)

airflow/providers/common/sql/sensors/sql.pyi

@@ -31,10 +31,7 @@
 Definition of the public interface for airflow.providers.common.sql.sensors.sql
 isort:skip_file
 """
-from _typeshed import Incomplete
-from airflow.exceptions import AirflowException as AirflowException
-from airflow.hooks.base import BaseHook as BaseHook
-from airflow.providers.common.sql.hooks.sql import DbApiHook as DbApiHook
+from _typeshed import Incomplete as Incomplete
 from airflow.sensors.base import BaseSensorOperator as BaseSensorOperator
 from airflow.utils.context import Context as Context
 from typing import Any, Callable, Mapping, Sequence

{apache_airflow_providers_common_sql-1.18.0.dist-info → apache_airflow_providers_common_sql-1.19.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-common-sql
-Version: 1.18.0
+Version: 1.19.0
 Summary: Provider package apache-airflow-providers-common-sql for Apache Airflow
 Keywords: airflow-provider,common.sql,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -27,8 +27,8 @@ Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
 Requires-Dist: pandas>=2.1.2,<2.2 ; extra == "pandas" and (python_version>="3.9")
 Requires-Dist: pandas>=1.5.3,<2.2 ; extra == "pandas" and (python_version<"3.9")
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.18.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.18.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.19.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.19.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -80,7 +80,7 @@ Provides-Extra: pandas
 
 Package ``apache-airflow-providers-common-sql``
 
-Release: ``1.18.0``
+Release: ``1.19.0``
 
 
 `Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
@@ -93,7 +93,7 @@ This is a provider package for ``common.sql`` provider. All classes for this pro
 are in ``airflow.providers.common.sql`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.18.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.19.0/>`_.
 
 Installation
 ------------
@@ -135,4 +135,4 @@ Dependent package
 ==============================================================================================================  ===============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.18.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/1.19.0/changelog.html>`_.

apache_airflow_providers_common_sql-1.19.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+airflow/providers/common/sql/LICENSE,sha256=FFb4jd2AXnOOf7XLP04pQW6jbdhG49TxlGY6fFpCV1Y,13609
+airflow/providers/common/sql/README_API.md,sha256=CxhaS8EedZ4dcbLUPC4-GLCMaY3OH96oHxXttUGU06E,5932
+airflow/providers/common/sql/__init__.py,sha256=Lt9f9Y4TRs8R4degfGTNMLaJ9UouMAatWgQxxe_pA5M,1498
+airflow/providers/common/sql/get_provider_info.py,sha256=UJMFz7XHr9N8c1rfr5qby71nh74L90ERZbYt57dDIBo,3387
+airflow/providers/common/sql/doc/adr/0001-record-architecture-decisions.md,sha256=TfANqrzoFto9PMOMza3MitIkXHGLx2kY_BhhF-N0_ow,1675
+airflow/providers/common/sql/doc/adr/0002-return-common-data-structure-from-dbapihook-derived-hooks.md,sha256=ze5w9IVS-HkUwdZvPW8_JaJaVwel7-N6XdEVN4pTuCE,8457
+airflow/providers/common/sql/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/common/sql/hooks/sql.py,sha256=3XAyEjbCL7FD7VSMu623tGQgTjnFdKH3_GmqHzRloqA,32629
+airflow/providers/common/sql/hooks/sql.pyi,sha256=-h5-2yKgXZMmuXzClYiM1VTTdbdDHD6N3ZBC9dL1Twc,5953
+airflow/providers/common/sql/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/common/sql/operators/sql.py,sha256=KzJR2npRO93nMPaG0kxdeRKSrX7LRCmoLD7ShT8d_nw,48055
+airflow/providers/common/sql/operators/sql.pyi,sha256=ae7xKolM49nSaAfvcH_J9HpIyVSgCj0-EC_szCFkqYc,8164
+airflow/providers/common/sql/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/common/sql/sensors/sql.py,sha256=UYOQficcLJlumJcZwKIBYMcnCe5PkGPiU2ZaYta858Q,5492
+airflow/providers/common/sql/sensors/sql.pyi,sha256=3agRcNF_FT3_6QnD-sZOI8pV0lBjEalOAIKRINfQ0MI,2467
+apache_airflow_providers_common_sql-1.19.0.dist-info/entry_points.txt,sha256=h8UXRp2crPuGmYVYRM5oe168qIh7g-4t2QQbVMizKjI,106
+apache_airflow_providers_common_sql-1.19.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_common_sql-1.19.0.dist-info/METADATA,sha256=8WwQuKYCFEUKy-r__7WOIL4mmdmYI6tM0fIAFXbq7nM,6163
+apache_airflow_providers_common_sql-1.19.0.dist-info/RECORD,,

apache_airflow_providers_common_sql-1.18.0.dist-info/RECORD

@@ -1,19 +0,0 @@
-airflow/providers/common/sql/LICENSE,sha256=FFb4jd2AXnOOf7XLP04pQW6jbdhG49TxlGY6fFpCV1Y,13609
-airflow/providers/common/sql/README_API.md,sha256=CxhaS8EedZ4dcbLUPC4-GLCMaY3OH96oHxXttUGU06E,5932
-airflow/providers/common/sql/__init__.py,sha256=i78BZYyjaUcUTp1Srl3QKWxJPX8LSio9FNipmR0i0fo,1498
-airflow/providers/common/sql/get_provider_info.py,sha256=5bkf-8XsZG9VPXR_DdHLmkTOBA9lqtUZKUmapzw09ZU,3365
-airflow/providers/common/sql/doc/adr/0001-record-architecture-decisions.md,sha256=TfANqrzoFto9PMOMza3MitIkXHGLx2kY_BhhF-N0_ow,1675
-airflow/providers/common/sql/doc/adr/0002-return-common-data-structure-from-dbapihook-derived-hooks.md,sha256=ze5w9IVS-HkUwdZvPW8_JaJaVwel7-N6XdEVN4pTuCE,8457
-airflow/providers/common/sql/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/common/sql/hooks/sql.py,sha256=gf4LIG0fslnVoTeKrTSwC0HghqgYwZQskkHCGXy5HKc,31814
-airflow/providers/common/sql/hooks/sql.pyi,sha256=niCOE_7j0nBvSZUgI6KAf6zG9E8fhdR_PbwYTRjjN1w,6138
-airflow/providers/common/sql/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/common/sql/operators/sql.py,sha256=ceFuszvASPCUlDpeYEPCWWvj49lQd2ciZJP_Gp2bgXo,47574
-airflow/providers/common/sql/operators/sql.pyi,sha256=nu_vdWYN1g6r_m8nN9LEU8ttBlUCHfETZm-7ekszXOc,8604
-airflow/providers/common/sql/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/common/sql/sensors/sql.py,sha256=OD5GEC_TIOQQDenLV4rjaukQDWEofLaXEUf4K-wzBlk,5184
-airflow/providers/common/sql/sensors/sql.pyi,sha256=mv3MOV7hRgmMCIYRkbP_rEhcXiPhWgjgwa5hAzz8xEw,2647
-apache_airflow_providers_common_sql-1.18.0.dist-info/entry_points.txt,sha256=h8UXRp2crPuGmYVYRM5oe168qIh7g-4t2QQbVMizKjI,106
-apache_airflow_providers_common_sql-1.18.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-apache_airflow_providers_common_sql-1.18.0.dist-info/METADATA,sha256=yBIG8n9UkEj1ybzcmARR43a9w5_yupqePqciZYssx6Q,6163
-apache_airflow_providers_common_sql-1.18.0.dist-info/RECORD,,