apache-airflow-providers-common-sql 2.1.1__tar.gz → 2.2.0__tar.gz

{apache_airflow_providers_common_sql-2.1.1 → apache_airflow_providers_common_sql-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: apache-airflow-providers-common-sql
-Version: 2.1.1
+Version: 2.2.0
 Summary: Provider package apache-airflow-providers-common-sql for Apache Airflow
 Keywords: airflow-provider,common.sql,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -39,8 +39,8 @@ Requires-Dist: polars>=1.26.0 ; extra == "polars"
 Requires-Dist: pyiceberg-core>=0.8.0 ; extra == "pyiceberg-core"
 Requires-Dist: sqlalchemy>=1.4.54 ; extra == "sqlalchemy"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.1/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.2.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.2.0
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -83,7 +83,7 @@ Import-Namespace: airflow.providers.common
 
 Package ``apache-airflow-providers-common-sql``
 
-Release: ``2.1.1``
+Release: ``2.2.0``
 
 
 `Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
@@ -96,7 +96,7 @@ This is a provider package for ``common.sql`` provider. All classes for this pro
 are in ``airflow.providers.common.sql`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.2.0/>`_.
 
 Installation
 ------------
@@ -158,5 +158,5 @@ Extra               Dependencies
 ==================  =======================================================================================================================================================================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.2.0/changelog.html>`_.
 

{apache_airflow_providers_common_sql-2.1.1 → apache_airflow_providers_common_sql-2.2.0}/README.rst

@@ -23,7 +23,7 @@
 
 Package ``apache-airflow-providers-common-sql``
 
-Release: ``2.1.1``
+Release: ``2.2.0``
 
 
 `Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
@@ -36,7 +36,7 @@ This is a provider package for ``common.sql`` provider. All classes for this pro
 are in ``airflow.providers.common.sql`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.2.0/>`_.
 
 Installation
 ------------
@@ -98,4 +98,4 @@ Extra               Dependencies
 ==================  =======================================================================================================================================================================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.2.0/changelog.html>`_.

{apache_airflow_providers_common_sql-2.1.1 → apache_airflow_providers_common_sql-2.2.0}/docs/changelog.rst

@@ -25,6 +25,37 @@
 Changelog
 ---------
 
+2.2.0
+.....
+
+Features
+~~~~~~~~
+
+* ``Template every connection id accepted by provider operators (#73286)``
+* ``Add GCS support to the DataFusion object storage layer (#73370)``
+* ``Allow DataSourceConfig to represent a plain database table (#73273)``
+
+Bug Fixes
+~~~~~~~~~
+
+* ``Read the legacy extra__google_cloud_platform__ prefix for GCS credentials (#73613)``
+* ``Fix DataFusion storage-type error message on Python 3.11+ (#73413)``
+* ``Reject a DataSourceConfig with no uri or format in DataFusionEngine (#73287)``
+* ``Fix exception messages rendering as a tuple instead of the value (#73296)``
+
+Doc-only
+~~~~~~~~
+
+* ``Rewrite the 'common.sql' connections guide and fix the dialect extra name (#73605)``
+* ``Fix grammar in the dialects documentation (#73145)``
+
+.. Below changes are excluded from the changelog. Move them to
+   appropriate section above if needed. Do not delete the lines(!):
+   * ``Revert "[main] Upgrade important CI environment (#73308)" (#73621)``
+   * ``[main] Upgrade important CI environment (#73308)``
+   * ``Add tests for DataFusion base classes (#71983)``
+   * ``Use common.compat.sdk for the remaining provider timezone imports (#71209)``
+
 2.1.1
 .....
 

apache_airflow_providers_common_sql-2.2.0/docs/connections.rst

@@ -0,0 +1,164 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+.. _howto/connection:sql:
+
+Connecting to SQL Databases
+===========================
+
+The common SQL provider has no connection type of its own. Its operators, sensor and
+``@task.sql`` decorator work with a connection from a database provider: a ``postgres``
+connection from the Postgres provider, a ``snowflake`` connection from the Snowflake provider,
+and so on. Create the connection as the database provider documents it, then pass its ID to
+the operator.
+
+.. code-block:: python
+
+    from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator
+
+    create_table = SQLExecuteQueryOperator(
+        task_id="create_table",
+        conn_id="my_postgres",
+        sql="CREATE TABLE IF NOT EXISTS users (id INT, name TEXT)",
+    )
+
+The same task runs against MySQL, Snowflake or Trino by pointing ``conn_id`` at a connection
+of that type. The SQL itself still has to be valid for that database.
+
+Setting up the connection
+-------------------------
+
+1. Install the provider for your database, for example ``apache-airflow-providers-postgres``.
+   :doc:`supported-database-types` lists the providers that work with common SQL.
+2. Create a connection of that provider's type. The fields and extras are described on the
+   provider's own connection page, such as
+   :doc:`the Postgres connection <apache-airflow-providers-postgres:connections/postgres>`.
+3. Pass the connection ID as ``conn_id``. There is no default connection ID, so every task has
+   to name one. ``GenericTransfer`` takes two: ``source_conn_id`` and ``destination_conn_id``.
+
+How the hook is chosen
+----------------------
+
+At run time, a common SQL operator looks up the connection and instantiates the hook class that
+the database provider registered for that connection type. The hook has to be a subclass of
+:class:`~airflow.providers.common.sql.hooks.sql.DbApiHook`, otherwise the task fails with an
+error naming the hook class it found.
+
+To see which hook a connection type resolves to, run:
+
+.. code-block:: bash
+
+    airflow providers hooks
+
+If a connection type is missing from that list, its provider is not installed in the
+environment the task runs in. If the hook is listed but is not a ``DbApiHook``, that connection
+type cannot be used with common SQL (an HTTP or cloud storage connection, for example).
+
+Overriding connection settings per task
+---------------------------------------
+
+To change how one task connects without editing a connection other tasks share, set
+``database`` or ``hook_params`` on the operator.
+
+``database``
+    Runs the task against a different database from the one set in the connection. For most
+    connection types this replaces the connection's ``schema`` field, which is where Postgres,
+    MySQL and similar providers store the database name.
+
+``hook_params``
+    A dictionary passed as keyword arguments to the hook's constructor. Which keys a hook
+    accepts is up to the database provider. For example, the Snowflake hook accepts
+    ``warehouse`` and ``role``:
+
+    .. code-block:: python
+
+        SQLExecuteQueryOperator(
+            task_id="nightly_rollup",
+            conn_id="snowflake_default",
+            sql="CALL rollup_daily_sales()",
+            hook_params={"warehouse": "REPORTING_WH", "role": "REPORTER"},
+        )
+
+    For operators in ``airflow.providers.common.sql.operators.sql``, the connection's extras are
+    merged into ``hook_params`` before the hook is built, and a key set in ``hook_params`` wins
+    over the same key in the extras. ``SQLSensor`` takes ``hook_params`` too, and
+    ``GenericTransfer`` takes ``source_hook_params`` and ``destination_hook_params``.
+
+Connection extras read by common SQL
+------------------------------------
+
+Every ``DbApiHook`` reads the following keys from the connection's extras, on top of whatever
+the database provider documents. They control the SQL that ``insert_rows``,
+``SQLInsertRowsOperator`` and ``GenericTransfer`` generate, and they matter most for generic
+connection types such as ODBC and JDBC, where Airflow cannot tell from the connection alone
+which database is on the other end.
+
+.. list-table::
+   :header-rows: 1
+   :widths: 25 20 55
+
+   * - Extra
+     - Default
+     - Effect
+   * - ``placeholder``
+     - ``%s``
+     - Parameter placeholder in generated statements. Only ``%s`` and ``?`` are accepted; any
+       other value is ignored with a warning. Some hooks set their own default, such as ``?``
+       for SQLite.
+   * - ``sqlalchemy_scheme``
+     - none (``mssql+pyodbc`` for ODBC)
+     - SQLAlchemy scheme for connection types that have none of their own, such as ODBC and
+       JDBC. The :doc:`dialect <dialects>` is taken from it, so ``mssql+pyodbc`` selects the
+       ``mssql`` dialect. It takes precedence over ``dialect``.
+   * - ``dialect``
+     - ``default``
+     - Name of the dialect to use, such as ``mssql`` or ``postgresql``. Read only when neither the
+       connection URI nor ``sqlalchemy_scheme`` gives a dialect, as with a JDBC connection whose
+       ``host`` holds a ``jdbc:`` URL.
+   * - ``insert_statement_format``
+     - ``INSERT INTO {} {} VALUES ({})``
+     - Template for insert statements. The three fields are the table, the column list and the
+       placeholders.
+   * - ``replace_statement_format``
+     - ``REPLACE INTO {} {} VALUES ({})``
+     - Template for upsert statements, used when ``replace=True``.
+   * - ``escape_word_format``
+     - ``"{}"``
+     - How a column name is quoted when it needs escaping, for example ``[{}]`` for SQL Server.
+   * - ``escape_column_names``
+     - ``false``
+     - Quote every column name. By default only reserved words and names containing special
+       characters are quoted.
+
+For example, an ODBC connection to SQL Server that uses pyodbc's ``?`` placeholders and
+bracket-quoted column names can set:
+
+.. code-block:: json
+
+    {
+        "sqlalchemy_scheme": "mssql+pyodbc",
+        "placeholder": "?",
+        "escape_word_format": "[{}]"
+    }
+
+``mssql+pyodbc`` is already the ODBC default. An ODBC connection to any other database has to
+change it, because setting ``dialect`` alone keeps the ``mssql`` dialect.
+
+The ``mssql`` and ``postgresql`` dialects are registered by the Microsoft SQL Server and Postgres
+providers. If the provider for a dialect is not installed, the hook uses the default dialect
+without a warning, so an upsert through ODBC to SQL Server generates ``REPLACE INTO`` instead of
+``MERGE`` unless ``apache-airflow-providers-microsoft-mssql`` is installed.

{apache_airflow_providers_common_sql-2.1.1 → apache_airflow_providers_common_sql-2.2.0}/docs/dialects.rst

@@ -25,7 +25,7 @@ and the :class:`~airflow.providers.jdbc.hooks.jdbc.JdbcHook` are generic hooks w
 database that has a driver for it, it needed an abstraction layer which allows us to run specialized queries
 depending of the database to which we connect and that's why dialects where introduced.
 
-The default :class:`~airflow.providers.common.sql.dialects.dialect.Dialect` class has following operations
+The default :class:`~airflow.providers.common.sql.dialects.dialect.Dialect` class has the following operations
 available which underneath use SQLAlchemy to execute, but can be overloaded with specialized implementations
 per database:
 
@@ -45,11 +45,13 @@ At the moment there are only 3 dialects available:
 - ``mssql`` :class:`~airflow.providers.microsoft.mssql.dialects.mssql.MsSqlDialect` specialized for Microsoft SQL Server;
 - ``postgresql`` :class:`~airflow.providers.postgres.dialects.postgres.PostgresDialect` specialized for PostgreSQL;
 
-The dialect to be used will be derived from the connection string, which sometimes won't be possible.  There is always
-the possibility to specify the dialect name through the extra options of the connection:
+The dialect is derived from the connection URI. ODBC and JDBC connections have no database in their connection type, so
+set the ``sqlalchemy_scheme`` extra to name it, for example ``mssql+pyodbc``. The dialect is the part before the
+``+``. The ``dialect`` extra is read only when neither the URI nor ``sqlalchemy_scheme`` gives a dialect, as with a JDBC
+connection whose ``host`` holds a ``jdbc:`` URL:
 
 .. code-block::
 
-  dialect_name: 'mssql'
+  dialect: 'mssql'
 
 If a specific dialect isn't available for a database, the default one will be used, same when a non-existing dialect name is specified.

{apache_airflow_providers_common_sql-2.1.1 → apache_airflow_providers_common_sql-2.2.0}/docs/index.rst

@@ -79,7 +79,7 @@ apache-airflow-providers-common-sql package
 `Common SQL Provider <https://en.wikipedia.org/wiki/SQL>`__
 
 
-Release: 2.1.1
+Release: 2.2.0
 
 Provider package
 ----------------
@@ -160,5 +160,5 @@ Downloading official packages
 You can download officially released packages and verify their checksums and signatures from the
 `Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
 
-* `The apache-airflow-providers-common-sql 2.1.1 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.1.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.1.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.1.tar.gz.sha512>`__)
-* `The apache-airflow-providers-common-sql 2.1.1 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.1-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.1-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.1.1-py3-none-any.whl.sha512>`__)
+* `The apache-airflow-providers-common-sql 2.2.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.2.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.2.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.2.0.tar.gz.sha512>`__)
+* `The apache-airflow-providers-common-sql 2.2.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.2.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.2.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_common_sql-2.2.0-py3-none-any.whl.sha512>`__)

{apache_airflow_providers_common_sql-2.1.1 → apache_airflow_providers_common_sql-2.2.0}/docs/operators.rst

@@ -296,10 +296,11 @@ The Analytics Operator is ideal for performing efficient, high-performance analy
 Supported Storage Systems
 -------------------------
 - S3
+- GCS
 - Local File System
 
 .. note::
-   GCS, Azure, HTTP, Delta, Iceberg are not yet supported but will be added in the future.
+   Azure, HTTP, Delta are not yet supported but will be added in the future.
 
 
 
@@ -341,6 +342,26 @@ S3 Storage
     :start-after: [START howto_analytics_operator_with_s3]
     :end-before: [END howto_analytics_operator_with_s3]
 
+GCS Storage
+-----------
+Use a ``conn_id`` pointing to a ``google_cloud_platform`` connection. Credentials are
+resolved in this order:
+
+1. ``key_path`` -- a service account JSON file on disk
+2. ``keyfile_dict`` -- the service account JSON contents, inline
+3. the ``GOOGLE_APPLICATION_CREDENTIALS`` environment variable
+4. Application Default Credentials -- the local ``gcloud`` CLI credentials file, or
+   the GCE/GKE metadata server
+
+``key_path`` and ``keyfile_dict`` are mutually exclusive. ``key_secret_name``,
+``credential_config_file``, and ``impersonation_chain`` are not supported.
+
+.. exampleinclude:: /../../sql/src/airflow/providers/common/sql/example_dags/example_analytics.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_analytics_operator_with_gcs]
+    :end-before: [END howto_analytics_operator_with_gcs]
+
 Local File System Storage
 -------------------------
 .. exampleinclude:: /../../sql/src/airflow/providers/common/sql/example_dags/example_analytics.py

{apache_airflow_providers_common_sql-2.1.1 → apache_airflow_providers_common_sql-2.2.0}/provider.yaml

@@ -23,12 +23,13 @@ description: |
 
 state: ready
 lifecycle: production
-source-date-epoch: 1787011200
+source-date-epoch: 1790023613
 # Note that those versions are maintained by release manager - do not update them manually
 # with the exception of case where other provider in sources has >= new provider version.
 # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
 # to be done in the same PR
 versions:
+  - 2.2.0
   - 2.1.1
   - 2.1.0
   - 2.0.3

{apache_airflow_providers_common_sql-2.1.1 → apache_airflow_providers_common_sql-2.2.0}/pyproject.toml

@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-common-sql"
-version = "2.1.1"
+version = "2.2.0"
 description = "Provider package apache-airflow-providers-common-sql for Apache Airflow"
 readme = "README.rst"
 license = "Apache-2.0"
@@ -148,8 +148,8 @@ apache-airflow-providers-common-sql = {workspace = true}
 apache-airflow-providers-standard = {workspace = true}
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.1"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.1.1/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.2.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-common-sql/2.2.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"

{apache_airflow_providers_common_sql-2.1.1 → apache_airflow_providers_common_sql-2.2.0}/src/airflow/providers/common/sql/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "2.1.1"
+__version__ = "2.2.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.11.0"

{apache_airflow_providers_common_sql-2.1.1 → apache_airflow_providers_common_sql-2.2.0}/src/airflow/providers/common/sql/config.py

@@ -47,6 +47,7 @@ class StorageType(str, Enum):
     """Storage types for Data Fusion."""
 
     S3 = "s3"
+    GCS = "gcs"
     LOCAL = "local"
 
 
@@ -62,6 +63,9 @@ class DataSourceConfig:
     require ``uri`` or ``storage_type``; they use ``conn_id`` and format-specific
     keys in ``options`` (e.g. ``catalog_table_name`` for Iceberg).
 
+    **Plain database tables** (neither ``uri`` nor ``format`` set) are not
+    object-store backed either; ``storage_type`` stays ``None`` and is not inferred.
+
     :param conn_id: The connection ID to use for accessing the data source.
     :param uri: The URI of the data source (e.g., file path, S3 bucket, etc.).
         Not required for catalog-managed formats.
@@ -88,22 +92,31 @@ class DataSourceConfig:
         return bool(self.format and self.format.lower() in TABLE_PROVIDERS)
 
     def __post_init__(self):
+        if not self.table_name or not self.table_name.strip():
+            raise ValueError("Table name must be provided for storage type")
+
         if self.is_table_provider:
             if self.db_name is None:
                 raise ValueError(f"Database name must be provided for table providers {TABLE_PROVIDERS}")
             return
 
+        if not self.format and not self.uri:
+            # Plain database table: no object store involved, so storage_type stays unset.
+            return
+
+        if not self.uri:
+            raise ValueError("URI must be provided when format is set")
+
         if self.storage_type is None:
             self.storage_type = self._extract_storage_type
 
-        if self.storage_type is not None and (not self.table_name or not self.table_name.strip()):
-            raise ValueError("Table name must be provided for storage type")
-
     @property
     def _extract_storage_type(self) -> StorageType | None:
         """Extract storage type."""
         if self.uri.startswith("s3://"):
             return StorageType.S3
+        if self.uri.startswith("gs://"):
+            return StorageType.GCS
         if self.uri.startswith("file://"):
             return StorageType.LOCAL
         raise ValueError(f"Unsupported storage type for URI: {self.uri}")

{apache_airflow_providers_common_sql-2.1.1 → apache_airflow_providers_common_sql-2.2.0}/src/airflow/providers/common/sql/datafusion/engine.py

@@ -16,6 +16,7 @@
 # under the License.
 from __future__ import annotations
 
+import os
 from typing import TYPE_CHECKING, Any
 
 from datafusion import SessionContext
@@ -50,6 +51,12 @@ class DataFusionEngine(LoggingMixin):
         if not isinstance(datasource_config, DataSourceConfig):
             raise ValueError("datasource_config must be of type DataSourceConfig")
 
+        if not datasource_config.is_table_provider and datasource_config.storage_type is None:
+            raise ValueError(
+                f"DataSourceConfig for table {datasource_config.table_name!r} has no uri or format; "
+                "DataFusionEngine only registers object-store or catalog-managed sources."
+            )
+
         if not datasource_config.is_table_provider:
             if datasource_config.storage_type == StorageType.LOCAL:
                 connection_config = None
@@ -146,6 +153,14 @@ class DataFusionEngine(LoggingMixin):
                     conf[key] = conn.extra_dejson[key]
             return conf
 
+        def _get_gcp_extra_field(extra_dejson: dict[str, Any], field_name: str) -> Any:
+            # Older Airflow connection UIs wrote custom extra fields as
+            # extra__google_cloud_platform__<field_name> instead of the bare key; GoogleBaseHook
+            # still reads that legacy spelling as a fallback, so this must too.
+            if field_name in extra_dejson:
+                return extra_dejson[field_name]
+            return extra_dejson.get(f"extra__google_cloud_platform__{field_name}")
+
         match conn.conn_type:
             case "aws":
                 try:
@@ -169,6 +184,26 @@ class DataFusionEngine(LoggingMixin):
                 credentials = self._remove_none_values(credentials)
                 extra_config = _fetch_extra_configs(["region", "endpoint"])
 
+            case "google_cloud_platform":
+                extra_dejson = conn.extra_dejson
+                for unsupported_field in ("key_secret_name", "credential_config_file", "impersonation_chain"):
+                    if _get_gcp_extra_field(extra_dejson, unsupported_field):
+                        raise ValueError(
+                            f"Connection field {unsupported_field!r} is not supported for DataFusion "
+                            "GCS access; only key_path, keyfile_dict, GOOGLE_APPLICATION_CREDENTIALS, or "
+                            "ambient credentials (gcloud ADC file / metadata server) are used."
+                        )
+                key_path = _get_gcp_extra_field(extra_dejson, "key_path") or None
+                keyfile_dict = _get_gcp_extra_field(extra_dejson, "keyfile_dict") or None
+                if key_path and keyfile_dict:
+                    raise ValueError(
+                        "The `keyfile_dict` and `key_path` fields are mutually exclusive. "
+                        "Please provide only one value."
+                    )
+                if not key_path and not keyfile_dict:
+                    key_path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
+                credentials = self._remove_none_values({"key_path": key_path, "keyfile_dict": keyfile_dict})
+
             case _:
                 raise ValueError(f"Unknown connection type {conn.conn_type}")
         return credentials, extra_config

{apache_airflow_providers_common_sql-2.1.1 → apache_airflow_providers_common_sql-2.2.0}/src/airflow/providers/common/sql/datafusion/object_storage_provider.py

@@ -16,7 +16,11 @@
 # under the License.
 from __future__ import annotations
 
-from datafusion.object_store import AmazonS3, LocalFileSystem
+import json
+import tempfile
+from pathlib import Path
+
+from datafusion.object_store import AmazonS3, GoogleCloud, LocalFileSystem
 
 from airflow.providers.common.sql.config import ConnectionConfig, StorageType
 from airflow.providers.common.sql.datafusion.base import ObjectStorageProvider
@@ -34,7 +38,7 @@ class S3ObjectStorageProvider(ObjectStorageProvider):
     def create_object_store(self, path: str, connection_config: ConnectionConfig | None = None):
         """Create an S3 object store using DataFusion's AmazonS3."""
         if connection_config is None:
-            raise ValueError("connection_config must be provided for %s", self.get_storage_type)
+            raise ValueError(f"connection_config must be provided for {self.get_storage_type.value}")
 
         try:
             credentials = connection_config.credentials
@@ -53,6 +57,56 @@ class S3ObjectStorageProvider(ObjectStorageProvider):
         return "s3://"
 
 
+class GCSObjectStorageProvider(ObjectStorageProvider):
+    """GCS Object Storage Provider using DataFusion's GoogleCloud."""
+
+    @property
+    def get_storage_type(self) -> StorageType:
+        """Return the storage type."""
+        return StorageType.GCS
+
+    def create_object_store(self, path: str, connection_config: ConnectionConfig | None = None):
+        """Create a GCS object store using DataFusion's GoogleCloud."""
+        if connection_config is None:
+            raise ValueError(f"connection_config must be provided for {self.get_storage_type.value}")
+
+        credentials = connection_config.credentials
+        key_path = credentials.get("key_path")
+        keyfile_dict = credentials.get("keyfile_dict")
+        temp_key_path: str | None = None
+
+        try:
+            bucket = self.get_bucket(path)
+
+            if not key_path and keyfile_dict:
+                # GoogleCloud only accepts a file path, not inline JSON; safe to delete once
+                # constructed since the credentials are read only at construction time.
+                key_content = keyfile_dict if isinstance(keyfile_dict, str) else json.dumps(keyfile_dict)
+                with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as key_file:
+                    key_file.write(key_content)
+                temp_key_path = key_path = key_file.name
+
+            if key_path is not None and not Path(key_path).is_file():
+                raise FileNotFoundError(f"Service account key file not found: {key_path}")
+
+            gcs_store = GoogleCloud(bucket_name=bucket, service_account_path=key_path)
+            self.log.info("Created GCS object store for bucket %s", bucket)
+
+            return gcs_store
+
+        except BaseException as e:
+            # A bad key file panics as pyo3_runtime.PanicException, not a plain Exception.
+            raise ObjectStoreCreationException(f"Failed to create GCS object store: {e}")
+
+        finally:
+            if temp_key_path is not None:
+                Path(temp_key_path).unlink(missing_ok=True)
+
+    def get_scheme(self) -> str:
+        """Return the scheme for GCS."""
+        return "gs://"
+
+
 class LocalObjectStorageProvider(ObjectStorageProvider):
     """Local Object Storage Provider using DataFusion's LocalFileSystem."""
 
@@ -72,9 +126,10 @@ class LocalObjectStorageProvider(ObjectStorageProvider):
 
 def get_object_storage_provider(storage_type: StorageType) -> ObjectStorageProvider:
     """Get an object storage provider based on the storage type."""
-    # TODO: Add support for GCS, Azure, HTTP: https://datafusion.apache.org/python/autoapi/datafusion/object_store/index.html
+    # TODO: Add support for Azure, HTTP: https://datafusion.apache.org/python/autoapi/datafusion/object_store/index.html
     providers: dict[StorageType, type] = {
         StorageType.S3: S3ObjectStorageProvider,
+        StorageType.GCS: GCSObjectStorageProvider,
         StorageType.LOCAL: LocalObjectStorageProvider,
     }
 

{apache_airflow_providers_common_sql-2.1.1 → apache_airflow_providers_common_sql-2.2.0}/src/airflow/providers/common/sql/example_dags/example_analytics.py

@@ -30,6 +30,10 @@ datasource_config_local = DataSourceConfig(
     conn_id="", table_name="users_data", uri="file:///path/to/", format="parquet"
 )
 
+datasource_config_gcs = DataSourceConfig(
+    conn_id="google_cloud_default", table_name="users_data", uri="gs://bucket/path/", format="parquet"
+)
+
 datasource_config_iceberg = DataSourceConfig(
     conn_id="iceberg_default",
     table_name="users_data",
@@ -72,6 +76,15 @@ with DAG(
 
     # [END howto_analytics_operator_with_s3]
 
+    # [START howto_analytics_operator_with_gcs]
+    analytics_with_gcs = AnalyticsOperator(
+        task_id="analytics_with_gcs",
+        datasource_configs=[datasource_config_gcs],
+        queries=["SELECT * FROM users_data", "SELECT count(*) FROM users_data"],
+    )
+    analytics_with_s3 >> analytics_with_gcs
+    # [END howto_analytics_operator_with_gcs]
+
     # [START howto_analytics_operator_with_local]
     analytics_with_local = AnalyticsOperator(
         task_id="analytics_with_local",

{apache_airflow_providers_common_sql-2.1.1 → apache_airflow_providers_common_sql-2.2.0}/src/airflow/providers/common/sql/sensors/sql.py

@@ -57,7 +57,7 @@ class SqlSensor(BaseSensorOperator):
             Should match the desired hook constructor params.
     """
 
-    template_fields: Sequence[str] = ("sql", "hook_params", "parameters")
+    template_fields: Sequence[str] = ("sql", "hook_params", "parameters", "conn_id")
     template_ext: Sequence[str] = (".hql", ".sql")
     ui_color = "#7c7287"