apache-airflow-providers-google 10.19.0rc1__py3-none-any.whl → 10.20.0rc1__py3-none-any.whl

airflow/providers/google/cloud/transfers/bigquery_to_postgres.py

@@ -19,20 +19,42 @@
 
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
+from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook
 from airflow.providers.google.cloud.transfers.bigquery_to_sql import BigQueryToSqlBaseOperator
+from airflow.providers.google.cloud.utils.bigquery_get_data import bigquery_get_data
 from airflow.providers.postgres.hooks.postgres import PostgresHook
 
+if TYPE_CHECKING:
+    from airflow.utils.context import Context
+
 
 class BigQueryToPostgresOperator(BigQueryToSqlBaseOperator):
     """
     Fetch data from a BigQuery table (alternatively fetch selected columns) and insert into PostgreSQL table.
 
+    Due to constraints of the PostgreSQL's ON CONFLICT clause both `selected_fields` and `replace_index`
+    parameters need to be specified when using the operator with parameter `replace=True`.
+    In effect this means that in order to run this operator with `replace=True` your target table MUST
+    already have a unique index column / columns, otherwise the INSERT command will fail with an error.
+    See more at https://www.postgresql.org/docs/current/sql-insert.html.
+
+    Please note that currently most of the clauses that can be used with PostgreSQL's INSERT
+    command, such as ON CONSTRAINT, WHERE, DEFAULT, etc.,  are not supported by this operator.
+    If you need the clauses for your queries, `SQLExecuteQueryOperator` will be a more suitable option.
+
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
         :ref:`howto/operator:BigQueryToPostgresOperator`
 
     :param target_table_name: target Postgres table (templated)
     :param postgres_conn_id: Reference to :ref:`postgres connection id <howto/connection:postgres>`.
+    :param replace: Whether to replace instead of insert
+    :param selected_fields: List of fields to return (comma-separated). If
+        unspecified, all fields are returned. Must be specified if `replace` is True
+    :param replace_index: the column or list of column names to act as
+        index for the ON CONFLICT clause. Must be specified if `replace` is True
     """
 
     def __init__(
@@ -40,10 +62,43 @@ class BigQueryToPostgresOperator(BigQueryToSqlBaseOperator):
         *,
         target_table_name: str,
         postgres_conn_id: str = "postgres_default",
+        replace: bool = False,
+        selected_fields: list[str] | str | None = None,
+        replace_index: list[str] | str | None = None,
         **kwargs,
     ) -> None:
-        super().__init__(target_table_name=target_table_name, **kwargs)
+        if replace and not (selected_fields and replace_index):
+            raise ValueError("PostgreSQL ON CONFLICT upsert syntax requires column names and a unique index.")
+        super().__init__(
+            target_table_name=target_table_name, replace=replace, selected_fields=selected_fields, **kwargs
+        )
         self.postgres_conn_id = postgres_conn_id
+        self.replace_index = replace_index
 
     def get_sql_hook(self) -> PostgresHook:
         return PostgresHook(schema=self.database, postgres_conn_id=self.postgres_conn_id)
+
+    def execute(self, context: Context) -> None:
+        big_query_hook = BigQueryHook(
+            gcp_conn_id=self.gcp_conn_id,
+            location=self.location,
+            impersonation_chain=self.impersonation_chain,
+        )
+        self.persist_links(context)
+        sql_hook: PostgresHook = self.get_sql_hook()
+        for rows in bigquery_get_data(
+            self.log,
+            self.dataset_id,
+            self.table_id,
+            big_query_hook,
+            self.batch_size,
+            self.selected_fields,
+        ):
+            sql_hook.insert_rows(
+                table=self.target_table_name,
+                rows=rows,
+                target_fields=self.selected_fields,
+                replace=self.replace,
+                commit_every=self.batch_size,
+                replace_index=self.replace_index,
+            )

airflow/providers/google/cloud/transfers/gcs_to_gcs.py

@@ -408,20 +408,9 @@ class GCSToGCSOperator(BaseOperator):
                 msg = f"{prefix} does not exist in bucket {self.source_bucket}"
                 self.log.warning(msg)
                 raise AirflowException(msg)
-        if len(objects) == 1 and objects[0][-1] != "/":
-            self._copy_file(hook=hook, source_object=objects[0])
         elif len(objects):
             self._copy_multiple_objects(hook=hook, source_objects=objects, prefix=prefix)
 
-    def _copy_file(self, hook, source_object):
-        destination_object = self.destination_object or source_object
-        if self.destination_object and self.destination_object[-1] == "/":
-            file_name = source_object.split("/")[-1]
-            destination_object += file_name
-        self._copy_single_object(
-            hook=hook, source_object=source_object, destination_object=destination_object
-        )
-
     def _copy_multiple_objects(self, hook, source_objects, prefix):
         # Check whether the prefix is a root directory for all the rest of objects.
         _pref = prefix.rstrip("/")
@@ -441,7 +430,12 @@ class GCSToGCSOperator(BaseOperator):
                 destination_object = source_obj
             else:
                 file_name_postfix = source_obj.replace(base_path, "", 1)
-                destination_object = self.destination_object.rstrip("/") + "/" + file_name_postfix
+
+                destination_object = (
+                    self.destination_object.rstrip("/")[0 : self.destination_object.rfind("/")]
+                    + "/"
+                    + file_name_postfix
+                )
 
             self._copy_single_object(
                 hook=hook, source_object=source_obj, destination_object=destination_object

airflow/providers/google/cloud/triggers/cloud_composer.py

@@ -19,8 +19,13 @@
 from __future__ import annotations
 
 import asyncio
+import json
+from datetime import datetime
 from typing import Any, Sequence
 
+from dateutil import parser
+from google.cloud.orchestration.airflow.service_v1.types import ExecuteAirflowCommandResponse
+
 from airflow.exceptions import AirflowException
 from airflow.providers.google.cloud.hooks.cloud_composer import CloudComposerAsyncHook
 from airflow.triggers.base import BaseTrigger, TriggerEvent
@@ -146,3 +151,113 @@ class CloudComposerAirflowCLICommandTrigger(BaseTrigger):
             }
         )
         return
+
+
+class CloudComposerDAGRunTrigger(BaseTrigger):
+    """The trigger wait for the DAG run completion."""
+
+    def __init__(
+        self,
+        project_id: str,
+        region: str,
+        environment_id: str,
+        composer_dag_id: str,
+        start_date: datetime,
+        end_date: datetime,
+        allowed_states: list[str],
+        gcp_conn_id: str = "google_cloud_default",
+        impersonation_chain: str | Sequence[str] | None = None,
+        poll_interval: int = 10,
+    ):
+        super().__init__()
+        self.project_id = project_id
+        self.region = region
+        self.environment_id = environment_id
+        self.composer_dag_id = composer_dag_id
+        self.start_date = start_date
+        self.end_date = end_date
+        self.allowed_states = allowed_states
+        self.gcp_conn_id = gcp_conn_id
+        self.impersonation_chain = impersonation_chain
+        self.poll_interval = poll_interval
+
+        self.gcp_hook = CloudComposerAsyncHook(
+            gcp_conn_id=self.gcp_conn_id,
+            impersonation_chain=self.impersonation_chain,
+        )
+
+    def serialize(self) -> tuple[str, dict[str, Any]]:
+        return (
+            "airflow.providers.google.cloud.triggers.cloud_composer.CloudComposerDAGRunTrigger",
+            {
+                "project_id": self.project_id,
+                "region": self.region,
+                "environment_id": self.environment_id,
+                "composer_dag_id": self.composer_dag_id,
+                "start_date": self.start_date,
+                "end_date": self.end_date,
+                "allowed_states": self.allowed_states,
+                "gcp_conn_id": self.gcp_conn_id,
+                "impersonation_chain": self.impersonation_chain,
+                "poll_interval": self.poll_interval,
+            },
+        )
+
+    async def _pull_dag_runs(self) -> list[dict]:
+        """Pull the list of dag runs."""
+        dag_runs_cmd = await self.gcp_hook.execute_airflow_command(
+            project_id=self.project_id,
+            region=self.region,
+            environment_id=self.environment_id,
+            command="dags",
+            subcommand="list-runs",
+            parameters=["-d", self.composer_dag_id, "-o", "json"],
+        )
+        cmd_result = await self.gcp_hook.wait_command_execution_result(
+            project_id=self.project_id,
+            region=self.region,
+            environment_id=self.environment_id,
+            execution_cmd_info=ExecuteAirflowCommandResponse.to_dict(dag_runs_cmd),
+        )
+        dag_runs = json.loads(cmd_result["output"][0]["content"])
+        return dag_runs
+
+    def _check_dag_runs_states(
+        self,
+        dag_runs: list[dict],
+        start_date: datetime,
+        end_date: datetime,
+    ) -> bool:
+        for dag_run in dag_runs:
+            if (
+                start_date.timestamp()
+                < parser.parse(dag_run["execution_date"]).timestamp()
+                < end_date.timestamp()
+            ) and dag_run["state"] not in self.allowed_states:
+                return False
+        return True
+
+    async def run(self):
+        try:
+            while True:
+                if datetime.now(self.end_date.tzinfo).timestamp() > self.end_date.timestamp():
+                    dag_runs = await self._pull_dag_runs()
+
+                    self.log.info("Sensor waits for allowed states: %s", self.allowed_states)
+                    if self._check_dag_runs_states(
+                        dag_runs=dag_runs,
+                        start_date=self.start_date,
+                        end_date=self.end_date,
+                    ):
+                        yield TriggerEvent({"status": "success"})
+                        return
+                self.log.info("Sleeping for %s seconds.", self.poll_interval)
+                await asyncio.sleep(self.poll_interval)
+        except AirflowException as ex:
+            yield TriggerEvent(
+                {
+                    "status": "error",
+                    "message": str(ex),
+                }
+            )
+            return

airflow/providers/google/cloud/triggers/kubernetes_engine.py

@@ -142,6 +142,8 @@ class GKEStartPodTrigger(KubernetesPodTrigger):
                 "on_finish_action": self.on_finish_action.value,
                 "gcp_conn_id": self.gcp_conn_id,
                 "impersonation_chain": self.impersonation_chain,
+                "logging_interval": self.logging_interval,
+                "last_log_time": self.last_log_time,
             },
         )
 

airflow/providers/google/cloud/utils/credentials_provider.py

@@ -35,6 +35,9 @@ from google.auth.environment_vars import CREDENTIALS, LEGACY_PROJECT, PROJECT
 
 from airflow.exceptions import AirflowException
 from airflow.providers.google.cloud._internal_client.secret_manager_client import _SecretManagerClient
+from airflow.providers.google.cloud.utils.external_token_supplier import (
+    ClientCredentialsGrantFlowTokenSupplier,
+)
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.utils.process_utils import patch_environ
 
@@ -210,6 +213,10 @@ class _CredentialProvider(LoggingMixin):
         target_principal: str | None = None,
         delegates: Sequence[str] | None = None,
         is_anonymous: bool | None = None,
+        idp_issuer_url: str | None = None,
+        client_id: str | None = None,
+        client_secret: str | None = None,
+        idp_extra_params_dict: dict[str, str] | None = None,
     ) -> None:
         super().__init__()
         key_options = [key_path, keyfile_dict, credential_config_file, key_secret_name, is_anonymous]
@@ -229,6 +236,10 @@ class _CredentialProvider(LoggingMixin):
         self.target_principal = target_principal
         self.delegates = delegates
         self.is_anonymous = is_anonymous
+        self.idp_issuer_url = idp_issuer_url
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.idp_extra_params_dict = idp_extra_params_dict
 
     def get_credentials_and_project(self) -> tuple[Credentials, str]:
         """
@@ -239,7 +250,8 @@ class _CredentialProvider(LoggingMixin):
         :return: Google Auth Credentials
         """
         if self.is_anonymous:
-            credentials, project_id = AnonymousCredentials(), ""
+            credentials: Credentials = AnonymousCredentials()
+            project_id = ""
         else:
             if self.key_path:
                 credentials, project_id = self._get_credentials_using_key_path()
@@ -247,6 +259,10 @@ class _CredentialProvider(LoggingMixin):
                 credentials, project_id = self._get_credentials_using_key_secret_name()
             elif self.keyfile_dict:
                 credentials, project_id = self._get_credentials_using_keyfile_dict()
+            elif self.idp_issuer_url:
+                credentials, project_id = (
+                    self._get_credentials_using_credential_config_file_and_token_supplier()
+                )
             elif self.credential_config_file:
                 credentials, project_id = self._get_credentials_using_credential_config_file()
             else:
@@ -273,10 +289,12 @@ class _CredentialProvider(LoggingMixin):
 
         return credentials, project_id
 
-    def _get_credentials_using_keyfile_dict(self):
+    def _get_credentials_using_keyfile_dict(self) -> tuple[Credentials, str]:
         self._log_debug("Getting connection using JSON Dict")
         # Depending on how the JSON was formatted, it may contain
         # escaped newlines. Convert those to actual newlines.
+        if self.keyfile_dict is None:
+            raise ValueError("The keyfile_dict field is None, and we need it for keyfile_dict auth.")
         self.keyfile_dict["private_key"] = self.keyfile_dict["private_key"].replace("\\n", "\n")
         credentials = google.oauth2.service_account.Credentials.from_service_account_info(
             self.keyfile_dict, scopes=self.scopes
@@ -284,7 +302,9 @@ class _CredentialProvider(LoggingMixin):
         project_id = credentials.project_id
         return credentials, project_id
 
-    def _get_credentials_using_key_path(self):
+    def _get_credentials_using_key_path(self) -> tuple[Credentials, str]:
+        if self.key_path is None:
+            raise ValueError("The ky_path field is None, and we need it for keyfile_dict auth.")
         if self.key_path.endswith(".p12"):
             raise AirflowException("Legacy P12 key file are not supported, use a JSON key file.")
 
@@ -298,13 +318,15 @@ class _CredentialProvider(LoggingMixin):
         project_id = credentials.project_id
         return credentials, project_id
 
-    def _get_credentials_using_key_secret_name(self):
+    def _get_credentials_using_key_secret_name(self) -> tuple[Credentials, str]:
         self._log_debug("Getting connection using JSON key data from GCP secret: %s", self.key_secret_name)
 
         # Use ADC to access GCP Secret Manager.
         adc_credentials, adc_project_id = google.auth.default(scopes=self.scopes)
         secret_manager_client = _SecretManagerClient(credentials=adc_credentials)
 
+        if self.key_secret_name is None:
+            raise ValueError("The key_secret_name field is None, and we need it for keyfile_dict auth.")
         if not secret_manager_client.is_valid_secret_name(self.key_secret_name):
             raise AirflowException("Invalid secret name specified for fetching JSON key data.")
 
@@ -326,7 +348,7 @@ class _CredentialProvider(LoggingMixin):
         project_id = credentials.project_id
         return credentials, project_id
 
-    def _get_credentials_using_credential_config_file(self):
+    def _get_credentials_using_credential_config_file(self) -> tuple[Credentials, str]:
         if isinstance(self.credential_config_file, str) and os.path.exists(self.credential_config_file):
             self._log_info(
                 f"Getting connection using credential configuration file: `{self.credential_config_file}`"
@@ -350,7 +372,25 @@ class _CredentialProvider(LoggingMixin):
 
         return credentials, project_id
 
-    def _get_credentials_using_adc(self):
+    def _get_credentials_using_credential_config_file_and_token_supplier(self):
+        self._log_info(
+            "Getting connection using credential configuration file and external Identity Provider."
+        )
+
+        if not self.credential_config_file:
+            raise AirflowException(
+                "Credential Configuration File is needed to use authentication by External Identity Provider."
+            )
+
+        info = _get_info_from_credential_configuration_file(self.credential_config_file)
+        info["subject_token_supplier"] = ClientCredentialsGrantFlowTokenSupplier(
+            oidc_issuer_url=self.idp_issuer_url, client_id=self.client_id, client_secret=self.client_secret
+        )
+
+        credentials, project_id = google.auth.load_credentials_from_dict(info=info, scopes=self.scopes)
+        return credentials, project_id
+
+    def _get_credentials_using_adc(self) -> tuple[Credentials, str]:
         self._log_info(
             "Getting connection using `google.auth.default()` since no explicit credentials are provided."
         )
@@ -419,3 +459,38 @@ def _get_project_id_from_service_account_email(service_account_email: str) -> st
         raise AirflowException(
             f"Could not extract project_id from service account's email: {service_account_email}."
         )
+
+
+def _get_info_from_credential_configuration_file(
+    credential_configuration_file: str | dict[str, str],
+) -> dict[str, str]:
+    """
+    Extract the Credential Configuration File information, either from a json file, json string or dictionary.
+
+    :param credential_configuration_file: File path or content (as json string or dictionary) of a GCP credential configuration file.
+
+    :return: Returns a dictionary containing the Credential Configuration File information.
+    """
+    # if it's already a dict, just return it
+    if isinstance(credential_configuration_file, dict):
+        return credential_configuration_file
+
+    if not isinstance(credential_configuration_file, str):
+        raise AirflowException(
+            f"Invalid argument type, expected str or dict, got {type(credential_configuration_file)}."
+        )
+
+    if os.path.exists(credential_configuration_file):  # attempts to load from json file
+        with open(credential_configuration_file) as file_obj:
+            try:
+                return json.load(file_obj)
+            except ValueError:
+                raise AirflowException(
+                    f"Credential Configuration File '{credential_configuration_file}' is not a valid json file."
+                )
+
+    # if not a file, attempt to load it from a json string
+    try:
+        return json.loads(credential_configuration_file)
+    except ValueError:
+        raise AirflowException("Credential Configuration File is not a valid json string.")

airflow/providers/google/cloud/utils/external_token_supplier.py

@@ -0,0 +1,175 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import abc
+import time
+from functools import wraps
+from typing import TYPE_CHECKING, Any
+
+import requests
+from google.auth.exceptions import RefreshError
+from google.auth.identity_pool import SubjectTokenSupplier
+
+if TYPE_CHECKING:
+    from google.auth.external_account import SupplierContext
+    from google.auth.transport import Request
+
+from airflow.utils.log.logging_mixin import LoggingMixin
+
+
+def cache_token_decorator(get_subject_token_method):
+    """Cache calls to ``SubjectTokenSupplier`` instances' ``get_token_supplier`` methods.
+
+    Different instances of a same SubjectTokenSupplier class with the same attributes
+    share the OIDC token cache.
+
+    :param get_subject_token_method: A method that returns both a token and an integer specifying
+        the time in seconds until the token expires
+
+    See also:
+        https://googleapis.dev/python/google-auth/latest/reference/google.auth.identity_pool.html#google.auth.identity_pool.SubjectTokenSupplier.get_subject_token
+    """
+    cache = {}
+
+    @wraps(get_subject_token_method)
+    def wrapper(supplier_instance: CacheTokenSupplier, *args, **kwargs) -> str:
+        """Obeys the interface set by ``SubjectTokenSupplier`` for ``get_subject_token`` methods.
+
+        :param supplier_instance: the SubjectTokenSupplier instance whose get_subject_token method is being decorated
+        :return: The token string
+        """
+        nonlocal cache
+
+        cache_key = supplier_instance.get_subject_key()
+        token: dict[str, str | float] = {}
+
+        if cache_key not in cache or cache[cache_key]["expiration_time"] < time.monotonic():
+            supplier_instance.log.info("OIDC token missing or expired")
+            try:
+                access_token, expires_in = get_subject_token_method(supplier_instance, *args, **kwargs)
+                if not isinstance(expires_in, int) or not isinstance(access_token, str):
+                    raise RefreshError  # assume error if strange values are provided
+
+            except RefreshError:
+                supplier_instance.log.error("Failed retrieving new OIDC Token from IdP")
+                raise
+
+            expiration_time = time.monotonic() + float(expires_in)
+            token["access_token"] = access_token
+            token["expiration_time"] = expiration_time
+            cache[cache_key] = token
+
+            supplier_instance.log.info("New OIDC token retrieved, expires in %s seconds.", expires_in)
+
+        return cache[cache_key]["access_token"]
+
+    return wrapper
+
+
+class CacheTokenSupplier(LoggingMixin, SubjectTokenSupplier):
+    """
+    A superclass for all Subject Token Supplier classes that wish to implement a caching mechanism.
+
+    Child classes must implement the ``get_subject_key`` method to generate a string that serves as the cache key,
+    ensuring that tokens are shared appropriately among instances.
+
+    Methods:
+        get_subject_key: Abstract method to be implemented by child classes. It should return a string that serves as the cache key.
+    """
+
+    def __init__(self):
+        super().__init__()
+
+    @abc.abstractmethod
+    def get_subject_key(self) -> str:
+        raise NotImplementedError("")
+
+
+class ClientCredentialsGrantFlowTokenSupplier(CacheTokenSupplier):
+    """
+    Class that retrieves an OIDC token from an external IdP using OAuth2.0 Client Credentials Grant flow.
+
+    This class implements the ``SubjectTokenSupplier`` interface class used by ``google.auth.identity_pool.Credentials``
+
+    :params oidc_issuer_url: URL of the IdP that performs OAuth2.0 Client Credentials Grant flow and returns an OIDC token.
+    :params client_id: Client ID of the application requesting the token
+    :params client_secret: Client secret of the application requesting the token
+    :params extra_params_kwargs: Extra parameters to be passed in the payload of the POST request to the `oidc_issuer_url`
+
+    See also:
+        https://googleapis.dev/python/google-auth/latest/reference/google.auth.identity_pool.html#google.auth.identity_pool.SubjectTokenSupplier
+    """
+
+    def __init__(
+        self,
+        oidc_issuer_url: str,
+        client_id: str,
+        client_secret: str,
+        **extra_params_kwargs: Any,
+    ) -> None:
+        super().__init__()
+        self.oidc_issuer_url = oidc_issuer_url
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.extra_params_kwargs = extra_params_kwargs
+
+    @cache_token_decorator
+    def get_subject_token(self, context: SupplierContext, request: Request) -> tuple[str, int]:
+        """Perform Client Credentials Grant flow with IdP and retrieves an OIDC token and expiration time."""
+        self.log.info("Requesting new OIDC token from external IdP.")
+        try:
+            response = requests.post(
+                self.oidc_issuer_url,
+                data={
+                    "grant_type": "client_credentials",
+                    "client_id": self.client_id,
+                    "client_secret": self.client_secret,
+                    **self.extra_params_kwargs,
+                },
+            )
+            response.raise_for_status()
+        except requests.HTTPError as e:
+            raise RefreshError(str(e))
+        except requests.ConnectionError as e:
+            raise RefreshError(str(e))
+
+        try:
+            response_dict = response.json()
+        except requests.JSONDecodeError:
+            raise RefreshError(f"Didn't get a json response from {self.oidc_issuer_url}")
+
+        # These fields are required
+        if {"access_token", "expires_in"} - set(response_dict.keys()):
+            # TODO more information about the error can be provided in the exception by inspecting the response
+            raise RefreshError(f"No access token returned from {self.oidc_issuer_url}")
+
+        return response_dict["access_token"], response_dict["expires_in"]
+
+    def get_subject_key(self) -> str:
+        """
+        Create a cache key using the OIDC issuer URL, client ID, client secret and additional parameters.
+
+        Instances with the same credentials will share tokens.
+        """
+        cache_key = (
+            self.oidc_issuer_url
+            + self.client_id
+            + self.client_secret
+            + ",".join(sorted(self.extra_params_kwargs))
+        )
+        return cache_key

airflow/providers/google/common/hooks/base_google.py

@@ -248,6 +248,20 @@ class GoogleBaseHook(BaseHook):
             "impersonation_chain": StringField(
                 lazy_gettext("Impersonation Chain"), widget=BS3TextFieldWidget()
             ),
+            "idp_issuer_url": StringField(
+                lazy_gettext("IdP Token Issue URL (Client Credentials Grant Flow)"),
+                widget=BS3TextFieldWidget(),
+            ),
+            "client_id": StringField(
+                lazy_gettext("Client ID (Client Credentials Grant Flow)"), widget=BS3TextFieldWidget()
+            ),
+            "client_secret": StringField(
+                lazy_gettext("Client Secret (Client Credentials Grant Flow)"),
+                widget=BS3PasswordFieldWidget(),
+            ),
+            "idp_extra_parameters": StringField(
+                lazy_gettext("IdP Extra Request Parameters"), widget=BS3TextFieldWidget()
+            ),
             "is_anonymous": BooleanField(
                 lazy_gettext("Anonymous credentials (ignores all other settings)"), default=False
             ),
@@ -305,6 +319,18 @@ class GoogleBaseHook(BaseHook):
         target_principal, delegates = _get_target_principal_and_delegates(self.impersonation_chain)
         is_anonymous = self._get_field("is_anonymous")
 
+        idp_issuer_url: str | None = self._get_field("idp_issuer_url", None)
+        client_id: str | None = self._get_field("client_id", None)
+        client_secret: str | None = self._get_field("client_secret", None)
+        idp_extra_params: str | None = self._get_field("idp_extra_params", None)
+
+        idp_extra_params_dict: dict[str, str] | None = None
+        if idp_extra_params:
+            try:
+                idp_extra_params_dict = json.loads(idp_extra_params)
+            except json.decoder.JSONDecodeError:
+                raise AirflowException("Invalid JSON.")
+
         credentials, project_id = get_credentials_and_project_id(
             key_path=key_path,
             keyfile_dict=keyfile_dict_json,
@@ -316,6 +342,10 @@ class GoogleBaseHook(BaseHook):
             target_principal=target_principal,
             delegates=delegates,
             is_anonymous=is_anonymous,
+            idp_issuer_url=idp_issuer_url,
+            client_id=client_id,
+            client_secret=client_secret,
+            idp_extra_params_dict=idp_extra_params_dict,
         )
 
         overridden_project_id = self._get_field("project")
@@ -731,7 +761,11 @@ class GoogleBaseAsyncHook(BaseHook):
 
     sync_hook_class: Any = None
 
-    def __init__(self, **kwargs: Any):
+    def __init__(self, **kwargs: Any) -> None:
+        # add default value to gcp_conn_id
+        if "gcp_conn_id" not in kwargs:
+            kwargs["gcp_conn_id"] = "google_cloud_default"
+
         self._hook_kwargs = kwargs
         self._sync_hook = None
 

airflow/providers/google/common/utils/id_token_credentials.py

@@ -190,7 +190,7 @@ def _get_gce_credentials(
 
 
 def get_default_id_token_credentials(
-    target_audience: str | None, request: google.auth.transport.Request = None
+    target_audience: str | None, request: google.auth.transport.Request | None = None
 ) -> google_auth_credentials.Credentials:
     """Get the default ID Token credentials for the current environment.