apache-airflow-providers-google 10.17.0rc1__py3-none-any.whl → 10.18.0__py3-none-any.whl

airflow/providers/google/cloud/operators/bigquery.py

@@ -29,7 +29,7 @@ from typing import TYPE_CHECKING, Any, Iterable, Sequence, SupportsAbs
 import attr
 from deprecated import deprecated
 from google.api_core.exceptions import Conflict
-from google.cloud.bigquery import DEFAULT_RETRY, CopyJob, ExtractJob, LoadJob, QueryJob
+from google.cloud.bigquery import DEFAULT_RETRY, CopyJob, ExtractJob, LoadJob, QueryJob, Row
 from google.cloud.bigquery.table import RowIterator
 
 from airflow.configuration import conf
@@ -56,6 +56,8 @@ from airflow.providers.google.cloud.triggers.bigquery import (
     BigQueryValueCheckTrigger,
 )
 from airflow.providers.google.cloud.utils.bigquery import convert_job_id
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
+from airflow.utils.helpers import exactly_one
 
 if TYPE_CHECKING:
     from google.api_core.retry import Retry
@@ -66,7 +68,7 @@ if TYPE_CHECKING:
 
 BIGQUERY_JOB_DETAILS_LINK_FMT = "https://console.cloud.google.com/bigquery?j={job_id}"
 
-LABEL_REGEX = re.compile(r"^[a-z][\w-]{0,63}$")
+LABEL_REGEX = re.compile(r"^[\w-]{0,63}$")
 
 
 class BigQueryUIColors(enum.Enum):
@@ -201,7 +203,25 @@ class _BigQueryOpenLineageMixin:
         )
 
 
-class BigQueryCheckOperator(_BigQueryDbHookMixin, SQLCheckOperator):
+class _BigQueryOperatorsEncryptionConfigurationMixin:
+    """A class to handle the configuration for BigQueryHook.insert_job method."""
+
+    # Note: If you want to add this feature to a new operator you can include the class name in the type
+    # annotation of the `self`. Then you can inherit this class in the target operator.
+    # e.g: BigQueryCheckOperator, BigQueryTableCheckOperator
+    def include_encryption_configuration(  # type:ignore[misc]
+        self: BigQueryCheckOperator | BigQueryTableCheckOperator,
+        configuration: dict,
+        config_key: str,
+    ) -> None:
+        """Add encryption_configuration to destinationEncryptionConfiguration key if it is not None."""
+        if self.encryption_configuration is not None:
+            configuration[config_key]["destinationEncryptionConfiguration"] = self.encryption_configuration
+
+
+class BigQueryCheckOperator(
+    _BigQueryDbHookMixin, SQLCheckOperator, _BigQueryOperatorsEncryptionConfigurationMixin
+):
     """Performs checks against BigQuery.
 
     This operator expects a SQL query that returns a single row. Each value on
@@ -246,6 +266,13 @@ class BigQueryCheckOperator(_BigQueryDbHookMixin, SQLCheckOperator):
         Token Creator IAM role to the directly preceding identity, with first
         account from the list granting this role to the originating account. (templated)
     :param labels: a dictionary containing labels for the table, passed to BigQuery.
+    :param encryption_configuration: [Optional] Custom encryption configuration (e.g., Cloud KMS keys).
+
+        .. code-block:: python
+
+            encryption_configuration = {
+                "kmsKeyName": "projects/PROJECT/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY",
+            }
     :param deferrable: Run operator in the deferrable mode.
     :param poll_interval: (Deferrable mode only) polling period in seconds to
         check for the status of job.
@@ -270,6 +297,7 @@ class BigQueryCheckOperator(_BigQueryDbHookMixin, SQLCheckOperator):
         location: str | None = None,
         impersonation_chain: str | Sequence[str] | None = None,
         labels: dict | None = None,
+        encryption_configuration: dict | None = None,
         deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         poll_interval: float = 4.0,
         **kwargs,
@@ -280,6 +308,7 @@ class BigQueryCheckOperator(_BigQueryDbHookMixin, SQLCheckOperator):
         self.location = location
         self.impersonation_chain = impersonation_chain
         self.labels = labels
+        self.encryption_configuration = encryption_configuration
         self.deferrable = deferrable
         self.poll_interval = poll_interval
 
@@ -291,6 +320,8 @@ class BigQueryCheckOperator(_BigQueryDbHookMixin, SQLCheckOperator):
         """Submit a new job and get the job id for polling the status using Trigger."""
         configuration = {"query": {"query": self.sql, "useLegacySql": self.use_legacy_sql}}
 
+        self.include_encryption_configuration(configuration, "query")
+
         return hook.insert_job(
             configuration=configuration,
             project_id=hook.project_id,
@@ -322,8 +353,26 @@ class BigQueryCheckOperator(_BigQueryDbHookMixin, SQLCheckOperator):
                     ),
                     method_name="execute_complete",
                 )
+            self._handle_job_error(job)
+            # job.result() returns a RowIterator. Mypy expects an instance of SupportsNext[Any] for
+            # the next() call which the RowIterator does not resemble to. Hence, ignore the arg-type error.
+            # Row passed to _validate_records is a collection of values only, without column names.
+            self._validate_records(next(iter(job.result()), []))  # type: ignore[arg-type]
             self.log.info("Current state of job %s is %s", job.job_id, job.state)
 
+    @staticmethod
+    def _handle_job_error(job: BigQueryJob | UnknownJob) -> None:
+        if job.error_result:
+            raise AirflowException(f"BigQuery job {job.job_id} failed: {job.error_result}")
+
+    def _validate_records(self, records) -> None:
+        if not records:
+            raise AirflowException(f"The following query returned zero rows: {self.sql}")
+        elif not all(records):
+            self._raise_exception(  # type: ignore[attr-defined]
+                f"Test failed.\nQuery:\n{self.sql}\nResults:\n{records!s}"
+            )
+
     def execute_complete(self, context: Context, event: dict[str, Any]) -> None:
         """Act as a callback for when the trigger fires.
 
@@ -333,13 +382,7 @@ class BigQueryCheckOperator(_BigQueryDbHookMixin, SQLCheckOperator):
         if event["status"] == "error":
             raise AirflowException(event["message"])
 
-        records = event["records"]
-        if not records:
-            raise AirflowException("The query returned empty results")
-        elif not all(records):
-            self._raise_exception(  # type: ignore[attr-defined]
-                f"Test failed.\nQuery:\n{self.sql}\nResults:\n{records!s}"
-            )
+        self._validate_records(event["records"])
         self.log.info("Record: %s", event["records"])
         self.log.info("Success.")
 
@@ -454,8 +497,8 @@ class BigQueryValueCheckOperator(_BigQueryDbHookMixin, SQLValueCheckOperator):
             self._handle_job_error(job)
             # job.result() returns a RowIterator. Mypy expects an instance of SupportsNext[Any] for
             # the next() call which the RowIterator does not resemble to. Hence, ignore the arg-type error.
-            records = next(job.result())  # type: ignore[arg-type]
-            self.check_value(records)  # type: ignore[attr-defined]
+            # Row passed to check_value is a collection of values only, without column names.
+            self.check_value(next(iter(job.result()), []))  # type: ignore[arg-type]
             self.log.info("Current state of job %s is %s", job.job_id, job.state)
 
     @staticmethod
@@ -542,7 +585,7 @@ class BigQueryIntervalCheckOperator(_BigQueryDbHookMixin, SQLIntervalCheckOperat
         labels: dict | None = None,
         deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         poll_interval: float = 4.0,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         **kwargs,
     ) -> None:
         super().__init__(
@@ -753,7 +796,9 @@ class BigQueryColumnCheckOperator(_BigQueryDbHookMixin, SQLColumnCheckOperator):
         self.log.info("All tests have passed")
 
 
-class BigQueryTableCheckOperator(_BigQueryDbHookMixin, SQLTableCheckOperator):
+class BigQueryTableCheckOperator(
+    _BigQueryDbHookMixin, SQLTableCheckOperator, _BigQueryOperatorsEncryptionConfigurationMixin
+):
     """
     Subclasses the SQLTableCheckOperator in order to provide a job id for OpenLineage to parse.
 
@@ -781,6 +826,13 @@ class BigQueryTableCheckOperator(_BigQueryDbHookMixin, SQLTableCheckOperator):
         Service Account Token Creator IAM role to the directly preceding identity, with first
         account from the list granting this role to the originating account (templated).
     :param labels: a dictionary containing labels for the table, passed to BigQuery
+    :param encryption_configuration: [Optional] Custom encryption configuration (e.g., Cloud KMS keys).
+
+        .. code-block:: python
+
+            encryption_configuration = {
+                "kmsKeyName": "projects/PROJECT/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY",
+            }
     """
 
     template_fields: Sequence[str] = tuple(set(SQLTableCheckOperator.template_fields) | {"gcp_conn_id"})
@@ -798,6 +850,7 @@ class BigQueryTableCheckOperator(_BigQueryDbHookMixin, SQLTableCheckOperator):
         location: str | None = None,
         impersonation_chain: str | Sequence[str] | None = None,
         labels: dict | None = None,
+        encryption_configuration: dict | None = None,
         **kwargs,
     ) -> None:
         super().__init__(table=table, checks=checks, partition_clause=partition_clause, **kwargs)
@@ -806,6 +859,7 @@ class BigQueryTableCheckOperator(_BigQueryDbHookMixin, SQLTableCheckOperator):
         self.location = location
         self.impersonation_chain = impersonation_chain
         self.labels = labels
+        self.encryption_configuration = encryption_configuration
 
     def _submit_job(
         self,
@@ -815,6 +869,8 @@ class BigQueryTableCheckOperator(_BigQueryDbHookMixin, SQLTableCheckOperator):
         """Submit a new job and get the job id for polling the status using Trigger."""
         configuration = {"query": {"query": self.sql, "useLegacySql": self.use_legacy_sql}}
 
+        self.include_encryption_configuration(configuration, "query")
+
         return hook.insert_job(
             configuration=configuration,
             project_id=hook.project_id,
@@ -858,9 +914,10 @@ class BigQueryTableCheckOperator(_BigQueryDbHookMixin, SQLTableCheckOperator):
 
 class BigQueryGetDataOperator(GoogleCloudBaseOperator):
     """
-    Fetches the data from a BigQuery table (alternatively fetch data for selected columns) and returns data.
+    Fetch data and return it, either from a BigQuery table, or results of a query job.
 
-    Data is returned in either of the following two formats, based on "as_dict" value:
+    Data could be narrowed down by specific columns or retrieved as a whole.
+    It is returned in either of the following two formats, based on "as_dict" value:
     1. False (Default) - A Python list of lists, with the number of nested lists equal to the number of rows
     fetched. Each nested list represents a row, where the elements within it correspond to the column values
     for that particular row.
@@ -880,27 +937,42 @@ class BigQueryGetDataOperator(GoogleCloudBaseOperator):
     .. note::
         If you pass fields to ``selected_fields`` which are in different order than the
         order of columns already in
-        BQ table, the data will still be in the order of BQ table.
+        BQ table/job, the data will still be in the order of BQ table.
         For example if the BQ table has 3 columns as
         ``[A,B,C]`` and you pass 'B,A' in the ``selected_fields``
         the data would still be of the form ``'A,B'``.
 
-    **Example**::
+    .. note::
+        When utilizing job id not in deferrable mode, the job should be in DONE state.
+
+    **Example - Retrieve data from BigQuery using table**::
 
         get_data = BigQueryGetDataOperator(
             task_id="get_data_from_bq",
             dataset_id="test_dataset",
             table_id="Transaction_partitions",
-            project_id="internal-gcp-project",
+            table_project_id="internal-gcp-project",
+            max_results=100,
+            selected_fields="DATE",
+            gcp_conn_id="airflow-conn-id",
+        )
+
+    **Example - Retrieve data from BigQuery using a job id**::
+
+        get_data = BigQueryGetDataOperator(
+            job_id="airflow_8999918812727394_86a1cecc69c5e3028d28247affd7563",
+            job_project_id="internal-gcp-project",
             max_results=100,
             selected_fields="DATE",
             gcp_conn_id="airflow-conn-id",
         )
 
     :param dataset_id: The dataset ID of the requested table. (templated)
-    :param table_id: The table ID of the requested table. (templated)
+    :param table_id: The table ID of the requested table. Mutually exclusive with job_id. (templated)
     :param table_project_id: (Optional) The project ID of the requested table.
         If None, it will be derived from the hook's project ID. (templated)
+    :param job_id: The job ID from which query results are retrieved.
+        Mutually exclusive with table_id. (templated)
     :param job_project_id: (Optional) Google Cloud Project where the job is running.
         If None, it will be derived from the hook's project ID. (templated)
     :param project_id: (Deprecated) (Optional) The name of the project where the data
@@ -931,6 +1003,7 @@ class BigQueryGetDataOperator(GoogleCloudBaseOperator):
         "dataset_id",
         "table_id",
         "table_project_id",
+        "job_id",
         "job_project_id",
         "project_id",
         "max_results",
@@ -942,11 +1015,12 @@ class BigQueryGetDataOperator(GoogleCloudBaseOperator):
     def __init__(
         self,
         *,
-        dataset_id: str,
-        table_id: str,
+        dataset_id: str | None = None,
+        table_id: str | None = None,
         table_project_id: str | None = None,
+        job_id: str | None = None,
         job_project_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         max_results: int = 100,
         selected_fields: str | None = None,
         gcp_conn_id: str = "google_cloud_default",
@@ -964,6 +1038,7 @@ class BigQueryGetDataOperator(GoogleCloudBaseOperator):
         self.dataset_id = dataset_id
         self.table_id = table_id
         self.job_project_id = job_project_id
+        self.job_id = job_id
         self.max_results = max_results
         self.selected_fields = selected_fields
         self.gcp_conn_id = gcp_conn_id
@@ -1000,7 +1075,7 @@ class BigQueryGetDataOperator(GoogleCloudBaseOperator):
             query += "*"
         query += (
             f" from `{self.table_project_id or hook.project_id}.{self.dataset_id}"
-            f".{self.table_id}` limit {int(self.max_results)}"
+            f".{self.table_id}` limit {self.max_results}"
         )
         return query
 
@@ -1013,7 +1088,13 @@ class BigQueryGetDataOperator(GoogleCloudBaseOperator):
             if not self.table_project_id:
                 self.table_project_id = self.project_id
             else:
-                self.log.info("Ignoring project_id parameter, as table_project_id is found.")
+                self.log.info("Ignoring 'project_id' parameter, as 'table_project_id' is found.")
+
+        if not exactly_one(self.job_id, self.table_id):
+            raise AirflowException(
+                "'job_id' and 'table_id' parameters are mutually exclusive, "
+                "ensure that exactly one of them is specified"
+            )
 
         hook = BigQueryHook(
             gcp_conn_id=self.gcp_conn_id,
@@ -1022,31 +1103,45 @@ class BigQueryGetDataOperator(GoogleCloudBaseOperator):
         )
 
         if not self.deferrable:
-            self.log.info(
-                "Fetching Data from %s.%s.%s max results: %s",
-                self.table_project_id or hook.project_id,
-                self.dataset_id,
-                self.table_id,
-                int(self.max_results),
-            )
-            if not self.selected_fields:
-                schema: dict[str, list] = hook.get_schema(
+            if not self.job_id:
+                self.log.info(
+                    "Fetching Data from %s.%s.%s max results: %s",
+                    self.table_project_id or hook.project_id,
+                    self.dataset_id,
+                    self.table_id,
+                    self.max_results,
+                )
+                if not self.selected_fields:
+                    schema: dict[str, list] = hook.get_schema(
+                        dataset_id=self.dataset_id,
+                        table_id=self.table_id,
+                        project_id=self.table_project_id or hook.project_id,
+                    )
+                    if "fields" in schema:
+                        self.selected_fields = ",".join([field["name"] for field in schema["fields"]])
+                rows: list[Row] | RowIterator | list[dict[str, Any]] = hook.list_rows(
                     dataset_id=self.dataset_id,
                     table_id=self.table_id,
+                    max_results=self.max_results,
+                    selected_fields=self.selected_fields,
+                    location=self.location,
                     project_id=self.table_project_id or hook.project_id,
                 )
-                if "fields" in schema:
-                    self.selected_fields = ",".join([field["name"] for field in schema["fields"]])
-
-            rows = hook.list_rows(
-                dataset_id=self.dataset_id,
-                table_id=self.table_id,
-                max_results=int(self.max_results),
-                selected_fields=self.selected_fields,
-                location=self.location,
-                project_id=self.table_project_id or hook.project_id,
-            )
-
+            else:
+                self.log.info(
+                    "Fetching data from job '%s:%s.%s' max results: %s",
+                    self.job_project_id or hook.project_id,
+                    self.location,
+                    self.job_id,
+                    self.max_results,
+                )
+                rows = hook.get_query_results(
+                    job_id=self.job_id,
+                    location=self.location,
+                    selected_fields=self.selected_fields,
+                    max_results=self.max_results,
+                    project_id=self.job_project_id or hook.project_id,
+                )
             if isinstance(rows, RowIterator):
                 raise TypeError(
                     "BigQueryHook.list_rows() returns iterator when return_iterator is False (default)"
@@ -1056,11 +1151,16 @@ class BigQueryGetDataOperator(GoogleCloudBaseOperator):
             if self.as_dict:
                 table_data = [dict(row) for row in rows]
             else:
-                table_data = [row.values() for row in rows]
+                table_data = [row.values() if isinstance(row, Row) else list(row.values()) for row in rows]
 
             return table_data
 
-        job = self._submit_job(hook, job_id="")
+        if not self.job_id:
+            job: BigQueryJob | UnknownJob = self._submit_job(hook, job_id="")
+        else:
+            job = hook.get_job(
+                job_id=self.job_id, project_id=self.job_project_id or hook.project_id, location=self.location
+            )
 
         context["ti"].xcom_push(key="job_id", value=job.job_id)
         self.defer(
@@ -1075,6 +1175,7 @@ class BigQueryGetDataOperator(GoogleCloudBaseOperator):
                 poll_interval=self.poll_interval,
                 as_dict=self.as_dict,
                 impersonation_chain=self.impersonation_chain,
+                selected_fields=self.selected_fields,
             ),
             method_name="execute_complete",
         )
@@ -1163,7 +1264,7 @@ class BigQueryExecuteQueryOperator(GoogleCloudBaseOperator):
         .. code-block:: python
 
             encryption_configuration = {
-                "kmsKeyName": "projects/testp/locations/us/keyRings/test-kr/cryptoKeys/test-key",
+                "kmsKeyName": "projects/PROJECT/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY",
             }
     :param impersonation_chain: Optional service account to impersonate using short-term
         credentials, or chained list of accounts required to get the access_token
@@ -1403,7 +1504,7 @@ class BigQueryCreateEmptyTableOperator(GoogleCloudBaseOperator):
         .. code-block:: python
 
             encryption_configuration = {
-                "kmsKeyName": "projects/testp/locations/us/keyRings/test-kr/cryptoKeys/test-key",
+                "kmsKeyName": "projects/PROJECT/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY",
             }
     :param location: The location used for the operation.
     :param cluster_fields: [Optional] The fields used for clustering.
@@ -1447,7 +1548,7 @@ class BigQueryCreateEmptyTableOperator(GoogleCloudBaseOperator):
         dataset_id: str,
         table_id: str,
         table_resource: dict[str, Any] | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         schema_fields: list | None = None,
         gcs_schema_object: str | None = None,
         time_partitioning: dict | None = None,
@@ -1631,7 +1732,7 @@ class BigQueryCreateExternalTableOperator(GoogleCloudBaseOperator):
         .. code-block:: python
 
             encryption_configuration = {
-                "kmsKeyName": "projects/testp/locations/us/keyRings/test-kr/cryptoKeys/test-key",
+                "kmsKeyName": "projects/PROJECT/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY",
             }
     :param location: The location used for the operation.
     :param impersonation_chain: Optional service account to impersonate using short-term
@@ -1907,7 +2008,7 @@ class BigQueryDeleteDatasetOperator(GoogleCloudBaseOperator):
         self,
         *,
         dataset_id: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         delete_contents: bool = False,
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -1987,7 +2088,7 @@ class BigQueryCreateEmptyDatasetOperator(GoogleCloudBaseOperator):
         self,
         *,
         dataset_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         dataset_reference: dict | None = None,
         location: str | None = None,
         gcp_conn_id: str = "google_cloud_default",
@@ -2091,7 +2192,7 @@ class BigQueryGetDatasetOperator(GoogleCloudBaseOperator):
         self,
         *,
         dataset_id: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
         **kwargs,
@@ -2154,7 +2255,7 @@ class BigQueryGetDatasetTablesOperator(GoogleCloudBaseOperator):
         self,
         *,
         dataset_id: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         max_results: int | None = None,
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -2223,7 +2324,7 @@ class BigQueryPatchDatasetOperator(GoogleCloudBaseOperator):
         *,
         dataset_id: str,
         dataset_resource: dict,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
         **kwargs,
@@ -2296,7 +2397,7 @@ class BigQueryUpdateTableOperator(GoogleCloudBaseOperator):
         fields: list[str] | None = None,
         dataset_id: str | None = None,
         table_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
         **kwargs,
@@ -2380,7 +2481,7 @@ class BigQueryUpdateDatasetOperator(GoogleCloudBaseOperator):
         dataset_resource: dict[str, Any],
         fields: list[str] | None = None,
         dataset_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
         **kwargs,
@@ -2516,7 +2617,7 @@ class BigQueryUpsertTableOperator(GoogleCloudBaseOperator):
         *,
         dataset_id: str,
         table_resource: dict,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         location: str | None = None,
         impersonation_chain: str | Sequence[str] | None = None,
@@ -2623,7 +2724,7 @@ class BigQueryUpdateTableSchemaOperator(GoogleCloudBaseOperator):
         dataset_id: str,
         table_id: str,
         include_policy_tags: bool = False,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
         **kwargs,
@@ -2732,7 +2833,7 @@ class BigQueryInsertJobOperator(GoogleCloudBaseOperator, _BigQueryOpenLineageMix
     def __init__(
         self,
         configuration: dict[str, Any],
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         location: str | None = None,
         job_id: str | None = None,
         force_rerun: bool = True,
@@ -2903,6 +3004,7 @@ class BigQueryInsertJobOperator(GoogleCloudBaseOperator, _BigQueryOpenLineageMix
                         location=self.location or hook.location,
                         poll_interval=self.poll_interval,
                         impersonation_chain=self.impersonation_chain,
+                        cancel_on_kill=self.cancel_on_kill,
                     ),
                     method_name="execute_complete",
                 )

airflow/providers/google/cloud/operators/bigquery_dts.py

@@ -37,6 +37,7 @@ from airflow.providers.google.cloud.hooks.bigquery_dts import BiqQueryDataTransf
 from airflow.providers.google.cloud.links.bigquery_dts import BigQueryDataTransferConfigLink
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
 from airflow.providers.google.cloud.triggers.bigquery_dts import BigQueryDataTransferRunTrigger
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
 
 if TYPE_CHECKING:
     from google.api_core.retry import Retry
@@ -94,7 +95,7 @@ class BigQueryCreateDataTransferOperator(GoogleCloudBaseOperator):
         self,
         *,
         transfer_config: dict,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         location: str | None = None,
         authorization_code: str | None = None,
         retry: Retry | _MethodDefault = DEFAULT,
@@ -187,7 +188,7 @@ class BigQueryDeleteDataTransferConfigOperator(GoogleCloudBaseOperator):
         self,
         *,
         transfer_config_id: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         location: str | None = None,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
@@ -273,7 +274,7 @@ class BigQueryDataTransferServiceStartTransferRunsOperator(GoogleCloudBaseOperat
         self,
         *,
         transfer_config_id: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         location: str | None = None,
         requested_time_range: dict | None = None,
         requested_run_time: dict | None = None,

airflow/providers/google/cloud/operators/bigtable.py

@@ -31,6 +31,7 @@ from airflow.providers.google.cloud.links.bigtable import (
     BigtableTablesLink,
 )
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
 
 if TYPE_CHECKING:
     import enum
@@ -112,7 +113,7 @@ class BigtableCreateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
         instance_id: str,
         main_cluster_id: str,
         main_cluster_zone: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         replica_clusters: list[dict[str, str]] | None = None,
         instance_display_name: str | None = None,
         instance_type: enums.Instance.Type | None = None,
@@ -218,7 +219,7 @@ class BigtableUpdateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
         self,
         *,
         instance_id: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         instance_display_name: str | None = None,
         instance_type: enums.Instance.Type | enum.IntEnum | None = None,
         instance_labels: dict | None = None,
@@ -298,7 +299,7 @@ class BigtableDeleteInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
         self,
         *,
         instance_id: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
         **kwargs,
@@ -374,7 +375,7 @@ class BigtableCreateTableOperator(GoogleCloudBaseOperator, BigtableValidationMix
         *,
         instance_id: str,
         table_id: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         initial_split_keys: list | None = None,
         column_families: dict[str, GarbageCollectionRule] | None = None,
         gcp_conn_id: str = "google_cloud_default",
@@ -478,7 +479,7 @@ class BigtableDeleteTableOperator(GoogleCloudBaseOperator, BigtableValidationMix
         *,
         instance_id: str,
         table_id: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         app_profile_id: str | None = None,
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -559,7 +560,7 @@ class BigtableUpdateClusterOperator(GoogleCloudBaseOperator, BigtableValidationM
         instance_id: str,
         cluster_id: str,
         nodes: int,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
         **kwargs,