apache-airflow-providers-google 10.17.0rc1__py3-none-any.whl → 10.18.0rc1__py3-none-any.whl

airflow/providers/google/cloud/operators/compute.py

@@ -36,6 +36,7 @@ from airflow.providers.google.cloud.links.compute import (
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
 from airflow.providers.google.cloud.utils.field_sanitizer import GcpBodyFieldSanitizer
 from airflow.providers.google.cloud.utils.field_validator import GcpBodyFieldValidator
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
 
 if TYPE_CHECKING:
     from google.api_core.retry import Retry
@@ -51,7 +52,7 @@ class ComputeEngineBaseOperator(GoogleCloudBaseOperator):
         *,
         zone: str,
         resource_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -138,7 +139,7 @@ class ComputeEngineInsertInstanceOperator(ComputeEngineBaseOperator):
         body: dict,
         zone: str,
         resource_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         request_id: str | None = None,
         retry: Retry | None = None,
         timeout: float | None = None,
@@ -320,7 +321,7 @@ class ComputeEngineInsertInstanceFromTemplateOperator(ComputeEngineBaseOperator)
         body: dict,
         zone: str,
         resource_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         request_id: str | None = None,
         retry: Retry | None = None,
         timeout: float | None = None,
@@ -477,7 +478,7 @@ class ComputeEngineDeleteInstanceOperator(ComputeEngineBaseOperator):
         resource_id: str,
         zone: str,
         request_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | None = None,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -721,7 +722,7 @@ class ComputeEngineSetMachineTypeOperator(ComputeEngineBaseOperator):
         zone: str,
         resource_id: str,
         body: dict,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1",
         validate_body: bool = True,
@@ -882,7 +883,7 @@ class ComputeEngineInsertInstanceTemplateOperator(ComputeEngineBaseOperator):
         self,
         *,
         body: dict,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         resource_id: str | None = None,
         request_id: str | None = None,
         retry: Retry | None = None,
@@ -1044,7 +1045,7 @@ class ComputeEngineDeleteInstanceTemplateOperator(ComputeEngineBaseOperator):
         *,
         resource_id: str,
         request_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | None = None,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1164,7 +1165,7 @@ class ComputeEngineCopyInstanceTemplateOperator(ComputeEngineBaseOperator):
         *,
         resource_id: str,
         body_patch: dict,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         request_id: str | None = None,
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1",
@@ -1323,7 +1324,7 @@ class ComputeEngineInstanceGroupUpdateManagerTemplateOperator(ComputeEngineBaseO
         zone: str,
         source_template: str,
         destination_template: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         update_policy: dict[str, Any] | None = None,
         request_id: str | None = None,
         gcp_conn_id: str = "google_cloud_default",
@@ -1468,7 +1469,7 @@ class ComputeEngineInsertInstanceGroupManagerOperator(ComputeEngineBaseOperator)
         *,
         body: dict,
         zone: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         resource_id: str | None = None,
         request_id: str | None = None,
         gcp_conn_id: str = "google_cloud_default",
@@ -1630,7 +1631,7 @@ class ComputeEngineDeleteInstanceGroupManagerOperator(ComputeEngineBaseOperator)
         *,
         resource_id: str,
         zone: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         request_id: str | None = None,
         gcp_conn_id: str = "google_cloud_default",
         api_version="v1",

airflow/providers/google/cloud/operators/datacatalog.py

@@ -38,6 +38,7 @@ from airflow.providers.google.cloud.links.datacatalog import (
     DataCatalogTagTemplateLink,
 )
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
 
 if TYPE_CHECKING:
     from google.api_core.retry import Retry
@@ -104,7 +105,7 @@ class CloudDataCatalogCreateEntryOperator(GoogleCloudBaseOperator):
         entry_group: str,
         entry_id: str,
         entry: dict | Entry,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -220,7 +221,7 @@ class CloudDataCatalogCreateEntryGroupOperator(GoogleCloudBaseOperator):
         location: str,
         entry_group_id: str,
         entry_group: dict | EntryGroup,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -336,7 +337,7 @@ class CloudDataCatalogCreateTagOperator(GoogleCloudBaseOperator):
         entry: str,
         tag: dict | Tag,
         template_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -467,7 +468,7 @@ class CloudDataCatalogCreateTagTemplateOperator(GoogleCloudBaseOperator):
         location: str,
         tag_template_id: str,
         tag_template: dict | TagTemplate,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -583,7 +584,7 @@ class CloudDataCatalogCreateTagTemplateFieldOperator(GoogleCloudBaseOperator):
         tag_template: str,
         tag_template_field_id: str,
         tag_template_field: dict | TagTemplateField,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -690,7 +691,7 @@ class CloudDataCatalogDeleteEntryOperator(GoogleCloudBaseOperator):
         location: str,
         entry_group: str,
         entry: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -774,7 +775,7 @@ class CloudDataCatalogDeleteEntryGroupOperator(GoogleCloudBaseOperator):
         *,
         location: str,
         entry_group: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -860,7 +861,7 @@ class CloudDataCatalogDeleteTagOperator(GoogleCloudBaseOperator):
         entry_group: str,
         entry: str,
         tag: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -949,7 +950,7 @@ class CloudDataCatalogDeleteTagTemplateOperator(GoogleCloudBaseOperator):
         location: str,
         tag_template: str,
         force: bool,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1037,7 +1038,7 @@ class CloudDataCatalogDeleteTagTemplateFieldOperator(GoogleCloudBaseOperator):
         tag_template: str,
         field: str,
         force: bool,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1125,7 +1126,7 @@ class CloudDataCatalogGetEntryOperator(GoogleCloudBaseOperator):
         location: str,
         entry_group: str,
         entry: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1220,7 +1221,7 @@ class CloudDataCatalogGetEntryGroupOperator(GoogleCloudBaseOperator):
         location: str,
         entry_group: str,
         read_mask: FieldMask,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1308,7 +1309,7 @@ class CloudDataCatalogGetTagTemplateOperator(GoogleCloudBaseOperator):
         *,
         location: str,
         tag_template: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1403,7 +1404,7 @@ class CloudDataCatalogListTagsOperator(GoogleCloudBaseOperator):
         entry_group: str,
         entry: str,
         page_size: int = 100,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1497,7 +1498,7 @@ class CloudDataCatalogLookupEntryOperator(GoogleCloudBaseOperator):
         *,
         linked_resource: str | None = None,
         sql_resource: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1593,7 +1594,7 @@ class CloudDataCatalogRenameTagTemplateFieldOperator(GoogleCloudBaseOperator):
         tag_template: str,
         field: str,
         new_tag_template_field_id: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1808,7 +1809,7 @@ class CloudDataCatalogUpdateEntryOperator(GoogleCloudBaseOperator):
         location: str | None = None,
         entry_group: str | None = None,
         entry_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1920,7 +1921,7 @@ class CloudDataCatalogUpdateTagOperator(GoogleCloudBaseOperator):
         entry_group: str | None = None,
         entry: str | None = None,
         tag_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -2034,7 +2035,7 @@ class CloudDataCatalogUpdateTagTemplateOperator(GoogleCloudBaseOperator):
         update_mask: dict | FieldMask,
         location: str | None = None,
         tag_template_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -2151,7 +2152,7 @@ class CloudDataCatalogUpdateTagTemplateFieldOperator(GoogleCloudBaseOperator):
         location: str | None = None,
         tag_template: str | None = None,
         tag_template_field_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),

airflow/providers/google/cloud/operators/dataflow.py

@@ -41,6 +41,8 @@ from airflow.providers.google.cloud.hooks.gcs import GCSHook
 from airflow.providers.google.cloud.links.dataflow import DataflowJobLink
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
 from airflow.providers.google.cloud.triggers.dataflow import TemplateJobStartTrigger
+from airflow.providers.google.common.consts import GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
 from airflow.version import version
 
 if TYPE_CHECKING:
@@ -142,7 +144,7 @@ class DataflowConfiguration:
         *,
         job_name: str = "{{task.task_id}}",
         append_job_name: bool = True,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         location: str | None = DEFAULT_DATAFLOW_LOCATION,
         gcp_conn_id: str = "google_cloud_default",
         poll_sleep: int = 10,
@@ -348,7 +350,7 @@ class DataflowCreateJavaJobOperator(GoogleCloudBaseOperator):
         job_name: str = "{{task.task_id}}",
         dataflow_default_options: dict | None = None,
         options: dict | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         location: str = DEFAULT_DATAFLOW_LOCATION,
         gcp_conn_id: str = "google_cloud_default",
         poll_sleep: int = 10,
@@ -459,7 +461,7 @@ class DataflowCreateJavaJobOperator(GoogleCloudBaseOperator):
 
 class DataflowTemplatedJobStartOperator(GoogleCloudBaseOperator):
     """
-    Start a Templated Cloud Dataflow job; the parameters of the operation will be passed to the job.
+    Start a Dataflow job with a classic template; the parameters of the operation will be passed to the job.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -606,7 +608,7 @@ class DataflowTemplatedJobStartOperator(GoogleCloudBaseOperator):
         self,
         *,
         template: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         job_name: str = "{{task.task_id}}",
         options: dict[str, Any] | None = None,
         dataflow_default_options: dict[str, Any] | None = None,
@@ -642,7 +644,7 @@ class DataflowTemplatedJobStartOperator(GoogleCloudBaseOperator):
         self.deferrable = deferrable
         self.expected_terminal_state = expected_terminal_state
 
-        self.job: dict | None = None
+        self.job: dict[str, str] | None = None
 
         self._validate_deferrable_params()
 
@@ -680,29 +682,34 @@ class DataflowTemplatedJobStartOperator(GoogleCloudBaseOperator):
         if not self.location:
             self.location = DEFAULT_DATAFLOW_LOCATION
 
-        self.job = self.hook.start_template_dataflow(
+        if not self.deferrable:
+            self.job = self.hook.start_template_dataflow(
+                job_name=self.job_name,
+                variables=options,
+                parameters=self.parameters,
+                dataflow_template=self.template,
+                on_new_job_callback=set_current_job,
+                project_id=self.project_id,
+                location=self.location,
+                environment=self.environment,
+                append_job_name=self.append_job_name,
+            )
+            job_id = self.hook.extract_job_id(self.job)
+            self.xcom_push(context, key="job_id", value=job_id)
+            return job_id
+
+        self.job = self.hook.launch_job_with_template(
             job_name=self.job_name,
             variables=options,
             parameters=self.parameters,
             dataflow_template=self.template,
-            on_new_job_callback=set_current_job,
             project_id=self.project_id,
+            append_job_name=self.append_job_name,
             location=self.location,
             environment=self.environment,
-            append_job_name=self.append_job_name,
         )
-        job_id = self.job.get("id")
-
-        if job_id is None:
-            raise AirflowException(
-                "While reading job object after template execution error occurred. Job object has no id."
-            )
-
-        if not self.deferrable:
-            return job_id
-
-        context["ti"].xcom_push(key="job_id", value=job_id)
-
+        job_id = self.hook.extract_job_id(self.job)
+        DataflowJobLink.persist(self, context, self.project_id, self.location, job_id)
         self.defer(
             trigger=TemplateJobStartTrigger(
                 project_id=self.project_id,
@@ -713,16 +720,17 @@ class DataflowTemplatedJobStartOperator(GoogleCloudBaseOperator):
                 impersonation_chain=self.impersonation_chain,
                 cancel_timeout=self.cancel_timeout,
             ),
-            method_name="execute_complete",
+            method_name=GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME,
         )
 
-    def execute_complete(self, context: Context, event: dict[str, Any]):
+    def execute_complete(self, context: Context, event: dict[str, Any]) -> str:
         """Execute after trigger finishes its work."""
         if event["status"] in ("error", "stopped"):
             self.log.info("status: %s, msg: %s", event["status"], event["message"])
             raise AirflowException(event["message"])
 
         job_id = event["job_id"]
+        self.xcom_push(context, key="job_id", value=job_id)
         self.log.info("Task %s completed with response %s", self.task_id, event["message"])
         return job_id
 
@@ -740,7 +748,7 @@ class DataflowTemplatedJobStartOperator(GoogleCloudBaseOperator):
 
 class DataflowStartFlexTemplateOperator(GoogleCloudBaseOperator):
     """
-    Starts flex templates with the Dataflow pipeline.
+    Starts a Dataflow Job with a Flex Template.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -802,6 +810,9 @@ class DataflowStartFlexTemplateOperator(GoogleCloudBaseOperator):
     :param expected_terminal_state: The expected final status of the operator on which the corresponding
         Airflow task succeeds. When not specified, it will be determined by the hook.
     :param append_job_name: True if unique suffix has to be appended to job name.
+    :param poll_sleep: The time in seconds to sleep between polling Google
+        Cloud Platform for the dataflow job status while the job is in the
+        JOB_STATE_RUNNING state.
     """
 
     template_fields: Sequence[str] = ("body", "location", "project_id", "gcp_conn_id")
@@ -811,7 +822,7 @@ class DataflowStartFlexTemplateOperator(GoogleCloudBaseOperator):
         self,
         body: dict,
         location: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         drain_pipeline: bool = False,
         cancel_timeout: int | None = 10 * 60,
@@ -820,6 +831,7 @@ class DataflowStartFlexTemplateOperator(GoogleCloudBaseOperator):
         deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         append_job_name: bool = True,
         expected_terminal_state: str | None = None,
+        poll_sleep: int = 10,
         *args,
         **kwargs,
     ) -> None:
@@ -831,11 +843,12 @@ class DataflowStartFlexTemplateOperator(GoogleCloudBaseOperator):
         self.drain_pipeline = drain_pipeline
         self.cancel_timeout = cancel_timeout
         self.wait_until_finished = wait_until_finished
-        self.job: dict | None = None
+        self.job: dict[str, str] | None = None
         self.impersonation_chain = impersonation_chain
         self.deferrable = deferrable
         self.expected_terminal_state = expected_terminal_state
         self.append_job_name = append_job_name
+        self.poll_sleep = poll_sleep
 
         self._validate_deferrable_params()
 
@@ -870,32 +883,35 @@ class DataflowStartFlexTemplateOperator(GoogleCloudBaseOperator):
             self.job = current_job
             DataflowJobLink.persist(self, context, self.project_id, self.location, self.job.get("id"))
 
-        self.job = self.hook.start_flex_template(
+        if not self.deferrable:
+            self.job = self.hook.start_flex_template(
+                body=self.body,
+                location=self.location,
+                project_id=self.project_id,
+                on_new_job_callback=set_current_job,
+            )
+            job_id = self.hook.extract_job_id(self.job)
+            self.xcom_push(context, key="job_id", value=job_id)
+            return self.job
+
+        self.job = self.hook.launch_job_with_flex_template(
             body=self.body,
             location=self.location,
             project_id=self.project_id,
-            on_new_job_callback=set_current_job,
         )
-
-        job_id = self.job.get("id")
-        if job_id is None:
-            raise AirflowException(
-                "While reading job object after template execution error occurred. Job object has no id."
-            )
-
-        if not self.deferrable:
-            return self.job
-
+        job_id = self.hook.extract_job_id(self.job)
+        DataflowJobLink.persist(self, context, self.project_id, self.location, job_id)
         self.defer(
             trigger=TemplateJobStartTrigger(
                 project_id=self.project_id,
                 job_id=job_id,
                 location=self.location,
                 gcp_conn_id=self.gcp_conn_id,
+                poll_sleep=self.poll_sleep,
                 impersonation_chain=self.impersonation_chain,
                 cancel_timeout=self.cancel_timeout,
             ),
-            method_name="execute_complete",
+            method_name=GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME,
         )
 
     def _append_uuid_to_job_name(self):
@@ -906,7 +922,7 @@ class DataflowStartFlexTemplateOperator(GoogleCloudBaseOperator):
             job_body["jobName"] = job_name
             self.log.info("Job name was changed to %s", job_name)
 
-    def execute_complete(self, context: Context, event: dict):
+    def execute_complete(self, context: Context, event: dict) -> dict[str, str]:
         """Execute after trigger finishes its work."""
         if event["status"] in ("error", "stopped"):
             self.log.info("status: %s, msg: %s", event["status"], event["message"])
@@ -914,6 +930,7 @@ class DataflowStartFlexTemplateOperator(GoogleCloudBaseOperator):
 
         job_id = event["job_id"]
         self.log.info("Task %s completed with response %s", job_id, event["message"])
+        self.xcom_push(context, key="job_id", value=job_id)
         job = self.hook.get_job(job_id=job_id, project_id=self.project_id, location=self.location)
         return job
 
@@ -982,7 +999,7 @@ class DataflowStartSqlJobOperator(GoogleCloudBaseOperator):
         query: str,
         options: dict[str, Any],
         location: str = DEFAULT_DATAFLOW_LOCATION,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         drain_pipeline: bool = False,
         impersonation_chain: str | Sequence[str] | None = None,
@@ -1150,7 +1167,7 @@ class DataflowCreatePythonJobOperator(GoogleCloudBaseOperator):
         py_options: list[str] | None = None,
         py_requirements: list[str] | None = None,
         py_system_site_packages: bool = False,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         location: str = DEFAULT_DATAFLOW_LOCATION,
         gcp_conn_id: str = "google_cloud_default",
         poll_sleep: int = 10,
@@ -1297,7 +1314,7 @@ class DataflowStopJobOperator(GoogleCloudBaseOperator):
         self,
         job_name_prefix: str | None = None,
         job_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         location: str = DEFAULT_DATAFLOW_LOCATION,
         gcp_conn_id: str = "google_cloud_default",
         poll_sleep: int = 10,

airflow/providers/google/cloud/operators/datafusion.py

@@ -37,6 +37,7 @@ from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseO
 from airflow.providers.google.cloud.triggers.datafusion import DataFusionStartPipelineTrigger
 from airflow.providers.google.cloud.utils.datafusion import DataFusionPipelineType
 from airflow.providers.google.cloud.utils.helpers import resource_path_to_dict
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
 
 if TYPE_CHECKING:
     from airflow.utils.context import Context
@@ -97,7 +98,7 @@ class CloudDataFusionRestartInstanceOperator(GoogleCloudBaseOperator):
         *,
         instance_name: str,
         location: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         api_version: str = "v1beta1",
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -169,7 +170,7 @@ class CloudDataFusionDeleteInstanceOperator(GoogleCloudBaseOperator):
         *,
         instance_name: str,
         location: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         api_version: str = "v1beta1",
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -237,7 +238,7 @@ class CloudDataFusionCreateInstanceOperator(GoogleCloudBaseOperator):
         instance_name: str,
         instance: dict[str, Any],
         location: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         api_version: str = "v1beta1",
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -340,7 +341,7 @@ class CloudDataFusionUpdateInstanceOperator(GoogleCloudBaseOperator):
         instance: dict[str, Any],
         update_mask: str,
         location: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         api_version: str = "v1beta1",
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -417,7 +418,7 @@ class CloudDataFusionGetInstanceOperator(GoogleCloudBaseOperator):
         *,
         instance_name: str,
         location: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         api_version: str = "v1beta1",
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -499,7 +500,7 @@ class CloudDataFusionCreatePipelineOperator(GoogleCloudBaseOperator):
         instance_name: str,
         location: str,
         namespace: str = "default",
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         api_version: str = "v1beta1",
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -587,7 +588,7 @@ class CloudDataFusionDeletePipelineOperator(GoogleCloudBaseOperator):
         location: str,
         version_id: str | None = None,
         namespace: str = "default",
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         api_version: str = "v1beta1",
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -670,7 +671,7 @@ class CloudDataFusionListPipelinesOperator(GoogleCloudBaseOperator):
         artifact_name: str | None = None,
         artifact_version: str | None = None,
         namespace: str = "default",
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         api_version: str = "v1beta1",
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -777,7 +778,7 @@ class CloudDataFusionStartPipelineOperator(GoogleCloudBaseOperator):
         success_states: list[str] | None = None,
         namespace: str = "default",
         pipeline_timeout: int = 5 * 60,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         api_version: str = "v1beta1",
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -929,7 +930,7 @@ class CloudDataFusionStopPipelineOperator(GoogleCloudBaseOperator):
         instance_name: str,
         location: str,
         namespace: str = "default",
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         api_version: str = "v1beta1",
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,

airflow/providers/google/cloud/operators/datapipeline.py

@@ -24,6 +24,7 @@ from typing import TYPE_CHECKING, Sequence
 from airflow.exceptions import AirflowException
 from airflow.providers.google.cloud.hooks.datapipeline import DEFAULT_DATAPIPELINE_LOCATION, DataPipelineHook
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
 
 if TYPE_CHECKING:
     from airflow.utils.context import Context
@@ -58,7 +59,7 @@ class CreateDataPipelineOperator(GoogleCloudBaseOperator):
         self,
         *,
         body: dict,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         location: str = DEFAULT_DATAPIPELINE_LOCATION,
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -120,7 +121,7 @@ class RunDataPipelineOperator(GoogleCloudBaseOperator):
     def __init__(
         self,
         data_pipeline_name: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         location: str = DEFAULT_DATAPIPELINE_LOCATION,
         gcp_conn_id: str = "google_cloud_default",
         **kwargs,