apache-airflow-providers-google 10.14.0rc2__py3-none-any.whl → 10.15.0rc1__py3-none-any.whl

airflow/providers/google/cloud/hooks/workflows.py

@@ -50,11 +50,11 @@ class WorkflowsHook(GoogleBaseHook):
         super().__init__(**kwargs)
 
     def get_workflows_client(self) -> WorkflowsClient:
-        """Returns WorkflowsClient."""
+        """Return WorkflowsClient object."""
         return WorkflowsClient(credentials=self.get_credentials(), client_info=CLIENT_INFO)
 
     def get_executions_client(self) -> ExecutionsClient:
-        """Returns ExecutionsClient."""
+        """Return ExecutionsClient object."""
         return ExecutionsClient(credentials=self.get_credentials(), client_info=CLIENT_INFO)
 
     @GoogleBaseHook.fallback_to_default_project_id
@@ -69,7 +69,7 @@ class WorkflowsHook(GoogleBaseHook):
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Operation:
         """
-        Creates a new workflow.
+        Create a new workflow.
 
         If a workflow with the specified name already exists in the
         specified project and location, the long running operation will
@@ -106,7 +106,7 @@ class WorkflowsHook(GoogleBaseHook):
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Workflow:
         """
-        Gets details of a single Workflow.
+        Get details of a single Workflow.
 
         :param workflow_id: Required. The ID of the workflow to be created.
         :param project_id: Required. The ID of the Google Cloud project the cluster belongs to.
@@ -131,7 +131,7 @@ class WorkflowsHook(GoogleBaseHook):
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Operation:
         """
-        Updates an existing workflow.
+        Update an existing workflow.
 
         Running this method has no impact on already running
         executions of the workflow. A new revision of the
@@ -196,7 +196,7 @@ class WorkflowsHook(GoogleBaseHook):
         metadata: Sequence[tuple[str, str]] = (),
     ) -> ListWorkflowsPager:
         """
-        Lists Workflows in a given project and location; the default order is not specified.
+        List Workflows in a given project and location; the default order is not specified.
 
         :param filter_: Filter to restrict results to specific workflows.
         :param order_by: Comma-separated list of fields that
@@ -234,7 +234,7 @@ class WorkflowsHook(GoogleBaseHook):
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Execution:
         """
-        Creates a new execution using the latest revision of the given workflow.
+        Create a new execution using the latest revision of the given workflow.
 
         :param execution: Required. Input parameters of the execution represented as a dictionary.
         :param workflow_id: Required. The ID of the workflow.
@@ -269,7 +269,7 @@ class WorkflowsHook(GoogleBaseHook):
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Execution:
         """
-        Returns an execution for the given ``workflow_id`` and ``execution_id``.
+        Return an execution for the given ``workflow_id`` and ``execution_id``.
 
         :param workflow_id: Required. The ID of the workflow.
         :param execution_id: Required. The ID of the execution.
@@ -298,7 +298,7 @@ class WorkflowsHook(GoogleBaseHook):
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Execution:
         """
-        Cancels an execution using the given ``workflow_id`` and ``execution_id``.
+        Cancel an execution using the given ``workflow_id`` and ``execution_id``.
 
         :param workflow_id: Required. The ID of the workflow.
         :param execution_id: Required. The ID of the execution.
@@ -328,7 +328,7 @@ class WorkflowsHook(GoogleBaseHook):
         metadata: Sequence[tuple[str, str]] = (),
     ) -> ListExecutionsPager:
         """
-        Returns a list of executions which belong to the workflow with the given name.
+        Return a list of executions which belong to the workflow with the given name.
 
         The method returns executions of all workflow revisions. Returned
         executions are ordered by their start time (newest first).

airflow/providers/google/cloud/links/datafusion.py

@@ -15,7 +15,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-"""This module contains Google Compute Engine links."""
+"""This module contains Google Data Fusion links."""
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, ClassVar
@@ -30,8 +30,8 @@ if TYPE_CHECKING:
 
 BASE_LINK = "https://console.cloud.google.com/data-fusion"
 DATAFUSION_INSTANCE_LINK = BASE_LINK + "/locations/{region}/instances/{instance_name}?project={project_id}"
-DATAFUSION_PIPELINES_LINK = "{uri}/cdap/ns/default/pipelines"
-DATAFUSION_PIPELINE_LINK = "{uri}/pipelines/ns/default/view/{pipeline_name}"
+DATAFUSION_PIPELINES_LINK = "{uri}/cdap/ns/{namespace}/pipelines"
+DATAFUSION_PIPELINE_LINK = "{uri}/pipelines/ns/{namespace}/view/{pipeline_name}"
 
 
 class BaseGoogleLink(BaseOperatorLink):
@@ -52,10 +52,13 @@ class BaseGoogleLink(BaseOperatorLink):
         ti_key: TaskInstanceKey,
     ) -> str:
         conf = XCom.get_value(key=self.key, ti_key=ti_key)
+
         if not conf:
             return ""
-        if self.format_str.startswith("http"):
-            return self.format_str.format(**conf)
+
+        # Add a default value for the 'namespace' parameter for backward compatibility.
+        conf.setdefault("namespace", "default")
+
         return self.format_str.format(**conf)
 
 
@@ -98,6 +101,7 @@ class DataFusionPipelineLink(BaseGoogleLink):
         task_instance: BaseOperator,
         uri: str,
         pipeline_name: str,
+        namespace: str,
     ):
         task_instance.xcom_push(
             context=context,
@@ -105,6 +109,7 @@ class DataFusionPipelineLink(BaseGoogleLink):
             value={
                 "uri": uri,
                 "pipeline_name": pipeline_name,
+                "namespace": namespace,
             },
         )
 
@@ -121,11 +126,13 @@ class DataFusionPipelinesLink(BaseGoogleLink):
         context: Context,
         task_instance: BaseOperator,
         uri: str,
+        namespace: str,
     ):
         task_instance.xcom_push(
             context=context,
             key=DataFusionPipelinesLink.key,
             value={
                 "uri": uri,
+                "namespace": namespace,
             },
         )

airflow/providers/google/cloud/operators/bigquery.py

@@ -25,6 +25,7 @@ from functools import cached_property
 from typing import TYPE_CHECKING, Any, Iterable, Sequence, SupportsAbs
 
 import attr
+from deprecated import deprecated
 from google.api_core.exceptions import Conflict
 from google.cloud.bigquery import DEFAULT_RETRY, CopyJob, ExtractJob, LoadJob, QueryJob
 from google.cloud.bigquery.table import RowIterator
@@ -1087,6 +1088,10 @@ class BigQueryGetDataOperator(GoogleCloudBaseOperator):
         return event["records"]
 
 
+@deprecated(
+    reason="This operator is deprecated. Please use `BigQueryInsertJobOperator`.",
+    category=AirflowProviderDeprecationWarning,
+)
 class BigQueryExecuteQueryOperator(GoogleCloudBaseOperator):
     """Executes BigQuery SQL queries in a specific BigQuery database.
 
@@ -1211,12 +1216,6 @@ class BigQueryExecuteQueryOperator(GoogleCloudBaseOperator):
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
-        warnings.warn(
-            "This operator is deprecated. Please use `BigQueryInsertJobOperator`.",
-            AirflowProviderDeprecationWarning,
-            stacklevel=2,
-        )
-
         self.sql = sql
         self.destination_dataset_table = destination_dataset_table
         self.write_disposition = write_disposition
@@ -2171,6 +2170,10 @@ class BigQueryGetDatasetTablesOperator(GoogleCloudBaseOperator):
         )
 
 
+@deprecated(
+    reason="This operator is deprecated. Please use BigQueryUpdateDatasetOperator.",
+    category=AirflowProviderDeprecationWarning,
+)
 class BigQueryPatchDatasetOperator(GoogleCloudBaseOperator):
     """Patch a dataset for your Project in BigQuery.
 
@@ -2215,11 +2218,6 @@ class BigQueryPatchDatasetOperator(GoogleCloudBaseOperator):
         impersonation_chain: str | Sequence[str] | None = None,
         **kwargs,
     ) -> None:
-        warnings.warn(
-            "This operator is deprecated. Please use BigQueryUpdateDatasetOperator.",
-            AirflowProviderDeprecationWarning,
-            stacklevel=2,
-        )
         self.dataset_id = dataset_id
         self.project_id = project_id
         self.gcp_conn_id = gcp_conn_id

airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py

@@ -236,7 +236,9 @@ class CloudDataTransferServiceCreateJobOperator(GoogleCloudBaseOperator):
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
-        self.body = deepcopy(body)
+        self.body = body
+        if isinstance(self.body, dict):
+            self.body = deepcopy(body)
         self.aws_conn_id = aws_conn_id
         self.gcp_conn_id = gcp_conn_id
         self.api_version = api_version

airflow/providers/google/cloud/operators/dataflow.py

@@ -21,12 +21,13 @@ from __future__ import annotations
 import copy
 import re
 import uuid
-import warnings
 from contextlib import ExitStack
 from enum import Enum
 from functools import cached_property
 from typing import TYPE_CHECKING, Any, Sequence
 
+from deprecated import deprecated
+
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.providers.apache.beam.hooks.beam import BeamHook, BeamRunnerType
@@ -167,6 +168,11 @@ class DataflowConfiguration:
         self.service_account = service_account
 
 
+# TODO: Remove one day
+@deprecated(
+    reason="Please use `providers.apache.beam.operators.beam.BeamRunJavaPipelineOperator` instead.",
+    category=AirflowProviderDeprecationWarning,
+)
 class DataflowCreateJavaJobOperator(GoogleCloudBaseOperator):
     """
     Start a Java Cloud Dataflow batch job; the parameters of the operation will be passed to the job.
@@ -353,13 +359,6 @@ class DataflowCreateJavaJobOperator(GoogleCloudBaseOperator):
         expected_terminal_state: str | None = None,
         **kwargs,
     ) -> None:
-        # TODO: Remove one day
-        warnings.warn(
-            f"The `{self.__class__.__name__}` operator is deprecated, "
-            f"please use `providers.apache.beam.operators.beam.BeamRunJavaPipelineOperator` instead.",
-            AirflowProviderDeprecationWarning,
-            stacklevel=2,
-        )
         super().__init__(**kwargs)
 
         dataflow_default_options = dataflow_default_options or {}
@@ -677,6 +676,9 @@ class DataflowTemplatedJobStartOperator(GoogleCloudBaseOperator):
         options = self.dataflow_default_options
         options.update(self.options)
 
+        if not self.location:
+            self.location = DEFAULT_DATAFLOW_LOCATION
+
         self.job = self.hook.start_template_dataflow(
             job_name=self.job_name,
             variables=options,
@@ -704,7 +706,7 @@ class DataflowTemplatedJobStartOperator(GoogleCloudBaseOperator):
             trigger=TemplateJobStartTrigger(
                 project_id=self.project_id,
                 job_id=job_id,
-                location=self.location if self.location else DEFAULT_DATAFLOW_LOCATION,
+                location=self.location,
                 gcp_conn_id=self.gcp_conn_id,
                 poll_sleep=self.poll_sleep,
                 impersonation_chain=self.impersonation_chain,
@@ -1029,6 +1031,11 @@ class DataflowStartSqlJobOperator(GoogleCloudBaseOperator):
             )
 
 
+# TODO: Remove one day
+@deprecated(
+    reason="Please use `providers.apache.beam.operators.beam.BeamRunPythonPipelineOperator` instead.",
+    category=AirflowProviderDeprecationWarning,
+)
 class DataflowCreatePythonJobOperator(GoogleCloudBaseOperator):
     """
     Launching Cloud Dataflow jobs written in python.
@@ -1151,13 +1158,6 @@ class DataflowCreatePythonJobOperator(GoogleCloudBaseOperator):
         wait_until_finished: bool | None = None,
         **kwargs,
     ) -> None:
-        # TODO: Remove one day
-        warnings.warn(
-            f"The `{self.__class__.__name__}` operator is deprecated, "
-            "please use `providers.apache.beam.operators.beam.BeamRunPythonPipelineOperator` instead.",
-            AirflowProviderDeprecationWarning,
-            stacklevel=2,
-        )
         super().__init__(**kwargs)
 
         self.py_file = py_file

airflow/providers/google/cloud/operators/datafusion.py

@@ -537,6 +537,7 @@ class CloudDataFusionCreatePipelineOperator(GoogleCloudBaseOperator):
             task_instance=self,
             uri=instance["serviceEndpoint"],
             pipeline_name=self.pipeline_name,
+            namespace=self.namespace,
         )
         self.log.info("Pipeline %s created", self.pipeline_name)
 
@@ -705,7 +706,12 @@ class CloudDataFusionListPipelinesOperator(GoogleCloudBaseOperator):
         )
         self.log.info("Pipelines: %s", pipelines)
 
-        DataFusionPipelinesLink.persist(context=context, task_instance=self, uri=service_endpoint)
+        DataFusionPipelinesLink.persist(
+            context=context,
+            task_instance=self,
+            uri=service_endpoint,
+            namespace=self.namespace,
+        )
         return pipelines
 
 
@@ -825,6 +831,7 @@ class CloudDataFusionStartPipelineOperator(GoogleCloudBaseOperator):
             task_instance=self,
             uri=instance["serviceEndpoint"],
             pipeline_name=self.pipeline_name,
+            namespace=self.namespace,
         )
 
         if self.deferrable:
@@ -954,6 +961,7 @@ class CloudDataFusionStopPipelineOperator(GoogleCloudBaseOperator):
             task_instance=self,
             uri=instance["serviceEndpoint"],
             pipeline_name=self.pipeline_name,
+            namespace=self.namespace,
         )
         hook.stop_pipeline(
             pipeline_name=self.pipeline_name,