apache-airflow-providers-google 10.20.0rc1__py3-none-any.whl → 10.21.0__py3-none-any.whl

airflow/providers/google/cloud/hooks/dataproc.py

@@ -84,7 +84,8 @@ class DataProcJobBuilder:
             self.job["job"][job_type]["properties"] = properties
 
     def add_labels(self, labels: dict | None = None) -> None:
-        """Set labels for Dataproc job.
+        """
+        Set labels for Dataproc job.
 
         :param labels: Labels for the job query.
         """
@@ -92,7 +93,8 @@ class DataProcJobBuilder:
             self.job["job"]["labels"].update(labels)
 
     def add_variables(self, variables: dict | None = None) -> None:
-        """Set variables for Dataproc job.
+        """
+        Set variables for Dataproc job.
 
         :param variables: Variables for the job query.
         """
@@ -100,7 +102,8 @@ class DataProcJobBuilder:
             self.job["job"][self.job_type]["script_variables"] = variables
 
     def add_args(self, args: list[str] | None = None) -> None:
-        """Set args for Dataproc job.
+        """
+        Set args for Dataproc job.
 
         :param args: Args for the job query.
         """
@@ -108,21 +111,24 @@ class DataProcJobBuilder:
             self.job["job"][self.job_type]["args"] = args
 
     def add_query(self, query: str) -> None:
-        """Set query for Dataproc job.
+        """
+        Set query for Dataproc job.
 
         :param query: query for the job.
         """
         self.job["job"][self.job_type]["query_list"] = {"queries": [query]}
 
     def add_query_uri(self, query_uri: str) -> None:
-        """Set query uri for Dataproc job.
+        """
+        Set query uri for Dataproc job.
 
         :param query_uri: URI for the job query.
         """
         self.job["job"][self.job_type]["query_file_uri"] = query_uri
 
     def add_jar_file_uris(self, jars: list[str] | None = None) -> None:
-        """Set jars uris for Dataproc job.
+        """
+        Set jars uris for Dataproc job.
 
         :param jars: List of jars URIs
         """
@@ -130,7 +136,8 @@ class DataProcJobBuilder:
             self.job["job"][self.job_type]["jar_file_uris"] = jars
 
     def add_archive_uris(self, archives: list[str] | None = None) -> None:
-        """Set archives uris for Dataproc job.
+        """
+        Set archives uris for Dataproc job.
 
         :param archives: List of archives URIs
         """
@@ -138,7 +145,8 @@ class DataProcJobBuilder:
             self.job["job"][self.job_type]["archive_uris"] = archives
 
     def add_file_uris(self, files: list[str] | None = None) -> None:
-        """Set file uris for Dataproc job.
+        """
+        Set file uris for Dataproc job.
 
         :param files: List of files URIs
         """
@@ -146,7 +154,8 @@ class DataProcJobBuilder:
             self.job["job"][self.job_type]["file_uris"] = files
 
     def add_python_file_uris(self, pyfiles: list[str] | None = None) -> None:
-        """Set python file uris for Dataproc job.
+        """
+        Set python file uris for Dataproc job.
 
         :param pyfiles: List of python files URIs
         """
@@ -154,7 +163,8 @@ class DataProcJobBuilder:
             self.job["job"][self.job_type]["python_file_uris"] = pyfiles
 
     def set_main(self, main_jar: str | None = None, main_class: str | None = None) -> None:
-        """Set Dataproc main class.
+        """
+        Set Dataproc main class.
 
         :param main_jar: URI for the main file.
         :param main_class: Name of the main class.
@@ -168,14 +178,16 @@ class DataProcJobBuilder:
             self.job["job"][self.job_type]["main_class"] = main_class
 
     def set_python_main(self, main: str) -> None:
-        """Set Dataproc main python file uri.
+        """
+        Set Dataproc main python file uri.
 
         :param main: URI for the python main file.
         """
         self.job["job"][self.job_type]["main_python_file_uri"] = main
 
     def set_job_name(self, name: str) -> None:
-        """Set Dataproc job name.
+        """
+        Set Dataproc job name.
 
         Job name is sanitized, replacing dots by underscores.
 
@@ -185,7 +197,8 @@ class DataProcJobBuilder:
         self.job["job"]["reference"]["job_id"] = sanitized_name
 
     def build(self) -> dict:
-        """Return Dataproc job.
+        """
+        Return Dataproc job.
 
         :return: Dataproc job
         """
@@ -193,7 +206,8 @@ class DataProcJobBuilder:
 
 
 class DataprocHook(GoogleBaseHook):
-    """Google Cloud Dataproc APIs.
+    """
+    Google Cloud Dataproc APIs.
 
     All the methods in the hook where project_id is used must be called with
     keyword arguments rather than positional.
@@ -283,7 +297,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Operation:
-        """Create a cluster in a specified project.
+        """
+        Create a cluster in a specified project.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region in which to handle the request.
@@ -350,7 +365,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Operation:
-        """Delete a cluster in a project.
+        """
+        Delete a cluster in a project.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region in which to handle the request.
@@ -397,7 +413,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Operation:
-        """Get cluster diagnostic information.
+        """
+        Get cluster diagnostic information.
 
         After the operation completes, the response contains the Cloud Storage URI of the diagnostic output report containing a summary of collected diagnostics.
 
@@ -442,7 +459,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Cluster:
-        """Get the resource representation for a cluster in a project.
+        """
+        Get the resource representation for a cluster in a project.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region to handle the request.
@@ -474,7 +492,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ):
-        """List all regions/{region}/clusters in a project.
+        """
+        List all regions/{region}/clusters in a project.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region to handle the request.
@@ -514,7 +533,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Operation:
-        """Update a cluster in a project.
+        """
+        Update a cluster in a project.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region to handle the request.
@@ -596,7 +616,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Operation:
-        """Start a cluster in a project.
+        """
+        Start a cluster in a project.
 
         :param region: Cloud Dataproc region to handle the request.
         :param project_id: Google Cloud project ID that the cluster belongs to.
@@ -640,7 +661,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Operation:
-        """Start a cluster in a project.
+        """
+        Start a cluster in a project.
 
         :param region: Cloud Dataproc region to handle the request.
         :param project_id: Google Cloud project ID that the cluster belongs to.
@@ -682,7 +704,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> WorkflowTemplate:
-        """Create a new workflow template.
+        """
+        Create a new workflow template.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region to handle the request.
@@ -718,7 +741,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Operation:
-        """Instantiate a template and begins execution.
+        """
+        Instantiate a template and begins execution.
 
         :param template_name: Name of template to instantiate.
         :param project_id: Google Cloud project ID that the cluster belongs to.
@@ -764,7 +788,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Operation:
-        """Instantiate a template and begin execution.
+        """
+        Instantiate a template and begin execution.
 
         :param template: The workflow template to instantiate. If a dict is
             provided, it must be of the same form as the protobuf message
@@ -803,7 +828,8 @@ class DataprocHook(GoogleBaseHook):
         wait_time: int = 10,
         timeout: int | None = None,
     ) -> None:
-        """Poll a job to check if it has finished.
+        """
+        Poll a job to check if it has finished.
 
         :param job_id: Dataproc job ID.
         :param project_id: Google Cloud project ID that the cluster belongs to.
@@ -840,7 +866,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Job:
-        """Get the resource representation for a job in a project.
+        """
+        Get the resource representation for a job in a project.
 
         :param job_id: Dataproc job ID.
         :param project_id: Google Cloud project ID that the cluster belongs to.
@@ -874,7 +901,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Job:
-        """Submit a job to a cluster.
+        """
+        Submit a job to a cluster.
 
         :param job: The job resource. If a dict is provided, it must be of the
             same form as the protobuf message Job.
@@ -910,7 +938,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Job:
-        """Start a job cancellation request.
+        """
+        Start a job cancellation request.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region to handle the request.
@@ -944,7 +973,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Operation:
-        """Create a batch workload.
+        """
+        Create a batch workload.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region to handle the request.
@@ -989,7 +1019,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> None:
-        """Delete the batch workload resource.
+        """
+        Delete the batch workload resource.
 
         :param batch_id: The batch ID.
         :param project_id: Google Cloud project ID that the cluster belongs to.
@@ -1023,7 +1054,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Batch:
-        """Get the batch workload resource representation.
+        """
+        Get the batch workload resource representation.
 
         :param batch_id: The batch ID.
         :param project_id: Google Cloud project ID that the cluster belongs to.
@@ -1061,7 +1093,8 @@ class DataprocHook(GoogleBaseHook):
         filter: str | None = None,
         order_by: str | None = None,
     ):
-        """List batch workloads.
+        """
+        List batch workloads.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region to handle the request.
@@ -1107,7 +1140,8 @@ class DataprocHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Batch:
-        """Wait for a batch job to complete.
+        """
+        Wait for a batch job to complete.
 
         After submission of a batch job, the operator waits for the job to
         complete. This hook is, however, useful in the case when Airflow is
@@ -1160,7 +1194,8 @@ class DataprocHook(GoogleBaseHook):
 
 
 class DataprocAsyncHook(GoogleBaseHook):
-    """Asynchronous interaction with Google Cloud Dataproc APIs.
+    """
+    Asynchronous interaction with Google Cloud Dataproc APIs.
 
     All the methods in the hook where project_id is used must be called with
     keyword arguments rather than positional.
@@ -1242,7 +1277,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> AsyncOperation:
-        """Create a cluster in a project.
+        """
+        Create a cluster in a project.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region in which to handle the request.
@@ -1309,7 +1345,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> AsyncOperation:
-        """Delete a cluster in a project.
+        """
+        Delete a cluster in a project.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region in which to handle the request.
@@ -1356,7 +1393,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> AsyncOperation:
-        """Get cluster diagnostic information.
+        """
+        Get cluster diagnostic information.
 
         After the operation completes, the response contains the Cloud Storage URI of the diagnostic output report containing a summary of collected diagnostics.
 
@@ -1401,7 +1439,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Cluster:
-        """Get the resource representation for a cluster in a project.
+        """
+        Get the resource representation for a cluster in a project.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region to handle the request.
@@ -1433,7 +1472,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ):
-        """List all regions/{region}/clusters in a project.
+        """
+        List all regions/{region}/clusters in a project.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region to handle the request.
@@ -1473,7 +1513,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> AsyncOperation:
-        """Update a cluster in a project.
+        """
+        Update a cluster in a project.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region to handle the request.
@@ -1553,7 +1594,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> WorkflowTemplate:
-        """Create a new workflow template.
+        """
+        Create a new workflow template.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region to handle the request.
@@ -1589,7 +1631,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> AsyncOperation:
-        """Instantiate a template and begins execution.
+        """
+        Instantiate a template and begins execution.
 
         :param template_name: Name of template to instantiate.
         :param project_id: Google Cloud project ID that the cluster belongs to.
@@ -1635,7 +1678,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> AsyncOperation:
-        """Instantiate a template and begin execution.
+        """
+        Instantiate a template and begin execution.
 
         :param template: The workflow template to instantiate. If a dict is
             provided, it must be of the same form as the protobuf message
@@ -1678,7 +1722,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Job:
-        """Get the resource representation for a job in a project.
+        """
+        Get the resource representation for a job in a project.
 
         :param job_id: Dataproc job ID.
         :param project_id: Google Cloud project ID that the cluster belongs to.
@@ -1712,7 +1757,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Job:
-        """Submit a job to a cluster.
+        """
+        Submit a job to a cluster.
 
         :param job: The job resource. If a dict is provided, it must be of the
             same form as the protobuf message Job.
@@ -1748,7 +1794,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Job:
-        """Start a job cancellation request.
+        """
+        Start a job cancellation request.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region to handle the request.
@@ -1782,7 +1829,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> AsyncOperation:
-        """Create a batch workload.
+        """
+        Create a batch workload.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region to handle the request.
@@ -1827,7 +1875,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> None:
-        """Delete the batch workload resource.
+        """
+        Delete the batch workload resource.
 
         :param batch_id: The batch ID.
         :param project_id: Google Cloud project ID that the cluster belongs to.
@@ -1861,7 +1910,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
     ) -> Batch:
-        """Get the batch workload resource representation.
+        """
+        Get the batch workload resource representation.
 
         :param batch_id: The batch ID.
         :param project_id: Google Cloud project ID that the cluster belongs to.
@@ -1899,7 +1949,8 @@ class DataprocAsyncHook(GoogleBaseHook):
         filter: str | None = None,
         order_by: str | None = None,
     ):
-        """List batch workloads.
+        """
+        List batch workloads.
 
         :param project_id: Google Cloud project ID that the cluster belongs to.
         :param region: Cloud Dataproc region to handle the request.

airflow/providers/google/cloud/hooks/functions.py

@@ -33,7 +33,8 @@ TIME_TO_SLEEP_IN_SECONDS = 1
 
 
 class CloudFunctionsHook(GoogleBaseHook):
-    """Google Cloud Functions APIs.
+    """
+    Google Cloud Functions APIs.
 
     All the methods in the hook where project_id is used must be called with
     keyword arguments rather than positional.
@@ -61,7 +62,8 @@ class CloudFunctionsHook(GoogleBaseHook):
 
     @staticmethod
     def _full_location(project_id: str, location: str) -> str:
-        """Retrieve full location of the function.
+        """
+        Retrieve full location of the function.
 
         :param project_id: Google Cloud Project ID where the function belongs.
         :param location: The location where the function is created.
@@ -71,7 +73,8 @@ class CloudFunctionsHook(GoogleBaseHook):
         return f"projects/{project_id}/locations/{location}"
 
     def get_conn(self) -> build:
-        """Retrieve the connection to Cloud Functions.
+        """
+        Retrieve the connection to Cloud Functions.
 
         :return: Google Cloud Function services object.
         """
@@ -83,7 +86,8 @@ class CloudFunctionsHook(GoogleBaseHook):
         return self._conn
 
     def get_function(self, name: str) -> dict:
-        """Get the Cloud Function with given name.
+        """
+        Get the Cloud Function with given name.
 
         :param name: Name of the function.
         :return: A Cloud Functions object representing the function.
@@ -93,7 +97,8 @@ class CloudFunctionsHook(GoogleBaseHook):
 
     @GoogleBaseHook.fallback_to_default_project_id
     def create_new_function(self, location: str, body: dict, project_id: str) -> None:
-        """Create a new function at the location specified in the body.
+        """
+        Create a new function at the location specified in the body.
 
         :param location: The location of the function.
         :param body: The body required by the Cloud Functions insert API.
@@ -113,7 +118,8 @@ class CloudFunctionsHook(GoogleBaseHook):
         self._wait_for_operation_to_complete(operation_name=operation_name)
 
     def update_function(self, name: str, body: dict, update_mask: list[str]) -> None:
-        """Update Cloud Functions according to the specified update mask.
+        """
+        Update Cloud Functions according to the specified update mask.
 
         :param name: The name of the function.
         :param body: The body required by the cloud function patch API.
@@ -132,7 +138,8 @@ class CloudFunctionsHook(GoogleBaseHook):
 
     @GoogleBaseHook.fallback_to_default_project_id
     def upload_function_zip(self, location: str, zip_path: str, project_id: str) -> str:
-        """Upload ZIP file with sources.
+        """
+        Upload ZIP file with sources.
 
         :param location: The location where the function is created.
         :param zip_path: The path of the valid .zip file to upload.
@@ -165,7 +172,8 @@ class CloudFunctionsHook(GoogleBaseHook):
         return upload_url
 
     def delete_function(self, name: str) -> None:
-        """Delete the specified Cloud Function.
+        """
+        Delete the specified Cloud Function.
 
         :param name: The name of the function.
         """
@@ -182,7 +190,8 @@ class CloudFunctionsHook(GoogleBaseHook):
         location: str,
         project_id: str = PROVIDE_PROJECT_ID,
     ) -> dict:
-        """Invoke a deployed Cloud Function.
+        """
+        Invoke a deployed Cloud Function.
 
         This is done synchronously and should only be used for testing purposes,
         as very limited traffic is allowed.
@@ -202,7 +211,8 @@ class CloudFunctionsHook(GoogleBaseHook):
         return response
 
     def _wait_for_operation_to_complete(self, operation_name: str) -> dict:
-        """Wait for the named operation to complete.
+        """
+        Wait for the named operation to complete.
 
         This is used to check the status of an asynchronous call.