apache-airflow-providers-google 10.2.0rc1__py3-none-any.whl → 10.3.0rc1__py3-none-any.whl

airflow/providers/google/cloud/hooks/functions.py

@@ -32,8 +32,7 @@ TIME_TO_SLEEP_IN_SECONDS = 1
 
 
 class CloudFunctionsHook(GoogleBaseHook):
-    """
-    Hook for the Google Cloud Functions APIs.
+    """Google Cloud Functions APIs.
 
     All the methods in the hook where project_id is used must be called with
     keyword arguments rather than positional.
@@ -61,19 +60,17 @@ class CloudFunctionsHook(GoogleBaseHook):
 
     @staticmethod
     def _full_location(project_id: str, location: str) -> str:
-        """
-        Retrieve full location of the function in the form of
-        ``projects/<GCP_PROJECT_ID>/locations/<GCP_LOCATION>``.
+        """Retrieve full location of the function.
 
-        :param project_id: The Google Cloud Project project_id where the function belongs.
+        :param project_id: Google Cloud Project ID where the function belongs.
         :param location: The location where the function is created.
-        :return:
+        :return: The full location, in the form of
+            ``projects/<GCP_PROJECT_ID>/locations/<GCP_LOCATION>``.
         """
         return f"projects/{project_id}/locations/{location}"
 
     def get_conn(self) -> build:
-        """
-        Retrieves the connection to Cloud Functions.
+        """Retrieve the connection to Cloud Functions.
 
         :return: Google Cloud Function services object.
         """
@@ -85,74 +82,72 @@ class CloudFunctionsHook(GoogleBaseHook):
         return self._conn
 
     def get_function(self, name: str) -> dict:
-        """
-        Returns the Cloud Function with the given name.
+        """Get the Cloud Function with given name.
 
         :param name: Name of the function.
         :return: A Cloud Functions object representing the function.
         """
-        # fmt: off
-        return self.get_conn().projects().locations().functions().get(
-            name=name).execute(num_retries=self.num_retries)
-        # fmt: on
+        operation = self.get_conn().projects().locations().functions().get(name=name)
+        return operation.execute(num_retries=self.num_retries)
 
     @GoogleBaseHook.fallback_to_default_project_id
     def create_new_function(self, location: str, body: dict, project_id: str) -> None:
-        """
-        Creates a new function in Cloud Function in the location specified in the body.
+        """Create a new function at the location specified in the body.
 
         :param location: The location of the function.
         :param body: The body required by the Cloud Functions insert API.
-        :param project_id: Optional, Google Cloud Project project_id where the function belongs.
-            If set to None or missing, the default project_id from the Google Cloud connection is used.
-        :return: None
-        """
-        # fmt: off
-        response = self.get_conn().projects().locations().functions().create(
-            location=self._full_location(project_id, location),
-            body=body
-        ).execute(num_retries=self.num_retries)
-        # fmt: on
+        :param project_id: Google Cloud Project ID where the function belongs.
+            If set to None or missing, the default project ID from the Google
+            Cloud connection is used.
+        """
+        operation = (
+            self.get_conn()
+            .projects()
+            .locations()
+            .functions()
+            .create(location=self._full_location(project_id, location), body=body)
+        )
+        response = operation.execute(num_retries=self.num_retries)
         operation_name = response["name"]
         self._wait_for_operation_to_complete(operation_name=operation_name)
 
     def update_function(self, name: str, body: dict, update_mask: list[str]) -> None:
-        """
-        Updates Cloud Functions according to the specified update mask.
+        """Update Cloud Functions according to the specified update mask.
 
         :param name: The name of the function.
         :param body: The body required by the cloud function patch API.
         :param update_mask: The update mask - array of fields that should be patched.
-        :return: None
         """
-        # fmt: off
-        response = self.get_conn().projects().locations().functions().patch(
-            updateMask=",".join(update_mask),
-            name=name,
-            body=body
-        ).execute(num_retries=self.num_retries)
-        # fmt: on
+        operation = (
+            self.get_conn()
+            .projects()
+            .locations()
+            .functions()
+            .patch(updateMask=",".join(update_mask), name=name, body=body)
+        )
+        response = operation.execute(num_retries=self.num_retries)
         operation_name = response["name"]
         self._wait_for_operation_to_complete(operation_name=operation_name)
 
     @GoogleBaseHook.fallback_to_default_project_id
     def upload_function_zip(self, location: str, zip_path: str, project_id: str) -> str:
-        """
-        Uploads zip file with sources.
+        """Upload ZIP file with sources.
 
         :param location: The location where the function is created.
         :param zip_path: The path of the valid .zip file to upload.
-        :param project_id: Optional, Google Cloud Project project_id where the function belongs.
-            If set to None or missing, the default project_id from the Google Cloud connection is used.
+        :param project_id: Google Cloud Project ID where the function belongs.
+            If set to None or missing, the default project ID from the Google
+            Cloud connection is used.
         :return: The upload URL that was returned by generateUploadUrl method.
         """
-        # fmt: off
-
-        response = \
-            self.get_conn().projects().locations().functions().generateUploadUrl(
-                parent=self._full_location(project_id, location)
-            ).execute(num_retries=self.num_retries)
-        # fmt: on
+        operation = (
+            self.get_conn()
+            .projects()
+            .locations()
+            .functions()
+            .generateUploadUrl(parent=self._full_location(project_id, location))
+        )
+        response = operation.execute(num_retries=self.num_retries)
 
         upload_url = response.get("uploadUrl")
         with open(zip_path, "rb") as file:
@@ -161,7 +156,6 @@ class CloudFunctionsHook(GoogleBaseHook):
                 data=file,
                 # Those two headers needs to be specified according to:
                 # https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions/generateUploadUrl
-                # nopep8
                 headers={
                     "Content-type": "application/zip",
                     "x-goog-content-length-range": "0,104857600",
@@ -170,16 +164,12 @@ class CloudFunctionsHook(GoogleBaseHook):
         return upload_url
 
     def delete_function(self, name: str) -> None:
-        """
-        Deletes the specified Cloud Function.
+        """Delete the specified Cloud Function.
 
         :param name: The name of the function.
-        :return: None
         """
-        # fmt: off
-        response = self.get_conn().projects().locations().functions().delete(
-            name=name).execute(num_retries=self.num_retries)
-        # fmt: on
+        operation = self.get_conn().projects().locations().functions().delete(name=name)
+        response = operation.execute(num_retries=self.num_retries)
         operation_name = response["name"]
         self._wait_for_operation_to_complete(operation_name=operation_name)
 
@@ -191,32 +181,29 @@ class CloudFunctionsHook(GoogleBaseHook):
         location: str,
         project_id: str = PROVIDE_PROJECT_ID,
     ) -> dict:
-        """
-        Synchronously invokes a deployed Cloud Function. To be used for testing
-        purposes as very limited traffic is allowed.
+        """Invoke a deployed Cloud Function.
+
+        This is done synchronously and should only be used for testing purposes,
+        as very limited traffic is allowed.
 
         :param function_id: ID of the function to be called
         :param input_data: Input to be passed to the function
         :param location: The location where the function is located.
-        :param project_id: Optional, Google Cloud Project project_id where the function belongs.
-            If set to None or missing, the default project_id from the Google Cloud connection is used.
-        :return: None
+        :param project_id: Google Cloud Project ID where the function belongs.
+            If set to None or missing, the default project ID from the Google
+            Cloud connection is used.
         """
         name = f"projects/{project_id}/locations/{location}/functions/{function_id}"
-        # fmt: off
-        response = self.get_conn().projects().locations().functions().call(
-            name=name,
-            body=input_data
-        ).execute(num_retries=self.num_retries)
-        # fmt: on
+        operation = self.get_conn().projects().locations().functions().call(name=name, body=input_data)
+        response = operation.execute(num_retries=self.num_retries)
         if "error" in response:
             raise AirflowException(response["error"])
         return response
 
     def _wait_for_operation_to_complete(self, operation_name: str) -> dict:
-        """
-        Waits for the named operation to complete - checks status of the
-        asynchronous call.
+        """Wait for the named operation to complete.
+
+        This is used to check the status of an asynchronous call.
 
         :param operation_name: The name of the operation.
         :return: The response returned by the operation.
@@ -224,11 +211,8 @@ class CloudFunctionsHook(GoogleBaseHook):
         """
         service = self.get_conn()
         while True:
-            # fmt: off
-            operation_response = service.operations().get(
-                name=operation_name,
-            ).execute(num_retries=self.num_retries)
-            # fmt: on
+            operation = service.operations().get(name=operation_name)
+            operation_response = operation.execute(num_retries=self.num_retries)
             if operation_response.get("done"):
                 response = operation_response.get("response")
                 error = operation_response.get("error")

airflow/providers/google/cloud/hooks/gcs.py

@@ -24,6 +24,7 @@ import json
 import os
 import shutil
 import time
+import warnings
 from contextlib import contextmanager
 from datetime import datetime
 from functools import partial
@@ -44,7 +45,7 @@ from google.cloud.exceptions import GoogleCloudError
 from google.cloud.storage.retry import DEFAULT_RETRY
 from requests import Session
 
-from airflow.exceptions import AirflowException
+from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.providers.google.cloud.utils.helpers import normalize_directory_path
 from airflow.providers.google.common.consts import CLIENT_INFO
 from airflow.providers.google.common.hooks.base_google import GoogleBaseAsyncHook, GoogleBaseHook
@@ -709,6 +710,7 @@ class GCSHook(GoogleBaseHook):
         max_results: int | None = None,
         prefix: str | List[str] | None = None,
         delimiter: str | None = None,
+        match_glob: str | None = None,
     ):
         """
         List all objects from the bucket with the given a single prefix or multiple prefixes.
@@ -717,9 +719,19 @@ class GCSHook(GoogleBaseHook):
         :param versions: if true, list all versions of the objects
         :param max_results: max count of items to return in a single page of responses
         :param prefix: string or list of strings which filter objects whose name begin with it/them
-        :param delimiter: filters objects based on the delimiter (for e.g '.csv')
+        :param delimiter: (Deprecated) filters objects based on the delimiter (for e.g '.csv')
+        :param match_glob: (Optional) filters objects based on the glob pattern given by the string
+            (e.g, ``'**/*/.json'``).
         :return: a stream of object names matching the filtering criteria
         """
+        if delimiter and delimiter != "/":
+            warnings.warn(
+                "Usage of 'delimiter' param is deprecated, please use 'match_glob' instead",
+                AirflowProviderDeprecationWarning,
+                stacklevel=2,
+            )
+        if match_glob and delimiter and delimiter != "/":
+            raise AirflowException("'match_glob' param cannot be used with 'delimiter' that differs than '/'")
         objects = []
         if isinstance(prefix, list):
             for prefix_item in prefix:
@@ -730,6 +742,7 @@ class GCSHook(GoogleBaseHook):
                         max_results=max_results,
                         prefix=prefix_item,
                         delimiter=delimiter,
+                        match_glob=match_glob,
                     )
                 )
         else:
@@ -740,6 +753,7 @@ class GCSHook(GoogleBaseHook):
                     max_results=max_results,
                     prefix=prefix,
                     delimiter=delimiter,
+                    match_glob=match_glob,
                 )
             )
         return objects
@@ -751,6 +765,7 @@ class GCSHook(GoogleBaseHook):
         max_results: int | None = None,
         prefix: str | None = None,
         delimiter: str | None = None,
+        match_glob: str | None = None,
     ) -> List:
         """
         List all objects from the bucket with the give string prefix in name.
@@ -759,7 +774,9 @@ class GCSHook(GoogleBaseHook):
         :param versions: if true, list all versions of the objects
         :param max_results: max count of items to return in a single page of responses
         :param prefix: string which filters objects whose name begin with it
-        :param delimiter: filters objects based on the delimiter (for e.g '.csv')
+        :param delimiter: (Deprecated) filters objects based on the delimiter (for e.g '.csv')
+        :param match_glob: (Optional) filters objects based on the glob pattern given by the string
+            (e.g, ``'**/*/.json'``).
         :return: a stream of object names matching the filtering criteria
         """
         client = self.get_conn()
@@ -768,13 +785,25 @@ class GCSHook(GoogleBaseHook):
         ids = []
         page_token = None
         while True:
-            blobs = bucket.list_blobs(
-                max_results=max_results,
-                page_token=page_token,
-                prefix=prefix,
-                delimiter=delimiter,
-                versions=versions,
-            )
+            if match_glob:
+                blobs = self._list_blobs_with_match_glob(
+                    bucket=bucket,
+                    client=client,
+                    match_glob=match_glob,
+                    max_results=max_results,
+                    page_token=page_token,
+                    path=bucket.path + "/o",
+                    prefix=prefix,
+                    versions=versions,
+                )
+            else:
+                blobs = bucket.list_blobs(
+                    max_results=max_results,
+                    page_token=page_token,
+                    prefix=prefix,
+                    delimiter=delimiter,
+                    versions=versions,
+                )
 
             blob_names = []
             for blob in blobs:
@@ -792,6 +821,52 @@ class GCSHook(GoogleBaseHook):
                 break
         return ids
 
+    @staticmethod
+    def _list_blobs_with_match_glob(
+        bucket,
+        client,
+        path: str,
+        max_results: int | None = None,
+        page_token: str | None = None,
+        match_glob: str | None = None,
+        prefix: str | None = None,
+        versions: bool | None = None,
+    ) -> Any:
+        """
+        List blobs when match_glob param is given.
+        This method is a patched version of google.cloud.storage Client.list_blobs().
+        It is used as a temporary workaround to support "match_glob" param,
+        as it isn't officially supported by GCS Python client.
+        (follow `issue #1035<https://github.com/googleapis/python-storage/issues/1035>`__).
+        """
+        from google.api_core import page_iterator
+        from google.cloud.storage.bucket import _blobs_page_start, _item_to_blob
+
+        extra_params: Any = {}
+        if prefix is not None:
+            extra_params["prefix"] = prefix
+        if match_glob is not None:
+            extra_params["matchGlob"] = match_glob
+        if versions is not None:
+            extra_params["versions"] = versions
+        api_request = functools.partial(
+            client._connection.api_request, timeout=DEFAULT_TIMEOUT, retry=DEFAULT_RETRY
+        )
+
+        blobs: Any = page_iterator.HTTPIterator(
+            client=client,
+            api_request=api_request,
+            path=path,
+            item_to_value=_item_to_blob,
+            page_token=page_token,
+            max_results=max_results,
+            extra_params=extra_params,
+            page_start=_blobs_page_start,
+        )
+        blobs.prefixes = set()
+        blobs.bucket = bucket
+        return blobs
+
     def list_by_timespan(
         self,
         bucket_name: str,
@@ -801,6 +876,7 @@ class GCSHook(GoogleBaseHook):
         max_results: int | None = None,
         prefix: str | None = None,
         delimiter: str | None = None,
+        match_glob: str | None = None,
     ) -> List[str]:
         """
         List all objects from the bucket with the give string prefix in name that were
@@ -813,7 +889,9 @@ class GCSHook(GoogleBaseHook):
         :param max_results: max count of items to return in a single page of responses
         :param prefix: prefix string which filters objects whose name begin with
             this prefix
-        :param delimiter: filters objects based on the delimiter (for e.g '.csv')
+        :param delimiter: (Deprecated) filters objects based on the delimiter (for e.g '.csv')
+        :param match_glob: (Optional) filters objects based on the glob pattern given by the string
+            (e.g, ``'**/*/.json'``).
         :return: a stream of object names matching the filtering criteria
         """
         client = self.get_conn()
@@ -823,13 +901,25 @@ class GCSHook(GoogleBaseHook):
         page_token = None
 
         while True:
-            blobs = bucket.list_blobs(
-                max_results=max_results,
-                page_token=page_token,
-                prefix=prefix,
-                delimiter=delimiter,
-                versions=versions,
-            )
+            if match_glob:
+                blobs = self._list_blobs_with_match_glob(
+                    bucket=bucket,
+                    client=client,
+                    match_glob=match_glob,
+                    max_results=max_results,
+                    page_token=page_token,
+                    path=bucket.path + "/o",
+                    prefix=prefix,
+                    versions=versions,
+                )
+            else:
+                blobs = bucket.list_blobs(
+                    max_results=max_results,
+                    page_token=page_token,
+                    prefix=prefix,
+                    delimiter=delimiter,
+                    versions=versions,
+                )
 
             blob_names = []
             for blob in blobs: