apache-airflow-providers-google 10.17.0rc1__py3-none-any.whl → 10.18.0rc1__py3-none-any.whl

airflow/providers/google/cloud/hooks/dataplex.py

@@ -36,7 +36,11 @@ from google.protobuf.field_mask_pb2 import FieldMask
 
 from airflow.exceptions import AirflowException
 from airflow.providers.google.common.consts import CLIENT_INFO
-from airflow.providers.google.common.hooks.base_google import GoogleBaseAsyncHook, GoogleBaseHook
+from airflow.providers.google.common.hooks.base_google import (
+    PROVIDE_PROJECT_ID,
+    GoogleBaseAsyncHook,
+    GoogleBaseHook,
+)
 
 if TYPE_CHECKING:
     from google.api_core.operation import Operation
@@ -665,7 +669,7 @@ class DataplexHook(GoogleBaseHook):
         self,
         data_scan_id: str,
         job_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         region: str | None = None,
         wait_time: int = 10,
         result_timeout: float | None = None,

airflow/providers/google/cloud/hooks/dlp.py

@@ -162,7 +162,7 @@ class CloudDLPHook(GoogleBaseHook):
     def create_deidentify_template(
         self,
         organization_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         deidentify_template: dict | DeidentifyTemplate | None = None,
         template_id: str | None = None,
         retry: Retry | _MethodDefault = DEFAULT,
@@ -287,7 +287,7 @@ class CloudDLPHook(GoogleBaseHook):
     def create_inspect_template(
         self,
         organization_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         inspect_template: InspectTemplate | None = None,
         template_id: str | None = None,
         retry: Retry | _MethodDefault = DEFAULT,
@@ -376,7 +376,7 @@ class CloudDLPHook(GoogleBaseHook):
     def create_stored_info_type(
         self,
         organization_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         config: dict | StoredInfoTypeConfig | None = None,
         stored_info_type_id: str | None = None,
         retry: Retry | _MethodDefault = DEFAULT,
@@ -565,7 +565,7 @@ class CloudDLPHook(GoogleBaseHook):
         self,
         template_id: str,
         organization_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -652,7 +652,7 @@ class CloudDLPHook(GoogleBaseHook):
         self,
         stored_info_type_id: str,
         organization_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -701,7 +701,7 @@ class CloudDLPHook(GoogleBaseHook):
         self,
         template_id: str,
         organization_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -788,7 +788,7 @@ class CloudDLPHook(GoogleBaseHook):
         self,
         template_id: str,
         organization_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -875,7 +875,7 @@ class CloudDLPHook(GoogleBaseHook):
         self,
         stored_info_type_id: str,
         organization_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -967,7 +967,7 @@ class CloudDLPHook(GoogleBaseHook):
     def list_deidentify_templates(
         self,
         organization_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         page_size: int | None = None,
         order_by: str | None = None,
         retry: Retry | _MethodDefault = DEFAULT,
@@ -1103,7 +1103,7 @@ class CloudDLPHook(GoogleBaseHook):
     def list_inspect_templates(
         self,
         organization_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         page_size: int | None = None,
         order_by: str | None = None,
         retry: Retry | _MethodDefault = DEFAULT,
@@ -1201,7 +1201,7 @@ class CloudDLPHook(GoogleBaseHook):
     def list_stored_info_types(
         self,
         organization_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         page_size: int | None = None,
         order_by: str | None = None,
         retry: Retry | _MethodDefault = DEFAULT,
@@ -1356,7 +1356,7 @@ class CloudDLPHook(GoogleBaseHook):
         self,
         template_id: str,
         organization_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         deidentify_template: dict | DeidentifyTemplate | None = None,
         update_mask: dict | FieldMask | None = None,
         retry: Retry | _MethodDefault = DEFAULT,
@@ -1411,7 +1411,7 @@ class CloudDLPHook(GoogleBaseHook):
         self,
         template_id: str,
         organization_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         inspect_template: dict | InspectTemplate | None = None,
         update_mask: dict | FieldMask | None = None,
         retry: Retry | _MethodDefault = DEFAULT,
@@ -1513,7 +1513,7 @@ class CloudDLPHook(GoogleBaseHook):
         self,
         stored_info_type_id: str,
         organization_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         config: dict | StoredInfoTypeConfig | None = None,
         update_mask: dict | FieldMask | None = None,
         retry: Retry | _MethodDefault = DEFAULT,

airflow/providers/google/cloud/hooks/gcs.py

@@ -45,7 +45,11 @@ from requests import Session
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.providers.google.cloud.utils.helpers import normalize_directory_path
 from airflow.providers.google.common.consts import CLIENT_INFO
-from airflow.providers.google.common.hooks.base_google import GoogleBaseAsyncHook, GoogleBaseHook
+from airflow.providers.google.common.hooks.base_google import (
+    PROVIDE_PROJECT_ID,
+    GoogleBaseAsyncHook,
+    GoogleBaseHook,
+)
 from airflow.typing_compat import ParamSpec
 from airflow.utils import timezone
 from airflow.version import version
@@ -1013,7 +1017,7 @@ class GCSHook(GoogleBaseHook):
         resource: dict | None = None,
         storage_class: str = "MULTI_REGIONAL",
         location: str = "US",
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         labels: dict | None = None,
     ) -> str:
         """

airflow/providers/google/cloud/hooks/gdm.py

@@ -22,7 +22,7 @@ from typing import Any, Sequence
 from googleapiclient.discovery import Resource, build
 
 from airflow.exceptions import AirflowException
-from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
 
 
 class GoogleDeploymentManagerHook(GoogleBaseHook):
@@ -56,7 +56,7 @@ class GoogleDeploymentManagerHook(GoogleBaseHook):
     @GoogleBaseHook.fallback_to_default_project_id
     def list_deployments(
         self,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         deployment_filter: str | None = None,
         order_by: str | None = None,
     ) -> list[dict[str, Any]]:

airflow/providers/google/cloud/hooks/kubernetes_engine.py

@@ -156,7 +156,7 @@ class GKEHook(GoogleBaseHook):
     def get_client(self) -> ClusterManagerClient:
         return self.get_conn()
 
-    def wait_for_operation(self, operation: Operation, project_id: str | None = None) -> Operation:
+    def wait_for_operation(self, operation: Operation, project_id: str = PROVIDE_PROJECT_ID) -> Operation:
         """Continuously fetch the status from Google Cloud.
 
         This is done until the given operation completes, or raises an error.
@@ -176,7 +176,7 @@ class GKEHook(GoogleBaseHook):
             operation = self.get_operation(operation.name, project_id=project_id or self.project_id)
         return operation
 
-    def get_operation(self, operation_name: str, project_id: str | None = None) -> Operation:
+    def get_operation(self, operation_name: str, project_id: str = PROVIDE_PROJECT_ID) -> Operation:
         """Get an operation from Google Cloud.
 
         :param operation_name: Name of operation to fetch

airflow/providers/google/cloud/hooks/mlengine.py

@@ -31,7 +31,11 @@ from googleapiclient.discovery import Resource, build
 from googleapiclient.errors import HttpError
 
 from airflow.exceptions import AirflowException
-from airflow.providers.google.common.hooks.base_google import GoogleBaseAsyncHook, GoogleBaseHook
+from airflow.providers.google.common.hooks.base_google import (
+    PROVIDE_PROJECT_ID,
+    GoogleBaseAsyncHook,
+    GoogleBaseHook,
+)
 from airflow.version import version as airflow_version
 
 if TYPE_CHECKING:
@@ -550,7 +554,7 @@ class MLEngineAsyncHook(GoogleBaseAsyncHook):
     def _check_fileds(
         self,
         job_id: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
     ):
         if not project_id:
             raise AirflowException("Google Cloud project id is required.")
@@ -569,7 +573,7 @@ class MLEngineAsyncHook(GoogleBaseAsyncHook):
 
         return job
 
-    async def get_job(self, job_id: str, session: Session, project_id: str | None = None):
+    async def get_job(self, job_id: str, session: Session, project_id: str = PROVIDE_PROJECT_ID):
         """Get the specified job resource by job ID and project ID."""
         self._check_fileds(project_id=project_id, job_id=job_id)
 
@@ -579,7 +583,7 @@ class MLEngineAsyncHook(GoogleBaseAsyncHook):
     async def get_job_status(
         self,
         job_id: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
     ) -> str | None:
         """
         Poll for job status asynchronously using gcloud-aio.

airflow/providers/google/cloud/hooks/pubsub.py

@@ -590,7 +590,7 @@ class PubSubAsyncHook(GoogleBaseAsyncHook):
 
     sync_hook_class = PubSubHook
 
-    def __init__(self, project_id: str | None = None, **kwargs: Any):
+    def __init__(self, project_id: str = PROVIDE_PROJECT_ID, **kwargs: Any):
         super().__init__(**kwargs)
         self.project_id = project_id
         self._client: SubscriberAsyncClient | None = None

airflow/providers/google/cloud/hooks/secret_manager.py

@@ -15,16 +15,38 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-"""Hook for Secrets Manager service."""
+"""This module contains a Secret Manager hook."""
 
 from __future__ import annotations
 
-from typing import Sequence
+from functools import cached_property
+from typing import TYPE_CHECKING, Sequence
 
+from deprecated import deprecated
+from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
+from google.cloud.secretmanager_v1 import (
+    AccessSecretVersionResponse,
+    Secret,
+    SecretManagerServiceClient,
+    SecretPayload,
+    SecretVersion,
+)
+
+from airflow.exceptions import AirflowProviderDeprecationWarning
 from airflow.providers.google.cloud._internal_client.secret_manager_client import _SecretManagerClient
-from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
+from airflow.providers.google.common.consts import CLIENT_INFO
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
+
+if TYPE_CHECKING:
+    from google.api_core.retry import Retry
+    from google.cloud.secretmanager_v1.services.secret_manager_service.pagers import ListSecretsPager
 
 
+@deprecated(
+    reason="The SecretsManagerHook is deprecated and will be removed after 01.11.2024. "
+    "Please use GoogleCloudSecretManagerHook instead.",
+    category=AirflowProviderDeprecationWarning,
+)
 class SecretsManagerHook(GoogleBaseHook):
     """
     Hook for the Google Secret Manager API.
@@ -72,7 +94,7 @@ class SecretsManagerHook(GoogleBaseHook):
 
     @GoogleBaseHook.fallback_to_default_project_id
     def get_secret(
-        self, secret_id: str, secret_version: str = "latest", project_id: str | None = None
+        self, secret_id: str, secret_version: str = "latest", project_id: str = PROVIDE_PROJECT_ID
     ) -> str | None:
         """
         Get secret value from the Secret Manager.
@@ -86,3 +108,229 @@ class SecretsManagerHook(GoogleBaseHook):
             secret_version=secret_version,
             project_id=project_id,  # type: ignore
         )
+
+
+class GoogleCloudSecretManagerHook(GoogleBaseHook):
+    """Hook for the Google Cloud Secret Manager API.
+
+    See https://cloud.google.com/secret-manager
+    """
+
+    @cached_property
+    def client(self):
+        """Create a Secret Manager Client.
+
+        :return: Secret Manager client.
+        """
+        return SecretManagerServiceClient(credentials=self.get_credentials(), client_info=CLIENT_INFO)
+
+    def get_conn(self) -> SecretManagerServiceClient:
+        """Retrieve the connection to Secret Manager.
+
+        :return: Secret Manager client.
+        """
+        return self.client
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def create_secret(
+        self,
+        project_id: str,
+        secret_id: str,
+        secret: dict | Secret | None = None,
+        retry: Retry | _MethodDefault = DEFAULT,
+        timeout: float | None = None,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> Secret:
+        """Create a secret.
+
+        .. seealso::
+            For more details see API documentation:
+            https://cloud.google.com/python/docs/reference/secretmanager/latest/google.cloud.secretmanager_v1.services.secret_manager_service.SecretManagerServiceClient#google_cloud_secretmanager_v1_services_secret_manager_service_SecretManagerServiceClient_create_secret
+
+        :param project_id: Required. ID of the GCP project that owns the job.
+            If set to ``None`` or missing, the default project_id from the GCP connection is used.
+        :param secret_id: Required. ID of the secret to create.
+        :param secret: Optional. Secret to create.
+        :param retry: Optional. Designation of what errors, if any, should be retried.
+        :param timeout: Optional. The timeout for this request.
+        :param metadata: Optional. Strings which should be sent along with the request as metadata.
+        :return: Secret object.
+        """
+        _secret = secret or {"replication": {"automatic": {}}}
+        response = self.client.create_secret(
+            request={
+                "parent": f"projects/{project_id}",
+                "secret_id": secret_id,
+                "secret": _secret,
+            },
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )
+        self.log.info("Secret Created: %s", response.name)
+        return response
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def add_secret_version(
+        self,
+        project_id: str,
+        secret_id: str,
+        secret_payload: dict | SecretPayload | None = None,
+        retry: Retry | _MethodDefault = DEFAULT,
+        timeout: float | None = None,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> SecretVersion:
+        """Add a version to the secret.
+
+        .. seealso::
+            For more details see API documentation:
+            https://cloud.google.com/python/docs/reference/secretmanager/latest/google.cloud.secretmanager_v1.services.secret_manager_service.SecretManagerServiceClient#google_cloud_secretmanager_v1_services_secret_manager_service_SecretManagerServiceClient_add_secret_version
+
+        :param project_id: Required. ID of the GCP project that owns the job.
+            If set to ``None`` or missing, the default project_id from the GCP connection is used.
+        :param secret_id: Required. ID of the secret to create.
+        :param secret_payload: Optional. A secret payload.
+        :param retry: Optional. Designation of what errors, if any, should be retried.
+        :param timeout: Optional. The timeout for this request.
+        :param metadata: Optional. Strings which should be sent along with the request as metadata.
+        :return: Secret version object.
+        """
+        response = self.client.add_secret_version(
+            request={
+                "parent": f"projects/{project_id}/secrets/{secret_id}",
+                "payload": secret_payload,
+            },
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )
+        self.log.info("Secret version added: %s", response.name)
+        return response
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def list_secrets(
+        self,
+        project_id: str,
+        page_size: int = 0,
+        page_token: str | None = None,
+        secret_filter: str | None = None,
+        retry: Retry | _MethodDefault = DEFAULT,
+        timeout: float | None = None,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> ListSecretsPager:
+        """List secrets.
+
+        .. seealso::
+            For more details see API documentation:
+            https://cloud.google.com/python/docs/reference/secretmanager/latest/google.cloud.secretmanager_v1.services.secret_manager_service.SecretManagerServiceClient#google_cloud_secretmanager_v1_services_secret_manager_service_SecretManagerServiceClient_list_secrets
+
+        :param project_id: Required. ID of the GCP project that owns the job.
+            If set to ``None`` or missing, the default project_id from the GCP connection is used.
+        :param page_size: Optional, number of results to return in the list.
+        :param page_token: Optional, token to provide to skip to a particular spot in the list.
+        :param secret_filter: Optional. Filter string.
+        :param retry: Optional. Designation of what errors, if any, should be retried.
+        :param timeout: Optional. The timeout for this request.
+        :param metadata: Optional. Strings which should be sent along with the request as metadata.
+        :return: Secret List object.
+        """
+        response = self.client.list_secrets(
+            request={
+                "parent": f"projects/{project_id}",
+                "page_size": page_size,
+                "page_token": page_token,
+                "filter": secret_filter,
+            },
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )
+        self.log.info("Secrets list obtained")
+        return response
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def secret_exists(self, project_id: str, secret_id: str) -> bool:
+        """Check whether secret exists.
+
+        :param project_id: Required. ID of the GCP project that owns the job.
+            If set to ``None`` or missing, the default project_id from the GCP connection is used.
+        :param secret_id: Required. ID of the secret to find.
+        :return: True if the secret exists, False otherwise.
+        """
+        secret_filter = f"name:{secret_id}"
+        secret_name = self.client.secret_path(project_id, secret_id)
+        for secret in self.list_secrets(project_id=project_id, page_size=100, secret_filter=secret_filter):
+            if secret.name.split("/")[-1] == secret_id:
+                self.log.info("Secret %s exists.", secret_name)
+                return True
+        self.log.info("Secret %s doesn't exists.", secret_name)
+        return False
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def access_secret(
+        self,
+        project_id: str,
+        secret_id: str,
+        secret_version: str = "latest",
+        retry: Retry | _MethodDefault = DEFAULT,
+        timeout: float | None = None,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> AccessSecretVersionResponse:
+        """Access a secret version.
+
+        .. seealso::
+            For more details see API documentation:
+            https://cloud.google.com/python/docs/reference/secretmanager/latest/google.cloud.secretmanager_v1.services.secret_manager_service.SecretManagerServiceClient#google_cloud_secretmanager_v1_services_secret_manager_service_SecretManagerServiceClient_access_secret_version
+
+        :param project_id: Required. ID of the GCP project that owns the job.
+            If set to ``None`` or missing, the default project_id from the GCP connection is used.
+        :param secret_id: Required. ID of the secret to access.
+        :param secret_version: Optional. Version of the secret to access. Default: latest.
+        :param retry: Optional. Designation of what errors, if any, should be retried.
+        :param timeout: Optional. The timeout for this request.
+        :param metadata: Optional. Strings which should be sent along with the request as metadata.
+        :return: Access secret version response object.
+        """
+        response = self.client.access_secret_version(
+            request={
+                "name": self.client.secret_version_path(project_id, secret_id, secret_version),
+            },
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )
+        self.log.info("Secret version accessed: %s", response.name)
+        return response
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def delete_secret(
+        self,
+        project_id: str,
+        secret_id: str,
+        retry: Retry | _MethodDefault = DEFAULT,
+        timeout: float | None = None,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> None:
+        """Delete a secret.
+
+        .. seealso::
+            For more details see API documentation:
+            https://cloud.google.com/python/docs/reference/secretmanager/latest/google.cloud.secretmanager_v1.services.secret_manager_service.SecretManagerServiceClient#google_cloud_secretmanager_v1_services_secret_manager_service_SecretManagerServiceClient_delete_secret
+
+        :param project_id: Required. ID of the GCP project that owns the job.
+            If set to ``None`` or missing, the default project_id from the GCP connection is used.
+        :param secret_id: Required. ID of the secret to delete.
+        :param retry: Optional. Designation of what errors, if any, should be retried.
+        :param timeout: Optional. The timeout for this request.
+        :param metadata: Optional. Strings which should be sent along with the request as metadata.
+        :return: Access secret version response object.
+        """
+        name = self.client.secret_path(project_id, secret_id)
+        self.client.delete_secret(
+            request={"name": name},
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )
+        self.log.info("Secret deleted: %s", name)
+        return None