apache-airflow-providers-google 14.0.0__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
- airflow/providers/google/3rd-party-licenses/NOTICE +5 -0
- airflow/providers/google/__init__.py +3 -3
- airflow/providers/google/_vendor/__init__.py +0 -0
- airflow/providers/google/_vendor/json_merge_patch.py +91 -0
- airflow/providers/google/ads/hooks/ads.py +52 -43
- airflow/providers/google/ads/operators/ads.py +2 -2
- airflow/providers/google/ads/transfers/ads_to_gcs.py +3 -19
- airflow/providers/google/assets/gcs.py +1 -11
- airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
- airflow/providers/google/cloud/bundles/gcs.py +161 -0
- airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
- airflow/providers/google/cloud/hooks/bigquery.py +195 -318
- airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
- airflow/providers/google/cloud/hooks/bigtable.py +3 -2
- airflow/providers/google/cloud/hooks/cloud_batch.py +8 -9
- airflow/providers/google/cloud/hooks/cloud_build.py +6 -65
- airflow/providers/google/cloud/hooks/cloud_composer.py +292 -24
- airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
- airflow/providers/google/cloud/hooks/cloud_run.py +20 -11
- airflow/providers/google/cloud/hooks/cloud_sql.py +136 -64
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +35 -15
- airflow/providers/google/cloud/hooks/compute.py +7 -6
- airflow/providers/google/cloud/hooks/compute_ssh.py +7 -4
- airflow/providers/google/cloud/hooks/datacatalog.py +12 -3
- airflow/providers/google/cloud/hooks/dataflow.py +87 -242
- airflow/providers/google/cloud/hooks/dataform.py +9 -14
- airflow/providers/google/cloud/hooks/datafusion.py +7 -9
- airflow/providers/google/cloud/hooks/dataplex.py +13 -12
- airflow/providers/google/cloud/hooks/dataprep.py +2 -2
- airflow/providers/google/cloud/hooks/dataproc.py +76 -74
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/hooks/dlp.py +5 -4
- airflow/providers/google/cloud/hooks/gcs.py +144 -33
- airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
- airflow/providers/google/cloud/hooks/kms.py +3 -2
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +22 -17
- airflow/providers/google/cloud/hooks/looker.py +6 -1
- airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
- airflow/providers/google/cloud/hooks/mlengine.py +7 -8
- airflow/providers/google/cloud/hooks/natural_language.py +3 -2
- airflow/providers/google/cloud/hooks/os_login.py +3 -2
- airflow/providers/google/cloud/hooks/pubsub.py +6 -6
- airflow/providers/google/cloud/hooks/secret_manager.py +105 -12
- airflow/providers/google/cloud/hooks/spanner.py +75 -10
- airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
- airflow/providers/google/cloud/hooks/stackdriver.py +18 -18
- airflow/providers/google/cloud/hooks/tasks.py +4 -3
- airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
- airflow/providers/google/cloud/hooks/translate.py +8 -17
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +8 -222
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +9 -15
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +33 -283
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +311 -10
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +7 -13
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +8 -12
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
- airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
- airflow/providers/google/cloud/hooks/vision.py +7 -7
- airflow/providers/google/cloud/hooks/workflows.py +4 -3
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +77 -7
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -46
- airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -90
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +11 -89
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +11 -61
- airflow/providers/google/cloud/links/managed_kafka.py +11 -51
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +17 -187
- airflow/providers/google/cloud/links/vertex_ai.py +28 -195
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/log/gcs_task_handler.py +166 -118
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +14 -9
- airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
- airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
- airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
- airflow/providers/google/cloud/openlineage/facets.py +141 -40
- airflow/providers/google/cloud/openlineage/mixins.py +14 -13
- airflow/providers/google/cloud/openlineage/utils.py +19 -3
- airflow/providers/google/cloud/operators/alloy_db.py +76 -61
- airflow/providers/google/cloud/operators/bigquery.py +104 -667
- airflow/providers/google/cloud/operators/bigquery_dts.py +12 -12
- airflow/providers/google/cloud/operators/bigtable.py +38 -7
- airflow/providers/google/cloud/operators/cloud_base.py +22 -1
- airflow/providers/google/cloud/operators/cloud_batch.py +18 -18
- airflow/providers/google/cloud/operators/cloud_build.py +80 -36
- airflow/providers/google/cloud/operators/cloud_composer.py +157 -71
- airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
- airflow/providers/google/cloud/operators/cloud_memorystore.py +74 -46
- airflow/providers/google/cloud/operators/cloud_run.py +39 -20
- airflow/providers/google/cloud/operators/cloud_sql.py +46 -61
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +92 -14
- airflow/providers/google/cloud/operators/compute.py +18 -50
- airflow/providers/google/cloud/operators/datacatalog.py +167 -29
- airflow/providers/google/cloud/operators/dataflow.py +38 -15
- airflow/providers/google/cloud/operators/dataform.py +19 -7
- airflow/providers/google/cloud/operators/datafusion.py +43 -43
- airflow/providers/google/cloud/operators/dataplex.py +212 -126
- airflow/providers/google/cloud/operators/dataprep.py +1 -5
- airflow/providers/google/cloud/operators/dataproc.py +134 -207
- airflow/providers/google/cloud/operators/dataproc_metastore.py +102 -84
- airflow/providers/google/cloud/operators/datastore.py +22 -6
- airflow/providers/google/cloud/operators/dlp.py +24 -45
- airflow/providers/google/cloud/operators/functions.py +21 -14
- airflow/providers/google/cloud/operators/gcs.py +15 -12
- airflow/providers/google/cloud/operators/gen_ai.py +389 -0
- airflow/providers/google/cloud/operators/kubernetes_engine.py +115 -106
- airflow/providers/google/cloud/operators/looker.py +1 -1
- airflow/providers/google/cloud/operators/managed_kafka.py +362 -40
- airflow/providers/google/cloud/operators/natural_language.py +5 -3
- airflow/providers/google/cloud/operators/pubsub.py +69 -21
- airflow/providers/google/cloud/operators/spanner.py +53 -45
- airflow/providers/google/cloud/operators/speech_to_text.py +5 -4
- airflow/providers/google/cloud/operators/stackdriver.py +5 -11
- airflow/providers/google/cloud/operators/tasks.py +6 -15
- airflow/providers/google/cloud/operators/text_to_speech.py +4 -3
- airflow/providers/google/cloud/operators/translate.py +46 -20
- airflow/providers/google/cloud/operators/translate_speech.py +4 -3
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +44 -34
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +34 -12
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +62 -53
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +75 -11
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +48 -12
- airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -116
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +16 -12
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +62 -14
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +35 -10
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
- airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
- airflow/providers/google/cloud/operators/vision.py +7 -5
- airflow/providers/google/cloud/operators/workflows.py +24 -19
- airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
- airflow/providers/google/cloud/sensors/bigquery.py +2 -2
- airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -4
- airflow/providers/google/cloud/sensors/bigtable.py +14 -6
- airflow/providers/google/cloud/sensors/cloud_composer.py +535 -33
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -5
- airflow/providers/google/cloud/sensors/dataflow.py +27 -10
- airflow/providers/google/cloud/sensors/dataform.py +2 -2
- airflow/providers/google/cloud/sensors/datafusion.py +4 -4
- airflow/providers/google/cloud/sensors/dataplex.py +7 -5
- airflow/providers/google/cloud/sensors/dataprep.py +2 -2
- airflow/providers/google/cloud/sensors/dataproc.py +10 -9
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/sensors/gcs.py +22 -21
- airflow/providers/google/cloud/sensors/looker.py +5 -5
- airflow/providers/google/cloud/sensors/pubsub.py +20 -20
- airflow/providers/google/cloud/sensors/tasks.py +2 -2
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -2
- airflow/providers/google/cloud/sensors/workflows.py +6 -4
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +14 -13
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
- airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +18 -22
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -5
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +45 -38
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/gcs_to_local.py +5 -3
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +10 -4
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
- airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
- airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +44 -12
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +12 -6
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +36 -14
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
- airflow/providers/google/cloud/triggers/bigquery.py +75 -34
- airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +3 -2
- airflow/providers/google/cloud/triggers/cloud_composer.py +303 -47
- airflow/providers/google/cloud/triggers/cloud_run.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +96 -5
- airflow/providers/google/cloud/triggers/dataflow.py +125 -2
- airflow/providers/google/cloud/triggers/datafusion.py +1 -1
- airflow/providers/google/cloud/triggers/dataplex.py +16 -3
- airflow/providers/google/cloud/triggers/dataproc.py +124 -53
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +46 -28
- airflow/providers/google/cloud/triggers/mlengine.py +1 -1
- airflow/providers/google/cloud/triggers/pubsub.py +17 -20
- airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
- airflow/providers/google/cloud/utils/bigquery.py +5 -7
- airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
- airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
- airflow/providers/google/cloud/utils/field_validator.py +1 -2
- airflow/providers/google/cloud/utils/validators.py +43 -0
- airflow/providers/google/common/auth_backend/google_openid.py +26 -9
- airflow/providers/google/common/consts.py +2 -1
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +40 -43
- airflow/providers/google/common/hooks/operation_helpers.py +78 -0
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/common/utils/get_secret.py +31 -0
- airflow/providers/google/common/utils/id_token_credentials.py +4 -5
- airflow/providers/google/firebase/operators/firestore.py +2 -2
- airflow/providers/google/get_provider_info.py +61 -216
- airflow/providers/google/go_module_utils.py +35 -3
- airflow/providers/google/leveldb/hooks/leveldb.py +30 -6
- airflow/providers/google/leveldb/operators/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
- airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
- airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +4 -5
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -6
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +5 -5
- airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
- airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
- airflow/providers/google/marketing_platform/sensors/display_video.py +4 -64
- airflow/providers/google/suite/hooks/calendar.py +1 -1
- airflow/providers/google/suite/hooks/drive.py +2 -2
- airflow/providers/google/suite/hooks/sheets.py +15 -1
- airflow/providers/google/suite/operators/sheets.py +8 -3
- airflow/providers/google/suite/sensors/drive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
- airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
- airflow/providers/google/version_compat.py +15 -1
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/METADATA +117 -72
- apache_airflow_providers_google-19.1.0rc1.dist-info/RECORD +331 -0
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/WHEEL +1 -1
- apache_airflow_providers_google-19.1.0rc1.dist-info/licenses/NOTICE +5 -0
- airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
- airflow/providers/google/cloud/hooks/automl.py +0 -679
- airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
- airflow/providers/google/cloud/links/automl.py +0 -193
- airflow/providers/google/cloud/operators/automl.py +0 -1360
- airflow/providers/google/cloud/operators/life_sciences.py +0 -119
- airflow/providers/google/cloud/operators/mlengine.py +0 -1515
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +0 -273
- apache_airflow_providers_google-14.0.0.dist-info/RECORD +0 -318
- /airflow/providers/google/cloud/{example_dags → bundles}/__init__.py +0 -0
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/entry_points.txt +0 -0
- {airflow/providers/google → apache_airflow_providers_google-19.1.0rc1.dist-info/licenses}/LICENSE +0 -0
|
@@ -19,23 +19,81 @@
|
|
|
19
19
|
|
|
20
20
|
from __future__ import annotations
|
|
21
21
|
|
|
22
|
+
import base64
|
|
23
|
+
import datetime
|
|
24
|
+
import json
|
|
25
|
+
import time
|
|
22
26
|
from collections.abc import Sequence
|
|
23
27
|
from copy import deepcopy
|
|
24
28
|
from typing import TYPE_CHECKING
|
|
25
29
|
|
|
30
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
31
|
+
from google.auth.transport import requests as google_requests
|
|
32
|
+
from google.cloud.managedkafka_v1 import Cluster, ConsumerGroup, ManagedKafkaClient, Topic, types
|
|
33
|
+
|
|
26
34
|
from airflow.exceptions import AirflowException
|
|
27
35
|
from airflow.providers.google.common.consts import CLIENT_INFO
|
|
28
36
|
from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
|
|
29
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
30
|
-
from google.cloud.managedkafka_v1 import Cluster, ManagedKafkaClient, Topic, types
|
|
31
37
|
|
|
32
38
|
if TYPE_CHECKING:
|
|
33
39
|
from google.api_core.operation import Operation
|
|
34
40
|
from google.api_core.retry import Retry
|
|
35
|
-
from google.
|
|
41
|
+
from google.auth.credentials import Credentials
|
|
42
|
+
from google.cloud.managedkafka_v1.services.managed_kafka.pagers import (
|
|
43
|
+
ListClustersPager,
|
|
44
|
+
ListConsumerGroupsPager,
|
|
45
|
+
ListTopicsPager,
|
|
46
|
+
)
|
|
36
47
|
from google.protobuf.field_mask_pb2 import FieldMask
|
|
37
48
|
|
|
38
49
|
|
|
50
|
+
class ManagedKafkaTokenProvider:
|
|
51
|
+
"""Helper for providing authentication token for establishing connection via confluent to Apache Kafka cluster managed by Google Cloud."""
|
|
52
|
+
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
credentials: Credentials,
|
|
56
|
+
):
|
|
57
|
+
self._credentials = credentials
|
|
58
|
+
self._header = json.dumps(dict(typ="JWT", alg="GOOG_OAUTH2_TOKEN"))
|
|
59
|
+
|
|
60
|
+
def _valid_credentials(self):
|
|
61
|
+
if not self._credentials.valid:
|
|
62
|
+
self._credentials.refresh(google_requests.Request())
|
|
63
|
+
return self._credentials
|
|
64
|
+
|
|
65
|
+
def _get_jwt(self, credentials):
|
|
66
|
+
return json.dumps(
|
|
67
|
+
dict(
|
|
68
|
+
exp=credentials.expiry.timestamp(),
|
|
69
|
+
iss="Google",
|
|
70
|
+
iat=datetime.datetime.now(datetime.timezone.utc).timestamp(),
|
|
71
|
+
scope="kafka",
|
|
72
|
+
sub=credentials.service_account_email,
|
|
73
|
+
)
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
def _b64_encode(self, source):
|
|
77
|
+
return base64.urlsafe_b64encode(source.encode("utf-8")).decode("utf-8").rstrip("=")
|
|
78
|
+
|
|
79
|
+
def _get_kafka_access_token(self, credentials):
|
|
80
|
+
return ".".join(
|
|
81
|
+
[
|
|
82
|
+
self._b64_encode(self._header),
|
|
83
|
+
self._b64_encode(self._get_jwt(credentials)),
|
|
84
|
+
self._b64_encode(credentials.token),
|
|
85
|
+
]
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
def confluent_token(self):
|
|
89
|
+
credentials = self._valid_credentials()
|
|
90
|
+
|
|
91
|
+
utc_expiry = credentials.expiry.replace(tzinfo=datetime.timezone.utc)
|
|
92
|
+
expiry_seconds = (utc_expiry - datetime.datetime.now(datetime.timezone.utc)).total_seconds()
|
|
93
|
+
|
|
94
|
+
return self._get_kafka_access_token(credentials), time.time() + expiry_seconds
|
|
95
|
+
|
|
96
|
+
|
|
39
97
|
class ManagedKafkaHook(GoogleBaseHook):
|
|
40
98
|
"""Hook for Managed Service for Apache Kafka APIs."""
|
|
41
99
|
|
|
@@ -62,6 +120,12 @@ class ManagedKafkaHook(GoogleBaseHook):
|
|
|
62
120
|
error = operation.exception(timeout=timeout)
|
|
63
121
|
raise AirflowException(error)
|
|
64
122
|
|
|
123
|
+
def get_confluent_token(self, config_str: str):
|
|
124
|
+
"""Get the authentication token for confluent client."""
|
|
125
|
+
token_provider = ManagedKafkaTokenProvider(credentials=self.get_credentials())
|
|
126
|
+
token = token_provider.confluent_token()
|
|
127
|
+
return token
|
|
128
|
+
|
|
65
129
|
@GoogleBaseHook.fallback_to_default_project_id
|
|
66
130
|
def create_cluster(
|
|
67
131
|
self,
|
|
@@ -480,3 +544,163 @@ class ManagedKafkaHook(GoogleBaseHook):
|
|
|
480
544
|
timeout=timeout,
|
|
481
545
|
metadata=metadata,
|
|
482
546
|
)
|
|
547
|
+
|
|
548
|
+
@GoogleBaseHook.fallback_to_default_project_id
|
|
549
|
+
def list_consumer_groups(
|
|
550
|
+
self,
|
|
551
|
+
project_id: str,
|
|
552
|
+
location: str,
|
|
553
|
+
cluster_id: str,
|
|
554
|
+
page_size: int | None = None,
|
|
555
|
+
page_token: str | None = None,
|
|
556
|
+
retry: Retry | _MethodDefault = DEFAULT,
|
|
557
|
+
timeout: float | None = None,
|
|
558
|
+
metadata: Sequence[tuple[str, str]] = (),
|
|
559
|
+
) -> ListConsumerGroupsPager:
|
|
560
|
+
"""
|
|
561
|
+
List the consumer groups in a given cluster.
|
|
562
|
+
|
|
563
|
+
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
|
564
|
+
:param location: Required. The ID of the Google Cloud region that the service belongs to.
|
|
565
|
+
:param cluster_id: Required. The ID of the cluster whose consumer groups are to be listed.
|
|
566
|
+
:param page_size: Optional. The maximum number of consumer groups to return. The service may return
|
|
567
|
+
fewer than this value. If unset or zero, all consumer groups for the parent is returned.
|
|
568
|
+
:param page_token: Optional. A page token, received from a previous ``ListConsumerGroups`` call.
|
|
569
|
+
Provide this to retrieve the subsequent page. When paginating, all other parameters provided to
|
|
570
|
+
``ListConsumerGroups`` must match the call that provided the page token.
|
|
571
|
+
:param retry: Designation of what errors, if any, should be retried.
|
|
572
|
+
:param timeout: The timeout for this request.
|
|
573
|
+
:param metadata: Strings which should be sent along with the request as metadata.
|
|
574
|
+
"""
|
|
575
|
+
client = self.get_managed_kafka_client()
|
|
576
|
+
parent = client.cluster_path(project_id, location, cluster_id)
|
|
577
|
+
|
|
578
|
+
result = client.list_consumer_groups(
|
|
579
|
+
request={
|
|
580
|
+
"parent": parent,
|
|
581
|
+
"page_size": page_size,
|
|
582
|
+
"page_token": page_token,
|
|
583
|
+
},
|
|
584
|
+
retry=retry,
|
|
585
|
+
timeout=timeout,
|
|
586
|
+
metadata=metadata,
|
|
587
|
+
)
|
|
588
|
+
return result
|
|
589
|
+
|
|
590
|
+
@GoogleBaseHook.fallback_to_default_project_id
|
|
591
|
+
def get_consumer_group(
|
|
592
|
+
self,
|
|
593
|
+
project_id: str,
|
|
594
|
+
location: str,
|
|
595
|
+
cluster_id: str,
|
|
596
|
+
consumer_group_id: str,
|
|
597
|
+
retry: Retry | _MethodDefault = DEFAULT,
|
|
598
|
+
timeout: float | None = None,
|
|
599
|
+
metadata: Sequence[tuple[str, str]] = (),
|
|
600
|
+
) -> types.ConsumerGroup:
|
|
601
|
+
"""
|
|
602
|
+
Return the properties of a single consumer group.
|
|
603
|
+
|
|
604
|
+
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
|
605
|
+
:param location: Required. The ID of the Google Cloud region that the service belongs to.
|
|
606
|
+
:param cluster_id: Required. The ID of the cluster whose consumer group is to be returned.
|
|
607
|
+
:param consumer_group_id: Required. The ID of the consumer group whose configuration to return.
|
|
608
|
+
:param retry: Designation of what errors, if any, should be retried.
|
|
609
|
+
:param timeout: The timeout for this request.
|
|
610
|
+
:param metadata: Strings which should be sent along with the request as metadata.
|
|
611
|
+
"""
|
|
612
|
+
client = self.get_managed_kafka_client()
|
|
613
|
+
name = client.consumer_group_path(project_id, location, cluster_id, consumer_group_id)
|
|
614
|
+
|
|
615
|
+
result = client.get_consumer_group(
|
|
616
|
+
request={
|
|
617
|
+
"name": name,
|
|
618
|
+
},
|
|
619
|
+
retry=retry,
|
|
620
|
+
timeout=timeout,
|
|
621
|
+
metadata=metadata,
|
|
622
|
+
)
|
|
623
|
+
return result
|
|
624
|
+
|
|
625
|
+
@GoogleBaseHook.fallback_to_default_project_id
|
|
626
|
+
def update_consumer_group(
|
|
627
|
+
self,
|
|
628
|
+
project_id: str,
|
|
629
|
+
location: str,
|
|
630
|
+
cluster_id: str,
|
|
631
|
+
consumer_group_id: str,
|
|
632
|
+
consumer_group: types.ConsumerGroup | dict,
|
|
633
|
+
update_mask: FieldMask | dict,
|
|
634
|
+
retry: Retry | _MethodDefault = DEFAULT,
|
|
635
|
+
timeout: float | None = None,
|
|
636
|
+
metadata: Sequence[tuple[str, str]] = (),
|
|
637
|
+
) -> types.ConsumerGroup:
|
|
638
|
+
"""
|
|
639
|
+
Update the properties of a single consumer group.
|
|
640
|
+
|
|
641
|
+
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
|
642
|
+
:param location: Required. The ID of the Google Cloud region that the service belongs to.
|
|
643
|
+
:param cluster_id: Required. The ID of the cluster whose topic is to be updated.
|
|
644
|
+
:param consumer_group_id: Required. The ID of the consumer group whose configuration to update.
|
|
645
|
+
:param consumer_group: Required. The consumer_group to update. Its ``name`` field must be populated.
|
|
646
|
+
:param update_mask: Required. Field mask is used to specify the fields to be overwritten in the
|
|
647
|
+
ConsumerGroup resource by the update. The fields specified in the update_mask are relative to the
|
|
648
|
+
resource, not the full request. A field will be overwritten if it is in the mask.
|
|
649
|
+
:param retry: Designation of what errors, if any, should be retried.
|
|
650
|
+
:param timeout: The timeout for this request.
|
|
651
|
+
:param metadata: Strings which should be sent along with the request as metadata.
|
|
652
|
+
"""
|
|
653
|
+
client = self.get_managed_kafka_client()
|
|
654
|
+
_consumer_group = (
|
|
655
|
+
deepcopy(consumer_group)
|
|
656
|
+
if isinstance(consumer_group, dict)
|
|
657
|
+
else ConsumerGroup.to_dict(consumer_group)
|
|
658
|
+
)
|
|
659
|
+
_consumer_group["name"] = client.consumer_group_path(
|
|
660
|
+
project_id, location, cluster_id, consumer_group_id
|
|
661
|
+
)
|
|
662
|
+
|
|
663
|
+
result = client.update_consumer_group(
|
|
664
|
+
request={
|
|
665
|
+
"update_mask": update_mask,
|
|
666
|
+
"consumer_group": _consumer_group,
|
|
667
|
+
},
|
|
668
|
+
retry=retry,
|
|
669
|
+
timeout=timeout,
|
|
670
|
+
metadata=metadata,
|
|
671
|
+
)
|
|
672
|
+
return result
|
|
673
|
+
|
|
674
|
+
@GoogleBaseHook.fallback_to_default_project_id
|
|
675
|
+
def delete_consumer_group(
|
|
676
|
+
self,
|
|
677
|
+
project_id: str,
|
|
678
|
+
location: str,
|
|
679
|
+
cluster_id: str,
|
|
680
|
+
consumer_group_id: str,
|
|
681
|
+
retry: Retry | _MethodDefault = DEFAULT,
|
|
682
|
+
timeout: float | None = None,
|
|
683
|
+
metadata: Sequence[tuple[str, str]] = (),
|
|
684
|
+
) -> None:
|
|
685
|
+
"""
|
|
686
|
+
Delete a single consumer group.
|
|
687
|
+
|
|
688
|
+
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
|
689
|
+
:param location: Required. The ID of the Google Cloud region that the service belongs to.
|
|
690
|
+
:param cluster_id: Required. The ID of the cluster whose consumer group is to be deleted.
|
|
691
|
+
:param consumer_group_id: Required. The ID of the consumer group to delete.
|
|
692
|
+
:param retry: Designation of what errors, if any, should be retried.
|
|
693
|
+
:param timeout: The timeout for this request.
|
|
694
|
+
:param metadata: Strings which should be sent along with the request as metadata.
|
|
695
|
+
"""
|
|
696
|
+
client = self.get_managed_kafka_client()
|
|
697
|
+
name = client.consumer_group_path(project_id, location, cluster_id, consumer_group_id)
|
|
698
|
+
|
|
699
|
+
client.delete_consumer_group(
|
|
700
|
+
request={
|
|
701
|
+
"name": name,
|
|
702
|
+
},
|
|
703
|
+
retry=retry,
|
|
704
|
+
timeout=timeout,
|
|
705
|
+
metadata=metadata,
|
|
706
|
+
)
|
|
@@ -23,7 +23,8 @@ import contextlib
|
|
|
23
23
|
import logging
|
|
24
24
|
import random
|
|
25
25
|
import time
|
|
26
|
-
from
|
|
26
|
+
from collections.abc import Callable
|
|
27
|
+
from typing import TYPE_CHECKING
|
|
27
28
|
|
|
28
29
|
from aiohttp import ClientSession
|
|
29
30
|
from gcloud.aio.auth import AioSession, Token
|
|
@@ -78,8 +79,7 @@ def _poll_with_exponential_delay(
|
|
|
78
79
|
if e.resp.status != 429:
|
|
79
80
|
log.info("Something went wrong. Not retrying: %s", format(e))
|
|
80
81
|
raise
|
|
81
|
-
|
|
82
|
-
time.sleep((2**i) + random.random())
|
|
82
|
+
time.sleep((2**i) + random.random())
|
|
83
83
|
|
|
84
84
|
raise ValueError(f"Connection could not be established after {max_n} retries.")
|
|
85
85
|
|
|
@@ -219,12 +219,11 @@ class MLEngineHook(GoogleBaseHook):
|
|
|
219
219
|
if e.resp.status == 404:
|
|
220
220
|
self.log.error("Job with job_id %s does not exist. ", job_id)
|
|
221
221
|
raise
|
|
222
|
-
|
|
222
|
+
if e.resp.status == 400:
|
|
223
223
|
self.log.info("Job with job_id %s is already complete, cancellation aborted.", job_id)
|
|
224
224
|
return {}
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
raise
|
|
225
|
+
self.log.error("Failed to cancel MLEngine job: %s", e)
|
|
226
|
+
raise
|
|
228
227
|
|
|
229
228
|
def get_job(self, project_id: str, job_id: str) -> dict:
|
|
230
229
|
"""
|
|
@@ -589,7 +588,7 @@ class MLEngineAsyncHook(GoogleBaseAsyncHook):
|
|
|
589
588
|
job = await self.get_job(
|
|
590
589
|
project_id=project_id,
|
|
591
590
|
job_id=job_id,
|
|
592
|
-
session=session, #
|
|
591
|
+
session=session, # type: ignore
|
|
593
592
|
)
|
|
594
593
|
job = await job.json(content_type=None)
|
|
595
594
|
self.log.info("Retrieving json_response: %s", job)
|
|
@@ -22,8 +22,6 @@ from __future__ import annotations
|
|
|
22
22
|
from collections.abc import Sequence
|
|
23
23
|
from typing import TYPE_CHECKING
|
|
24
24
|
|
|
25
|
-
from airflow.providers.google.common.consts import CLIENT_INFO
|
|
26
|
-
from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
|
|
27
25
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
28
26
|
from google.cloud.language_v1 import EncodingType, LanguageServiceClient
|
|
29
27
|
from google.cloud.language_v1.types import (
|
|
@@ -37,6 +35,9 @@ from google.cloud.language_v1.types import (
|
|
|
37
35
|
Document,
|
|
38
36
|
)
|
|
39
37
|
|
|
38
|
+
from airflow.providers.google.common.consts import CLIENT_INFO
|
|
39
|
+
from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
|
|
40
|
+
|
|
40
41
|
if TYPE_CHECKING:
|
|
41
42
|
from google.api_core.retry import Retry
|
|
42
43
|
|
|
@@ -27,11 +27,12 @@ from __future__ import annotations
|
|
|
27
27
|
from collections.abc import Sequence
|
|
28
28
|
from typing import TYPE_CHECKING
|
|
29
29
|
|
|
30
|
-
from airflow.providers.google.common.consts import CLIENT_INFO
|
|
31
|
-
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
|
32
30
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
33
31
|
from google.cloud.oslogin_v1 import ImportSshPublicKeyResponse, OsLoginServiceClient
|
|
34
32
|
|
|
33
|
+
from airflow.providers.google.common.consts import CLIENT_INFO
|
|
34
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
|
35
|
+
|
|
35
36
|
if TYPE_CHECKING:
|
|
36
37
|
from google.api_core.retry import Retry
|
|
37
38
|
|
|
@@ -33,6 +33,12 @@ from functools import cached_property
|
|
|
33
33
|
from typing import TYPE_CHECKING, Any
|
|
34
34
|
from uuid import uuid4
|
|
35
35
|
|
|
36
|
+
from google.api_core.exceptions import AlreadyExists, GoogleAPICallError
|
|
37
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
38
|
+
from google.cloud.exceptions import NotFound
|
|
39
|
+
from google.cloud.pubsub_v1 import PublisherClient, SubscriberClient
|
|
40
|
+
from google.cloud.pubsub_v1.types import PublisherOptions
|
|
41
|
+
from google.pubsub_v1.services.subscriber.async_client import SubscriberAsyncClient
|
|
36
42
|
from googleapiclient.errors import HttpError
|
|
37
43
|
|
|
38
44
|
from airflow.exceptions import AirflowProviderDeprecationWarning
|
|
@@ -43,12 +49,6 @@ from airflow.providers.google.common.hooks.base_google import (
|
|
|
43
49
|
GoogleBaseHook,
|
|
44
50
|
)
|
|
45
51
|
from airflow.version import version
|
|
46
|
-
from google.api_core.exceptions import AlreadyExists, GoogleAPICallError
|
|
47
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
48
|
-
from google.cloud.exceptions import NotFound
|
|
49
|
-
from google.cloud.pubsub_v1 import PublisherClient, SubscriberClient
|
|
50
|
-
from google.cloud.pubsub_v1.types import PublisherOptions
|
|
51
|
-
from google.pubsub_v1.services.subscriber.async_client import SubscriberAsyncClient
|
|
52
52
|
|
|
53
53
|
if TYPE_CHECKING:
|
|
54
54
|
from google.api_core.retry import Retry
|
|
@@ -23,8 +23,6 @@ from collections.abc import Sequence
|
|
|
23
23
|
from functools import cached_property
|
|
24
24
|
from typing import TYPE_CHECKING
|
|
25
25
|
|
|
26
|
-
from airflow.providers.google.common.consts import CLIENT_INFO
|
|
27
|
-
from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
|
|
28
26
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
29
27
|
from google.cloud.secretmanager_v1 import (
|
|
30
28
|
AccessSecretVersionResponse,
|
|
@@ -34,6 +32,9 @@ from google.cloud.secretmanager_v1 import (
|
|
|
34
32
|
SecretVersion,
|
|
35
33
|
)
|
|
36
34
|
|
|
35
|
+
from airflow.providers.google.common.consts import CLIENT_INFO
|
|
36
|
+
from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
|
|
37
|
+
|
|
37
38
|
if TYPE_CHECKING:
|
|
38
39
|
from google.api_core.retry import Retry
|
|
39
40
|
from google.cloud.secretmanager_v1.services.secret_manager_service.pagers import ListSecretsPager
|
|
@@ -46,6 +47,10 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
|
|
|
46
47
|
See https://cloud.google.com/secret-manager
|
|
47
48
|
"""
|
|
48
49
|
|
|
50
|
+
def __init__(self, location: str | None = None, **kwargs) -> None:
|
|
51
|
+
super().__init__(**kwargs)
|
|
52
|
+
self.location = location
|
|
53
|
+
|
|
49
54
|
@cached_property
|
|
50
55
|
def client(self):
|
|
51
56
|
"""
|
|
@@ -53,7 +58,16 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
|
|
|
53
58
|
|
|
54
59
|
:return: Secret Manager client.
|
|
55
60
|
"""
|
|
56
|
-
|
|
61
|
+
if self.location is not None:
|
|
62
|
+
return SecretManagerServiceClient(
|
|
63
|
+
credentials=self.get_credentials(),
|
|
64
|
+
client_info=CLIENT_INFO,
|
|
65
|
+
client_options={"api_endpoint": f"secretmanager.{self.location}.rep.googleapis.com"},
|
|
66
|
+
)
|
|
67
|
+
return SecretManagerServiceClient(
|
|
68
|
+
credentials=self.get_credentials(),
|
|
69
|
+
client_info=CLIENT_INFO,
|
|
70
|
+
)
|
|
57
71
|
|
|
58
72
|
def get_conn(self) -> SecretManagerServiceClient:
|
|
59
73
|
"""
|
|
@@ -63,6 +77,60 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
|
|
|
63
77
|
"""
|
|
64
78
|
return self.client
|
|
65
79
|
|
|
80
|
+
def _get_parent(self, project_id: str, location: str | None = None) -> str:
|
|
81
|
+
"""
|
|
82
|
+
Return parent path.
|
|
83
|
+
|
|
84
|
+
:param project_id: Required. ID of the GCP project that owns the job.
|
|
85
|
+
:param location: Optional. Target location. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
|
|
86
|
+
For more details : https://cloud.google.com/secret-manager/docs/locations
|
|
87
|
+
|
|
88
|
+
:return: Parent path.
|
|
89
|
+
"""
|
|
90
|
+
_location = location or self.location
|
|
91
|
+
if _location is not None:
|
|
92
|
+
return self.client.common_location_path(project_id, _location)
|
|
93
|
+
return self.client.common_project_path(project_id)
|
|
94
|
+
|
|
95
|
+
def _get_secret_path(self, project_id: str, secret_id: str, location: str | None = None) -> str:
|
|
96
|
+
"""
|
|
97
|
+
Return secret path.
|
|
98
|
+
|
|
99
|
+
:param project_id: Required. ID of the GCP project that owns the job.
|
|
100
|
+
:param secret_id: Required. Secret ID for which path is required.
|
|
101
|
+
:param location: Optional. Target location. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
|
|
102
|
+
For more details : https://cloud.google.com/secret-manager/docs/locations
|
|
103
|
+
|
|
104
|
+
:return: Parent path.
|
|
105
|
+
"""
|
|
106
|
+
_location = location or self.location
|
|
107
|
+
if _location is not None:
|
|
108
|
+
# Google's client library does not provide a method to construct regional secret paths, so constructing manually.
|
|
109
|
+
return f"projects/{project_id}/locations/{_location}/secrets/{secret_id}"
|
|
110
|
+
return self.client.secret_path(project_id, secret_id)
|
|
111
|
+
|
|
112
|
+
def _get_secret_version_path(
|
|
113
|
+
self, project_id: str, secret_id: str, secret_version: str, location: str | None = None
|
|
114
|
+
) -> str:
|
|
115
|
+
"""
|
|
116
|
+
Return secret version path.
|
|
117
|
+
|
|
118
|
+
:param project_id: Required. ID of the GCP project that owns the job.
|
|
119
|
+
:param secret_id: Required. Secret ID for which path is required.
|
|
120
|
+
:param secret_version: Required. Secret version for which path is required.
|
|
121
|
+
:param location: Optional. Target location. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
|
|
122
|
+
For more details : https://cloud.google.com/secret-manager/docs/locations
|
|
123
|
+
|
|
124
|
+
:return: Parent path.
|
|
125
|
+
"""
|
|
126
|
+
_location = location or self.location
|
|
127
|
+
if _location is not None:
|
|
128
|
+
# Google's client library does not provide a method to construct regional secret version paths, so constructing manually.
|
|
129
|
+
return (
|
|
130
|
+
f"projects/{project_id}/locations/{_location}/secrets/{secret_id}/versions/{secret_version}"
|
|
131
|
+
)
|
|
132
|
+
return self.client.secret_version_path(project_id, secret_id, secret_version)
|
|
133
|
+
|
|
66
134
|
@GoogleBaseHook.fallback_to_default_project_id
|
|
67
135
|
def create_secret(
|
|
68
136
|
self,
|
|
@@ -72,6 +140,7 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
|
|
|
72
140
|
retry: Retry | _MethodDefault = DEFAULT,
|
|
73
141
|
timeout: float | None = None,
|
|
74
142
|
metadata: Sequence[tuple[str, str]] = (),
|
|
143
|
+
location: str | None = None,
|
|
75
144
|
) -> Secret:
|
|
76
145
|
"""
|
|
77
146
|
Create a secret.
|
|
@@ -87,12 +156,20 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
|
|
|
87
156
|
:param retry: Optional. Designation of what errors, if any, should be retried.
|
|
88
157
|
:param timeout: Optional. The timeout for this request.
|
|
89
158
|
:param metadata: Optional. Strings which should be sent along with the request as metadata.
|
|
159
|
+
:param location: Optional. Location where secret should be created. Used for creating regional secret. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
|
|
160
|
+
For more details : https://cloud.google.com/secret-manager/docs/locations
|
|
90
161
|
:return: Secret object.
|
|
91
162
|
"""
|
|
92
|
-
|
|
163
|
+
if not secret:
|
|
164
|
+
_secret: dict | Secret = {}
|
|
165
|
+
if (location or self.location) is None:
|
|
166
|
+
_secret["replication"] = {"automatic": {}}
|
|
167
|
+
else:
|
|
168
|
+
_secret = secret
|
|
169
|
+
|
|
93
170
|
response = self.client.create_secret(
|
|
94
171
|
request={
|
|
95
|
-
"parent":
|
|
172
|
+
"parent": self._get_parent(project_id, location),
|
|
96
173
|
"secret_id": secret_id,
|
|
97
174
|
"secret": _secret,
|
|
98
175
|
},
|
|
@@ -112,6 +189,7 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
|
|
|
112
189
|
retry: Retry | _MethodDefault = DEFAULT,
|
|
113
190
|
timeout: float | None = None,
|
|
114
191
|
metadata: Sequence[tuple[str, str]] = (),
|
|
192
|
+
location: str | None = None,
|
|
115
193
|
) -> SecretVersion:
|
|
116
194
|
"""
|
|
117
195
|
Add a version to the secret.
|
|
@@ -127,11 +205,13 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
|
|
|
127
205
|
:param retry: Optional. Designation of what errors, if any, should be retried.
|
|
128
206
|
:param timeout: Optional. The timeout for this request.
|
|
129
207
|
:param metadata: Optional. Strings which should be sent along with the request as metadata.
|
|
208
|
+
:param location: Optional. Location where secret is located. Used for adding version to regional secret. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
|
|
209
|
+
For more details : https://cloud.google.com/secret-manager/docs/locations
|
|
130
210
|
:return: Secret version object.
|
|
131
211
|
"""
|
|
132
212
|
response = self.client.add_secret_version(
|
|
133
213
|
request={
|
|
134
|
-
"parent":
|
|
214
|
+
"parent": self._get_secret_path(project_id, secret_id, location),
|
|
135
215
|
"payload": secret_payload,
|
|
136
216
|
},
|
|
137
217
|
retry=retry,
|
|
@@ -151,6 +231,7 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
|
|
|
151
231
|
retry: Retry | _MethodDefault = DEFAULT,
|
|
152
232
|
timeout: float | None = None,
|
|
153
233
|
metadata: Sequence[tuple[str, str]] = (),
|
|
234
|
+
location: str | None = None,
|
|
154
235
|
) -> ListSecretsPager:
|
|
155
236
|
"""
|
|
156
237
|
List secrets.
|
|
@@ -167,11 +248,13 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
|
|
|
167
248
|
:param retry: Optional. Designation of what errors, if any, should be retried.
|
|
168
249
|
:param timeout: Optional. The timeout for this request.
|
|
169
250
|
:param metadata: Optional. Strings which should be sent along with the request as metadata.
|
|
251
|
+
:param location: Optional. The regional secrets stored in the provided location will be listed. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
|
|
252
|
+
For more details : https://cloud.google.com/secret-manager/docs/locations
|
|
170
253
|
:return: Secret List object.
|
|
171
254
|
"""
|
|
172
255
|
response = self.client.list_secrets(
|
|
173
256
|
request={
|
|
174
|
-
"parent":
|
|
257
|
+
"parent": self._get_parent(project_id, location),
|
|
175
258
|
"page_size": page_size,
|
|
176
259
|
"page_token": page_token,
|
|
177
260
|
"filter": secret_filter,
|
|
@@ -184,18 +267,22 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
|
|
|
184
267
|
return response
|
|
185
268
|
|
|
186
269
|
@GoogleBaseHook.fallback_to_default_project_id
|
|
187
|
-
def secret_exists(self, project_id: str, secret_id: str) -> bool:
|
|
270
|
+
def secret_exists(self, project_id: str, secret_id: str, location: str | None = None) -> bool:
|
|
188
271
|
"""
|
|
189
272
|
Check whether secret exists.
|
|
190
273
|
|
|
191
274
|
:param project_id: Required. ID of the GCP project that owns the job.
|
|
192
275
|
If set to ``None`` or missing, the default project_id from the GCP connection is used.
|
|
193
276
|
:param secret_id: Required. ID of the secret to find.
|
|
277
|
+
:param location: Optional. Location where secret is expected to be stored regionally. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
|
|
278
|
+
For more details : https://cloud.google.com/secret-manager/docs/locations
|
|
194
279
|
:return: True if the secret exists, False otherwise.
|
|
195
280
|
"""
|
|
196
281
|
secret_filter = f"name:{secret_id}"
|
|
197
|
-
secret_name = self.
|
|
198
|
-
for secret in self.list_secrets(
|
|
282
|
+
secret_name = self._get_secret_path(project_id, secret_id, location)
|
|
283
|
+
for secret in self.list_secrets(
|
|
284
|
+
project_id=project_id, page_size=100, secret_filter=secret_filter, location=location
|
|
285
|
+
):
|
|
199
286
|
if secret.name.split("/")[-1] == secret_id:
|
|
200
287
|
self.log.info("Secret %s exists.", secret_name)
|
|
201
288
|
return True
|
|
@@ -211,6 +298,7 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
|
|
|
211
298
|
retry: Retry | _MethodDefault = DEFAULT,
|
|
212
299
|
timeout: float | None = None,
|
|
213
300
|
metadata: Sequence[tuple[str, str]] = (),
|
|
301
|
+
location: str | None = None,
|
|
214
302
|
) -> AccessSecretVersionResponse:
|
|
215
303
|
"""
|
|
216
304
|
Access a secret version.
|
|
@@ -226,11 +314,13 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
|
|
|
226
314
|
:param retry: Optional. Designation of what errors, if any, should be retried.
|
|
227
315
|
:param timeout: Optional. The timeout for this request.
|
|
228
316
|
:param metadata: Optional. Strings which should be sent along with the request as metadata.
|
|
317
|
+
:param location: Optional. Location where secret is stored regionally. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
|
|
318
|
+
For more details : https://cloud.google.com/secret-manager/docs/locations
|
|
229
319
|
:return: Access secret version response object.
|
|
230
320
|
"""
|
|
231
321
|
response = self.client.access_secret_version(
|
|
232
322
|
request={
|
|
233
|
-
"name": self.
|
|
323
|
+
"name": self._get_secret_version_path(project_id, secret_id, secret_version, location),
|
|
234
324
|
},
|
|
235
325
|
retry=retry,
|
|
236
326
|
timeout=timeout,
|
|
@@ -247,6 +337,7 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
|
|
|
247
337
|
retry: Retry | _MethodDefault = DEFAULT,
|
|
248
338
|
timeout: float | None = None,
|
|
249
339
|
metadata: Sequence[tuple[str, str]] = (),
|
|
340
|
+
location: str | None = None,
|
|
250
341
|
) -> None:
|
|
251
342
|
"""
|
|
252
343
|
Delete a secret.
|
|
@@ -261,9 +352,11 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
|
|
|
261
352
|
:param retry: Optional. Designation of what errors, if any, should be retried.
|
|
262
353
|
:param timeout: Optional. The timeout for this request.
|
|
263
354
|
:param metadata: Optional. Strings which should be sent along with the request as metadata.
|
|
355
|
+
:param location: Optional. Location where secret is stored regionally. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used.
|
|
356
|
+
For more details : https://cloud.google.com/secret-manager/docs/locations
|
|
264
357
|
:return: Access secret version response object.
|
|
265
358
|
"""
|
|
266
|
-
name = self.
|
|
359
|
+
name = self._get_secret_path(project_id, secret_id, location)
|
|
267
360
|
self.client.delete_secret(
|
|
268
361
|
request={"name": name},
|
|
269
362
|
retry=retry,
|