apache-airflow-providers-google 12.0.0rc2__py3-none-any.whl → 13.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/LICENSE +0 -52
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/ads/hooks/ads.py +27 -13
- airflow/providers/google/ads/transfers/ads_to_gcs.py +18 -4
- airflow/providers/google/assets/bigquery.py +17 -0
- airflow/providers/google/cloud/_internal_client/secret_manager_client.py +2 -3
- airflow/providers/google/cloud/hooks/alloy_db.py +736 -8
- airflow/providers/google/cloud/hooks/automl.py +10 -4
- airflow/providers/google/cloud/hooks/bigquery.py +125 -22
- airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
- airflow/providers/google/cloud/hooks/bigtable.py +2 -3
- airflow/providers/google/cloud/hooks/cloud_batch.py +3 -4
- airflow/providers/google/cloud/hooks/cloud_build.py +4 -5
- airflow/providers/google/cloud/hooks/cloud_composer.py +3 -4
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +3 -4
- airflow/providers/google/cloud/hooks/cloud_run.py +3 -4
- airflow/providers/google/cloud/hooks/cloud_sql.py +7 -3
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +119 -7
- airflow/providers/google/cloud/hooks/compute.py +3 -3
- airflow/providers/google/cloud/hooks/datacatalog.py +3 -4
- airflow/providers/google/cloud/hooks/dataflow.py +12 -12
- airflow/providers/google/cloud/hooks/dataform.py +2 -3
- airflow/providers/google/cloud/hooks/datafusion.py +2 -2
- airflow/providers/google/cloud/hooks/dataplex.py +1032 -11
- airflow/providers/google/cloud/hooks/dataproc.py +4 -5
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +3 -4
- airflow/providers/google/cloud/hooks/dlp.py +3 -4
- airflow/providers/google/cloud/hooks/gcs.py +7 -6
- airflow/providers/google/cloud/hooks/kms.py +2 -3
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +8 -8
- airflow/providers/google/cloud/hooks/life_sciences.py +1 -1
- airflow/providers/google/cloud/hooks/managed_kafka.py +482 -0
- airflow/providers/google/cloud/hooks/natural_language.py +2 -3
- airflow/providers/google/cloud/hooks/os_login.py +2 -3
- airflow/providers/google/cloud/hooks/pubsub.py +6 -6
- airflow/providers/google/cloud/hooks/secret_manager.py +2 -3
- airflow/providers/google/cloud/hooks/spanner.py +2 -2
- airflow/providers/google/cloud/hooks/speech_to_text.py +2 -3
- airflow/providers/google/cloud/hooks/stackdriver.py +4 -4
- airflow/providers/google/cloud/hooks/tasks.py +3 -4
- airflow/providers/google/cloud/hooks/text_to_speech.py +2 -3
- airflow/providers/google/cloud/hooks/translate.py +236 -5
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +9 -4
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +3 -4
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +4 -5
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +3 -4
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +2 -3
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +3 -4
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +1 -181
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +3 -4
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +2 -3
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +3 -4
- airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +2 -3
- airflow/providers/google/cloud/hooks/video_intelligence.py +2 -3
- airflow/providers/google/cloud/hooks/vision.py +3 -4
- airflow/providers/google/cloud/hooks/workflows.py +2 -3
- airflow/providers/google/cloud/links/alloy_db.py +46 -0
- airflow/providers/google/cloud/links/bigquery.py +25 -0
- airflow/providers/google/cloud/links/dataplex.py +172 -2
- airflow/providers/google/cloud/links/kubernetes_engine.py +1 -2
- airflow/providers/google/cloud/links/managed_kafka.py +104 -0
- airflow/providers/google/cloud/links/translate.py +28 -0
- airflow/providers/google/cloud/log/gcs_task_handler.py +3 -3
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +11 -10
- airflow/providers/google/cloud/openlineage/facets.py +67 -0
- airflow/providers/google/cloud/openlineage/mixins.py +438 -173
- airflow/providers/google/cloud/openlineage/utils.py +394 -61
- airflow/providers/google/cloud/operators/alloy_db.py +980 -69
- airflow/providers/google/cloud/operators/automl.py +83 -245
- airflow/providers/google/cloud/operators/bigquery.py +377 -74
- airflow/providers/google/cloud/operators/bigquery_dts.py +126 -13
- airflow/providers/google/cloud/operators/bigtable.py +1 -3
- airflow/providers/google/cloud/operators/cloud_base.py +1 -2
- airflow/providers/google/cloud/operators/cloud_batch.py +2 -4
- airflow/providers/google/cloud/operators/cloud_build.py +3 -5
- airflow/providers/google/cloud/operators/cloud_composer.py +5 -7
- airflow/providers/google/cloud/operators/cloud_memorystore.py +4 -6
- airflow/providers/google/cloud/operators/cloud_run.py +6 -5
- airflow/providers/google/cloud/operators/cloud_sql.py +20 -8
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +62 -8
- airflow/providers/google/cloud/operators/compute.py +3 -4
- airflow/providers/google/cloud/operators/datacatalog.py +9 -11
- airflow/providers/google/cloud/operators/dataflow.py +1 -112
- airflow/providers/google/cloud/operators/dataform.py +3 -5
- airflow/providers/google/cloud/operators/datafusion.py +1 -1
- airflow/providers/google/cloud/operators/dataplex.py +2046 -7
- airflow/providers/google/cloud/operators/dataproc.py +102 -17
- airflow/providers/google/cloud/operators/dataproc_metastore.py +7 -9
- airflow/providers/google/cloud/operators/dlp.py +17 -19
- airflow/providers/google/cloud/operators/gcs.py +14 -17
- airflow/providers/google/cloud/operators/kubernetes_engine.py +2 -2
- airflow/providers/google/cloud/operators/managed_kafka.py +788 -0
- airflow/providers/google/cloud/operators/natural_language.py +3 -5
- airflow/providers/google/cloud/operators/pubsub.py +39 -7
- airflow/providers/google/cloud/operators/speech_to_text.py +3 -5
- airflow/providers/google/cloud/operators/stackdriver.py +3 -5
- airflow/providers/google/cloud/operators/tasks.py +4 -6
- airflow/providers/google/cloud/operators/text_to_speech.py +2 -4
- airflow/providers/google/cloud/operators/translate.py +414 -5
- airflow/providers/google/cloud/operators/translate_speech.py +2 -4
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +9 -8
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +4 -6
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +6 -8
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +4 -6
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +4 -6
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +0 -322
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +4 -6
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +4 -6
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +4 -6
- airflow/providers/google/cloud/operators/video_intelligence.py +3 -5
- airflow/providers/google/cloud/operators/vision.py +4 -6
- airflow/providers/google/cloud/operators/workflows.py +5 -7
- airflow/providers/google/cloud/secrets/secret_manager.py +1 -2
- airflow/providers/google/cloud/sensors/bigquery_dts.py +3 -5
- airflow/providers/google/cloud/sensors/bigtable.py +2 -3
- airflow/providers/google/cloud/sensors/cloud_composer.py +32 -8
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +39 -1
- airflow/providers/google/cloud/sensors/dataplex.py +4 -6
- airflow/providers/google/cloud/sensors/dataproc.py +2 -3
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +1 -2
- airflow/providers/google/cloud/sensors/gcs.py +2 -4
- airflow/providers/google/cloud/sensors/pubsub.py +2 -3
- airflow/providers/google/cloud/sensors/workflows.py +3 -5
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +5 -5
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +10 -12
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +36 -4
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +27 -2
- airflow/providers/google/cloud/transfers/mysql_to_gcs.py +27 -2
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +27 -2
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +34 -5
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +15 -0
- airflow/providers/google/cloud/transfers/trino_to_gcs.py +25 -2
- airflow/providers/google/cloud/triggers/bigquery_dts.py +1 -2
- airflow/providers/google/cloud/triggers/cloud_batch.py +1 -2
- airflow/providers/google/cloud/triggers/cloud_build.py +1 -2
- airflow/providers/google/cloud/triggers/cloud_composer.py +13 -3
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +102 -4
- airflow/providers/google/cloud/triggers/dataflow.py +2 -3
- airflow/providers/google/cloud/triggers/dataplex.py +1 -2
- airflow/providers/google/cloud/triggers/dataproc.py +2 -3
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +1 -1
- airflow/providers/google/cloud/triggers/pubsub.py +1 -2
- airflow/providers/google/cloud/triggers/vertex_ai.py +7 -8
- airflow/providers/google/cloud/utils/credentials_provider.py +15 -8
- airflow/providers/google/cloud/utils/external_token_supplier.py +1 -0
- airflow/providers/google/common/auth_backend/google_openid.py +4 -4
- airflow/providers/google/common/consts.py +1 -2
- airflow/providers/google/common/hooks/base_google.py +8 -7
- airflow/providers/google/get_provider_info.py +186 -134
- airflow/providers/google/marketing_platform/hooks/analytics_admin.py +2 -3
- airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +5 -7
- {apache_airflow_providers_google-12.0.0rc2.dist-info → apache_airflow_providers_google-13.0.0.dist-info}/METADATA +41 -58
- {apache_airflow_providers_google-12.0.0rc2.dist-info → apache_airflow_providers_google-13.0.0.dist-info}/RECORD +157 -159
- airflow/providers/google/cloud/example_dags/example_facebook_ads_to_gcs.py +0 -141
- airflow/providers/google/cloud/example_dags/example_looker.py +0 -64
- airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py +0 -194
- airflow/providers/google/cloud/example_dags/example_salesforce_to_gcs.py +0 -129
- airflow/providers/google/marketing_platform/example_dags/__init__.py +0 -16
- airflow/providers/google/marketing_platform/example_dags/example_display_video.py +0 -213
- {apache_airflow_providers_google-12.0.0rc2.dist-info → apache_airflow_providers_google-13.0.0.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-12.0.0rc2.dist-info → apache_airflow_providers_google-13.0.0.dist-info}/entry_points.txt +0 -0
@@ -22,18 +22,16 @@ from __future__ import annotations
|
|
22
22
|
from collections.abc import Sequence
|
23
23
|
from typing import TYPE_CHECKING
|
24
24
|
|
25
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
26
|
-
from google.protobuf.json_format import MessageToDict
|
27
|
-
|
28
25
|
from airflow.providers.google.cloud.hooks.natural_language import CloudNaturalLanguageHook
|
29
26
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
27
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
28
|
+
from google.protobuf.json_format import MessageToDict
|
30
29
|
|
31
30
|
if TYPE_CHECKING:
|
31
|
+
from airflow.utils.context import Context
|
32
32
|
from google.api_core.retry import Retry
|
33
33
|
from google.cloud.language_v1.types import Document, EncodingType
|
34
34
|
|
35
|
-
from airflow.utils.context import Context
|
36
|
-
|
37
35
|
|
38
36
|
MetaData = Sequence[tuple[str, str]]
|
39
37
|
|
@@ -28,6 +28,13 @@ from __future__ import annotations
|
|
28
28
|
from collections.abc import Sequence
|
29
29
|
from typing import TYPE_CHECKING, Any, Callable
|
30
30
|
|
31
|
+
from airflow.exceptions import AirflowException
|
32
|
+
from airflow.providers.google.cloud.hooks.pubsub import PubSubHook
|
33
|
+
from airflow.providers.google.cloud.links.pubsub import PubSubSubscriptionLink, PubSubTopicLink
|
34
|
+
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
35
|
+
from airflow.providers.google.cloud.triggers.pubsub import PubsubPullTrigger
|
36
|
+
from airflow.providers.google.common.consts import GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME
|
37
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
31
38
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
32
39
|
from google.cloud.pubsub_v1.types import (
|
33
40
|
DeadLetterPolicy,
|
@@ -40,15 +47,9 @@ from google.cloud.pubsub_v1.types import (
|
|
40
47
|
SchemaSettings,
|
41
48
|
)
|
42
49
|
|
43
|
-
from airflow.providers.google.cloud.hooks.pubsub import PubSubHook
|
44
|
-
from airflow.providers.google.cloud.links.pubsub import PubSubSubscriptionLink, PubSubTopicLink
|
45
|
-
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
46
|
-
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
47
|
-
|
48
50
|
if TYPE_CHECKING:
|
49
|
-
from google.api_core.retry import Retry
|
50
|
-
|
51
51
|
from airflow.utils.context import Context
|
52
|
+
from google.api_core.retry import Retry
|
52
53
|
|
53
54
|
|
54
55
|
class PubSubCreateTopicOperator(GoogleCloudBaseOperator):
|
@@ -746,6 +747,9 @@ class PubSubPullOperator(GoogleCloudBaseOperator):
|
|
746
747
|
If set as a sequence, the identities from the list must grant
|
747
748
|
Service Account Token Creator IAM role to the directly preceding identity, with first
|
748
749
|
account from the list granting this role to the originating account (templated).
|
750
|
+
:param deferrable: If True, run the task in the deferrable mode.
|
751
|
+
:param poll_interval: Time (seconds) to wait between two consecutive calls to check the job.
|
752
|
+
The default is 300 seconds.
|
749
753
|
"""
|
750
754
|
|
751
755
|
template_fields: Sequence[str] = (
|
@@ -764,6 +768,8 @@ class PubSubPullOperator(GoogleCloudBaseOperator):
|
|
764
768
|
messages_callback: Callable[[list[ReceivedMessage], Context], Any] | None = None,
|
765
769
|
gcp_conn_id: str = "google_cloud_default",
|
766
770
|
impersonation_chain: str | Sequence[str] | None = None,
|
771
|
+
deferrable: bool = False,
|
772
|
+
poll_interval: int = 300,
|
767
773
|
**kwargs,
|
768
774
|
) -> None:
|
769
775
|
super().__init__(**kwargs)
|
@@ -774,8 +780,23 @@ class PubSubPullOperator(GoogleCloudBaseOperator):
|
|
774
780
|
self.ack_messages = ack_messages
|
775
781
|
self.messages_callback = messages_callback
|
776
782
|
self.impersonation_chain = impersonation_chain
|
783
|
+
self.deferrable = deferrable
|
784
|
+
self.poll_interval = poll_interval
|
777
785
|
|
778
786
|
def execute(self, context: Context) -> list:
|
787
|
+
if self.deferrable:
|
788
|
+
self.defer(
|
789
|
+
trigger=PubsubPullTrigger(
|
790
|
+
subscription=self.subscription,
|
791
|
+
project_id=self.project_id,
|
792
|
+
max_messages=self.max_messages,
|
793
|
+
ack_messages=self.ack_messages,
|
794
|
+
gcp_conn_id=self.gcp_conn_id,
|
795
|
+
poke_interval=self.poll_interval,
|
796
|
+
impersonation_chain=self.impersonation_chain,
|
797
|
+
),
|
798
|
+
method_name=GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME,
|
799
|
+
)
|
779
800
|
hook = PubSubHook(
|
780
801
|
gcp_conn_id=self.gcp_conn_id,
|
781
802
|
impersonation_chain=self.impersonation_chain,
|
@@ -801,6 +822,17 @@ class PubSubPullOperator(GoogleCloudBaseOperator):
|
|
801
822
|
|
802
823
|
return ret
|
803
824
|
|
825
|
+
def execute_complete(self, context: Context, event: dict[str, Any]):
|
826
|
+
"""If messages_callback is provided, execute it; otherwise, return immediately with trigger event message."""
|
827
|
+
if event["status"] == "success":
|
828
|
+
self.log.info("Sensor pulls messages: %s", event["message"])
|
829
|
+
messages_callback = self.messages_callback or self._default_message_callback
|
830
|
+
_return_value = messages_callback(event["message"], context)
|
831
|
+
return _return_value
|
832
|
+
|
833
|
+
self.log.info("Sensor failed: %s", event["message"])
|
834
|
+
raise AirflowException(event["message"])
|
835
|
+
|
804
836
|
def _default_message_callback(
|
805
837
|
self,
|
806
838
|
pulled_messages: list[ReceivedMessage],
|
@@ -22,21 +22,19 @@ from __future__ import annotations
|
|
22
22
|
from collections.abc import Sequence
|
23
23
|
from typing import TYPE_CHECKING
|
24
24
|
|
25
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
26
|
-
from google.protobuf.json_format import MessageToDict
|
27
|
-
|
28
25
|
from airflow.exceptions import AirflowException
|
29
26
|
from airflow.providers.google.cloud.hooks.speech_to_text import CloudSpeechToTextHook, RecognitionAudio
|
30
27
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
31
28
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
32
29
|
from airflow.providers.google.common.links.storage import FileDetailsLink
|
30
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
31
|
+
from google.protobuf.json_format import MessageToDict
|
33
32
|
|
34
33
|
if TYPE_CHECKING:
|
34
|
+
from airflow.utils.context import Context
|
35
35
|
from google.api_core.retry import Retry
|
36
36
|
from google.cloud.speech_v1.types import RecognitionConfig
|
37
37
|
|
38
|
-
from airflow.utils.context import Context
|
39
|
-
|
40
38
|
|
41
39
|
class CloudSpeechToTextRecognizeSpeechOperator(GoogleCloudBaseOperator):
|
42
40
|
"""
|
@@ -20,9 +20,6 @@ from __future__ import annotations
|
|
20
20
|
from collections.abc import Sequence
|
21
21
|
from typing import TYPE_CHECKING
|
22
22
|
|
23
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
24
|
-
from google.cloud.monitoring_v3 import AlertPolicy, NotificationChannel
|
25
|
-
|
26
23
|
from airflow.providers.google.cloud.hooks.stackdriver import StackdriverHook
|
27
24
|
from airflow.providers.google.cloud.links.stackdriver import (
|
28
25
|
StackdriverNotificationsLink,
|
@@ -30,11 +27,12 @@ from airflow.providers.google.cloud.links.stackdriver import (
|
|
30
27
|
)
|
31
28
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
32
29
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
30
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
31
|
+
from google.cloud.monitoring_v3 import AlertPolicy, NotificationChannel
|
33
32
|
|
34
33
|
if TYPE_CHECKING:
|
35
|
-
from google.api_core.retry import Retry
|
36
|
-
|
37
34
|
from airflow.utils.context import Context
|
35
|
+
from google.api_core.retry import Retry
|
38
36
|
|
39
37
|
|
40
38
|
class StackdriverListAlertPoliciesOperator(GoogleCloudBaseOperator):
|
@@ -22,21 +22,19 @@ from __future__ import annotations
|
|
22
22
|
from collections.abc import Sequence
|
23
23
|
from typing import TYPE_CHECKING
|
24
24
|
|
25
|
-
from google.api_core.exceptions import AlreadyExists
|
26
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
27
|
-
from google.cloud.tasks_v2.types import Queue, Task
|
28
|
-
|
29
25
|
from airflow.providers.google.cloud.hooks.tasks import CloudTasksHook
|
30
26
|
from airflow.providers.google.cloud.links.cloud_tasks import CloudTasksLink, CloudTasksQueueLink
|
31
27
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
32
28
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
29
|
+
from google.api_core.exceptions import AlreadyExists
|
30
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
31
|
+
from google.cloud.tasks_v2.types import Queue, Task
|
33
32
|
|
34
33
|
if TYPE_CHECKING:
|
34
|
+
from airflow.utils.context import Context
|
35
35
|
from google.api_core.retry import Retry
|
36
36
|
from google.protobuf.field_mask_pb2 import FieldMask
|
37
37
|
|
38
|
-
from airflow.utils.context import Context
|
39
|
-
|
40
38
|
|
41
39
|
MetaData = Sequence[tuple[str, str]]
|
42
40
|
|
@@ -23,21 +23,19 @@ from collections.abc import Sequence
|
|
23
23
|
from tempfile import NamedTemporaryFile
|
24
24
|
from typing import TYPE_CHECKING
|
25
25
|
|
26
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
27
|
-
|
28
26
|
from airflow.exceptions import AirflowException
|
29
27
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
30
28
|
from airflow.providers.google.cloud.hooks.text_to_speech import CloudTextToSpeechHook
|
31
29
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
32
30
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
33
31
|
from airflow.providers.google.common.links.storage import FileDetailsLink
|
32
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
34
33
|
|
35
34
|
if TYPE_CHECKING:
|
35
|
+
from airflow.utils.context import Context
|
36
36
|
from google.api_core.retry import Retry
|
37
37
|
from google.cloud.texttospeech_v1.types import AudioConfig, SynthesisInput, VoiceSelectionParams
|
38
38
|
|
39
|
-
from airflow.utils.context import Context
|
40
|
-
|
41
39
|
|
42
40
|
class CloudTextToSpeechSynthesizeOperator(GoogleCloudBaseOperator):
|
43
41
|
"""
|
@@ -22,23 +22,24 @@ from __future__ import annotations
|
|
22
22
|
from collections.abc import MutableMapping, MutableSequence, Sequence
|
23
23
|
from typing import TYPE_CHECKING, cast
|
24
24
|
|
25
|
-
from google.api_core.exceptions import GoogleAPICallError
|
26
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
27
|
-
|
28
25
|
from airflow.exceptions import AirflowException
|
29
26
|
from airflow.providers.google.cloud.hooks.translate import CloudTranslateHook, TranslateHook
|
30
27
|
from airflow.providers.google.cloud.links.translate import (
|
31
28
|
TranslateResultByOutputConfigLink,
|
32
29
|
TranslateTextBatchLink,
|
33
30
|
TranslationDatasetsListLink,
|
31
|
+
TranslationGlossariesListLink,
|
34
32
|
TranslationModelLink,
|
35
33
|
TranslationModelsListLink,
|
36
34
|
TranslationNativeDatasetLink,
|
37
35
|
)
|
38
36
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
39
37
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
38
|
+
from google.api_core.exceptions import GoogleAPICallError
|
39
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
40
40
|
|
41
41
|
if TYPE_CHECKING:
|
42
|
+
from airflow.utils.context import Context
|
42
43
|
from google.api_core.retry import Retry
|
43
44
|
from google.cloud.translate_v3.types import (
|
44
45
|
BatchDocumentInputConfig,
|
@@ -52,8 +53,7 @@ if TYPE_CHECKING:
|
|
52
53
|
TransliterationConfig,
|
53
54
|
automl_translation,
|
54
55
|
)
|
55
|
-
|
56
|
-
from airflow.utils.context import Context
|
56
|
+
from google.cloud.translate_v3.types.translation_service import Glossary, GlossaryInputConfig
|
57
57
|
|
58
58
|
|
59
59
|
class CloudTranslateTextOperator(GoogleCloudBaseOperator):
|
@@ -158,6 +158,7 @@ class TranslateTextOperator(GoogleCloudBaseOperator):
|
|
158
158
|
|
159
159
|
:param project_id: Optional. The ID of the Google Cloud project that the
|
160
160
|
service belongs to.
|
161
|
+
If not provided default project_id is used.
|
161
162
|
:param location: optional. The ID of the Google Cloud location that the
|
162
163
|
service belongs to. if not specified, 'global' is used.
|
163
164
|
Non-global location is required for requests using AutoML models or custom glossaries.
|
@@ -1308,3 +1309,411 @@ class TranslateDocumentBatchOperator(GoogleCloudBaseOperator):
|
|
1308
1309
|
result = hook.wait_for_operation_result(batch_document_translate_operation)
|
1309
1310
|
self.log.info("Batch document translation job finished")
|
1310
1311
|
return cast(dict, type(result).to_dict(result))
|
1312
|
+
|
1313
|
+
|
1314
|
+
class TranslateCreateGlossaryOperator(GoogleCloudBaseOperator):
|
1315
|
+
"""
|
1316
|
+
Creates a Google Cloud Translation Glossary.
|
1317
|
+
|
1318
|
+
Creates a translation glossary, using API V3.
|
1319
|
+
For more information on how to use this operator, take a look at the guide:
|
1320
|
+
:ref:`howto/operator:TranslateCreateGlossaryOperator`.
|
1321
|
+
|
1322
|
+
:param glossary_id: User-specified id to built glossary resource name.
|
1323
|
+
:param input_config: The input configuration of examples to built glossary from.
|
1324
|
+
Total glossary must not exceed 10M Unicode codepoints.
|
1325
|
+
The headers should not be included into the input file table, as languages specified with the
|
1326
|
+
``language_pair`` or ``language_codes_set`` params.
|
1327
|
+
:param language_pair: Pair of language codes to be used for glossary creation.
|
1328
|
+
Used to built unidirectional glossary. If specified, the ``language_codes_set`` should be empty.
|
1329
|
+
:param language_codes_set: Set of language codes to create the equivalent term sets glossary.
|
1330
|
+
Meant multiple languages mapping. If specified, the ``language_pair`` should be empty.
|
1331
|
+
:param project_id: ID of the Google Cloud project where glossary is located.
|
1332
|
+
If not provided default project_id is used.
|
1333
|
+
:param location: The location of the project.
|
1334
|
+
:param retry: Designation of what errors, if any, should be retried.
|
1335
|
+
:param timeout: The timeout for this request.
|
1336
|
+
:param metadata: Strings which should be sent along with the request as metadata.
|
1337
|
+
:param gcp_conn_id: The connection ID to use connecting to Google Cloud.
|
1338
|
+
:param impersonation_chain: Optional service account to impersonate using short-term
|
1339
|
+
credentials, or chained list of accounts required to get the access_token
|
1340
|
+
of the last account in the list, which will be impersonated in the request.
|
1341
|
+
If set as a string, the account must grant the originating account
|
1342
|
+
the Service Account Token Creator IAM role.
|
1343
|
+
If set as a sequence, the identities from the list must grant
|
1344
|
+
Service Account Token Creator IAM role to the directly preceding identity, with first
|
1345
|
+
account from the list granting this role to the originating account (templated).
|
1346
|
+
"""
|
1347
|
+
|
1348
|
+
template_fields: Sequence[str] = (
|
1349
|
+
"glossary_id",
|
1350
|
+
"location",
|
1351
|
+
"project_id",
|
1352
|
+
"gcp_conn_id",
|
1353
|
+
"impersonation_chain",
|
1354
|
+
)
|
1355
|
+
|
1356
|
+
def __init__(
|
1357
|
+
self,
|
1358
|
+
*,
|
1359
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1360
|
+
location: str,
|
1361
|
+
glossary_id: str,
|
1362
|
+
input_config: GlossaryInputConfig | dict,
|
1363
|
+
language_pair: Glossary.LanguageCodePair | dict | None = None,
|
1364
|
+
language_codes_set: Glossary.LanguageCodesSet | MutableSequence[str] | None = None,
|
1365
|
+
timeout: float | None = None,
|
1366
|
+
retry: Retry | _MethodDefault = DEFAULT,
|
1367
|
+
gcp_conn_id: str = "google_cloud_default",
|
1368
|
+
metadata: Sequence[tuple[str, str]] = (),
|
1369
|
+
impersonation_chain: str | Sequence[str] | None = None,
|
1370
|
+
**kwargs,
|
1371
|
+
) -> None:
|
1372
|
+
super().__init__(**kwargs)
|
1373
|
+
self.project_id = project_id
|
1374
|
+
self.location = location
|
1375
|
+
self.glossary_id = glossary_id
|
1376
|
+
self.input_config = input_config
|
1377
|
+
self.language_pair = language_pair
|
1378
|
+
self.language_codes_set = language_codes_set
|
1379
|
+
self.metadata = metadata
|
1380
|
+
self.timeout = timeout
|
1381
|
+
self.retry = retry
|
1382
|
+
self.gcp_conn_id = gcp_conn_id
|
1383
|
+
self.impersonation_chain = impersonation_chain
|
1384
|
+
|
1385
|
+
def execute(self, context: Context) -> str:
|
1386
|
+
hook = TranslateHook(
|
1387
|
+
gcp_conn_id=self.gcp_conn_id,
|
1388
|
+
impersonation_chain=self.impersonation_chain,
|
1389
|
+
)
|
1390
|
+
project_id = self.project_id or hook.project_id
|
1391
|
+
try:
|
1392
|
+
result_operation = hook.create_glossary(
|
1393
|
+
glossary_id=self.glossary_id,
|
1394
|
+
input_config=self.input_config,
|
1395
|
+
language_pair=self.language_pair,
|
1396
|
+
language_codes_set=self.language_codes_set,
|
1397
|
+
project_id=project_id,
|
1398
|
+
location=self.location,
|
1399
|
+
retry=self.retry,
|
1400
|
+
timeout=self.timeout,
|
1401
|
+
metadata=self.metadata,
|
1402
|
+
)
|
1403
|
+
except GoogleAPICallError as e:
|
1404
|
+
self.log.error("Error submitting create_glossary operation ")
|
1405
|
+
raise AirflowException(e)
|
1406
|
+
self.log.info("Glossary creation started, glossary_id %s...", self.glossary_id)
|
1407
|
+
|
1408
|
+
result = hook.wait_for_operation_result(operation=result_operation)
|
1409
|
+
result = type(result).to_dict(result)
|
1410
|
+
|
1411
|
+
glossary_id = hook.extract_object_id(result)
|
1412
|
+
self.xcom_push(context, key="glossary_id", value=glossary_id)
|
1413
|
+
self.log.info("Glossary creation complete. The glossary_id: %s.", glossary_id)
|
1414
|
+
return result
|
1415
|
+
|
1416
|
+
|
1417
|
+
class TranslateUpdateGlossaryOperator(GoogleCloudBaseOperator):
|
1418
|
+
"""
|
1419
|
+
Update glossary item with values provided.
|
1420
|
+
|
1421
|
+
Updates the translation glossary, using translation API V3.
|
1422
|
+
Only ``display_name`` and ``input_config`` fields are allowed for update.
|
1423
|
+
|
1424
|
+
For more information on how to use this operator, take a look at the guide:
|
1425
|
+
:ref:`howto/operator:TranslateUpdateGlossaryOperator`.
|
1426
|
+
|
1427
|
+
:param glossary_id: User-specified id to built glossary resource name.
|
1428
|
+
:param input_config: The input configuration of examples to built glossary from.
|
1429
|
+
Total glossary must not exceed 10M Unicode codepoints.
|
1430
|
+
The headers should not be included into the input file table, as languages specified with the
|
1431
|
+
``language_pair`` or ``language_codes_set`` params.
|
1432
|
+
:param language_pair: Pair of language codes to be used for glossary creation.
|
1433
|
+
Used to built unidirectional glossary. If specified, the ``language_codes_set`` should be empty.
|
1434
|
+
:param language_codes_set: Set of language codes to create the equivalent term sets glossary.
|
1435
|
+
Meant multiple languages mapping. If specified, the ``language_pair`` should be empty.
|
1436
|
+
:param project_id: ID of the Google Cloud project where glossary is located.
|
1437
|
+
If not provided default project_id is used.
|
1438
|
+
:param location: The location of the project.
|
1439
|
+
:param retry: Designation of what errors, if any, should be retried.
|
1440
|
+
:param timeout: The timeout for this request.
|
1441
|
+
:param metadata: Strings which should be sent along with the request as metadata.
|
1442
|
+
:param gcp_conn_id: The connection ID to use connecting to Google Cloud.
|
1443
|
+
:param impersonation_chain: Optional service account to impersonate using short-term
|
1444
|
+
credentials, or chained list of accounts required to get the access_token
|
1445
|
+
of the last account in the list, which will be impersonated in the request.
|
1446
|
+
If set as a string, the account must grant the originating account
|
1447
|
+
the Service Account Token Creator IAM role.
|
1448
|
+
If set as a sequence, the identities from the list must grant
|
1449
|
+
Service Account Token Creator IAM role to the directly preceding identity, with first
|
1450
|
+
account from the list granting this role to the originating account (templated).
|
1451
|
+
"""
|
1452
|
+
|
1453
|
+
template_fields: Sequence[str] = (
|
1454
|
+
"glossary_id",
|
1455
|
+
"location",
|
1456
|
+
"project_id",
|
1457
|
+
"gcp_conn_id",
|
1458
|
+
"impersonation_chain",
|
1459
|
+
)
|
1460
|
+
|
1461
|
+
def __init__(
|
1462
|
+
self,
|
1463
|
+
*,
|
1464
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1465
|
+
location: str,
|
1466
|
+
glossary_id: str,
|
1467
|
+
new_display_name: str,
|
1468
|
+
new_input_config: GlossaryInputConfig | dict | None = None,
|
1469
|
+
timeout: float | None = None,
|
1470
|
+
retry: Retry | _MethodDefault = DEFAULT,
|
1471
|
+
gcp_conn_id: str = "google_cloud_default",
|
1472
|
+
metadata: Sequence[tuple[str, str]] = (),
|
1473
|
+
impersonation_chain: str | Sequence[str] | None = None,
|
1474
|
+
**kwargs,
|
1475
|
+
) -> None:
|
1476
|
+
super().__init__(**kwargs)
|
1477
|
+
self.project_id = project_id
|
1478
|
+
self.location = location
|
1479
|
+
self.glossary_id = glossary_id
|
1480
|
+
self.new_display_name = new_display_name
|
1481
|
+
self.new_input_config = new_input_config
|
1482
|
+
self.metadata = metadata
|
1483
|
+
self.timeout = timeout
|
1484
|
+
self.retry = retry
|
1485
|
+
self.gcp_conn_id = gcp_conn_id
|
1486
|
+
self.impersonation_chain = impersonation_chain
|
1487
|
+
|
1488
|
+
def execute(self, context: Context) -> str:
|
1489
|
+
hook = TranslateHook(
|
1490
|
+
gcp_conn_id=self.gcp_conn_id,
|
1491
|
+
impersonation_chain=self.impersonation_chain,
|
1492
|
+
)
|
1493
|
+
project_id = self.project_id or hook.project_id
|
1494
|
+
glossary = hook.get_glossary(
|
1495
|
+
glossary_id=self.glossary_id,
|
1496
|
+
project_id=project_id,
|
1497
|
+
location=self.location,
|
1498
|
+
retry=self.retry,
|
1499
|
+
timeout=self.timeout,
|
1500
|
+
metadata=self.metadata,
|
1501
|
+
)
|
1502
|
+
try:
|
1503
|
+
result_operation = hook.update_glossary(
|
1504
|
+
glossary=glossary,
|
1505
|
+
new_display_name=self.new_display_name,
|
1506
|
+
new_input_config=self.new_input_config,
|
1507
|
+
retry=self.retry,
|
1508
|
+
timeout=self.timeout,
|
1509
|
+
metadata=self.metadata,
|
1510
|
+
)
|
1511
|
+
except GoogleAPICallError as e:
|
1512
|
+
self.log.error("Error submitting update_glossary operation ")
|
1513
|
+
raise AirflowException(e)
|
1514
|
+
self.log.info("Glossary update started, glossary_id %s...", self.glossary_id)
|
1515
|
+
|
1516
|
+
result = hook.wait_for_operation_result(operation=result_operation)
|
1517
|
+
result = type(result).to_dict(result)
|
1518
|
+
self.log.info("Glossary update complete. The glossary_id: %s.", self.glossary_id)
|
1519
|
+
return result
|
1520
|
+
|
1521
|
+
|
1522
|
+
class TranslateListGlossariesOperator(GoogleCloudBaseOperator):
|
1523
|
+
"""
|
1524
|
+
Get a list of translation glossaries in a project.
|
1525
|
+
|
1526
|
+
List the translation glossaries, using translation API V3.
|
1527
|
+
|
1528
|
+
For more information on how to use this operator, take a look at the guide:
|
1529
|
+
:ref:`howto/operator:TranslateListGlossariesOperator`.
|
1530
|
+
|
1531
|
+
:param project_id: ID of the Google Cloud project where glossary is located.
|
1532
|
+
If not provided default project_id is used.
|
1533
|
+
:param page_size: Page size requested, if not set server use appropriate default.
|
1534
|
+
:param page_token: A token identifying a page of results the server should return.
|
1535
|
+
The first page is returned if ``page_token`` is empty or missing.
|
1536
|
+
:param filter_str: Filter specifying constraints of a list operation. Specify the constraint by the
|
1537
|
+
format of "key=value", where key must be ``src`` or ``tgt``, and the value must be a valid
|
1538
|
+
language code.
|
1539
|
+
For multiple restrictions, concatenate them by "AND" (uppercase only), such as:
|
1540
|
+
``src=en-US AND tgt=zh-CN``. Notice that the exact match is used here, which means using 'en-US'
|
1541
|
+
and 'en' can lead to different results, which depends on the language code you used when you
|
1542
|
+
create the glossary.
|
1543
|
+
For the unidirectional glossaries, the ``src`` and ``tgt`` add restrictions
|
1544
|
+
on the source and target language code separately.
|
1545
|
+
For the equivalent term set glossaries, the ``src`` and/or ``tgt``
|
1546
|
+
add restrictions on the term set.
|
1547
|
+
For example: ``src=en-US AND tgt=zh-CN`` will only pick the unidirectional glossaries which
|
1548
|
+
exactly match the source language code as ``en-US`` and the target language code ``zh-CN``,
|
1549
|
+
but all equivalent term set glossaries which contain ``en-US`` and ``zh-CN`` in their language
|
1550
|
+
set will be picked.
|
1551
|
+
If missing, no filtering is performed.
|
1552
|
+
:param location: The location of the project.
|
1553
|
+
:param retry: Designation of what errors, if any, should be retried.
|
1554
|
+
:param timeout: The timeout for this request.
|
1555
|
+
:param metadata: Strings which should be sent along with the request as metadata.
|
1556
|
+
:param gcp_conn_id: The connection ID to use connecting to Google Cloud.
|
1557
|
+
:param impersonation_chain: Optional service account to impersonate using short-term
|
1558
|
+
credentials, or chained list of accounts required to get the access_token
|
1559
|
+
of the last account in the list, which will be impersonated in the request.
|
1560
|
+
If set as a string, the account must grant the originating account
|
1561
|
+
the Service Account Token Creator IAM role.
|
1562
|
+
If set as a sequence, the identities from the list must grant
|
1563
|
+
Service Account Token Creator IAM role to the directly preceding identity, with first
|
1564
|
+
account from the list granting this role to the originating account (templated).
|
1565
|
+
"""
|
1566
|
+
|
1567
|
+
operator_extra_links = (TranslationGlossariesListLink(),)
|
1568
|
+
|
1569
|
+
template_fields: Sequence[str] = (
|
1570
|
+
"location",
|
1571
|
+
"project_id",
|
1572
|
+
"gcp_conn_id",
|
1573
|
+
"impersonation_chain",
|
1574
|
+
)
|
1575
|
+
|
1576
|
+
def __init__(
|
1577
|
+
self,
|
1578
|
+
*,
|
1579
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1580
|
+
location: str,
|
1581
|
+
page_size: int | None = None,
|
1582
|
+
page_token: str | None = None,
|
1583
|
+
filter_str: str | None = None,
|
1584
|
+
timeout: float | None = None,
|
1585
|
+
retry: Retry | _MethodDefault = DEFAULT,
|
1586
|
+
gcp_conn_id: str = "google_cloud_default",
|
1587
|
+
metadata: Sequence[tuple[str, str]] = (),
|
1588
|
+
impersonation_chain: str | Sequence[str] | None = None,
|
1589
|
+
**kwargs,
|
1590
|
+
) -> None:
|
1591
|
+
super().__init__(**kwargs)
|
1592
|
+
self.project_id = project_id
|
1593
|
+
self.location = location
|
1594
|
+
self.page_size = page_size
|
1595
|
+
self.page_token = page_token
|
1596
|
+
self.filter_str = filter_str
|
1597
|
+
self.metadata = metadata
|
1598
|
+
self.timeout = timeout
|
1599
|
+
self.retry = retry
|
1600
|
+
self.gcp_conn_id = gcp_conn_id
|
1601
|
+
self.impersonation_chain = impersonation_chain
|
1602
|
+
|
1603
|
+
def execute(self, context: Context) -> Sequence[str]:
|
1604
|
+
hook = TranslateHook(
|
1605
|
+
gcp_conn_id=self.gcp_conn_id,
|
1606
|
+
impersonation_chain=self.impersonation_chain,
|
1607
|
+
)
|
1608
|
+
project_id = self.project_id or hook.project_id
|
1609
|
+
TranslationGlossariesListLink.persist(
|
1610
|
+
context=context,
|
1611
|
+
task_instance=self,
|
1612
|
+
project_id=project_id,
|
1613
|
+
)
|
1614
|
+
self.log.info("Requesting glossaries list")
|
1615
|
+
try:
|
1616
|
+
results_pager = hook.list_glossaries(
|
1617
|
+
project_id=project_id,
|
1618
|
+
location=self.location,
|
1619
|
+
page_size=self.page_size,
|
1620
|
+
page_token=self.page_token,
|
1621
|
+
filter_str=self.filter_str,
|
1622
|
+
retry=self.retry,
|
1623
|
+
timeout=self.timeout,
|
1624
|
+
metadata=self.metadata,
|
1625
|
+
)
|
1626
|
+
except GoogleAPICallError as e:
|
1627
|
+
self.log.error("Error submitting list_glossaries request")
|
1628
|
+
raise AirflowException(e)
|
1629
|
+
|
1630
|
+
result_ids = []
|
1631
|
+
for glossary_item in results_pager:
|
1632
|
+
glossary_item = type(glossary_item).to_dict(glossary_item)
|
1633
|
+
glossary_id = hook.extract_object_id(glossary_item)
|
1634
|
+
result_ids.append(glossary_id)
|
1635
|
+
self.log.info("Fetching the glossaries list complete. Glossary id-s: %s", result_ids)
|
1636
|
+
return result_ids
|
1637
|
+
|
1638
|
+
|
1639
|
+
class TranslateDeleteGlossaryOperator(GoogleCloudBaseOperator):
|
1640
|
+
"""
|
1641
|
+
Delete a Google Cloud Translation Glossary.
|
1642
|
+
|
1643
|
+
Deletes a translation glossary, using API V3.
|
1644
|
+
For more information on how to use this operator, take a look at the guide:
|
1645
|
+
:ref:`howto/operator:TranslateDeleteGlossaryOperator`.
|
1646
|
+
|
1647
|
+
:param glossary_id: User-specified id to delete glossary resource item.
|
1648
|
+
:param project_id: ID of the Google Cloud project where glossary is located.
|
1649
|
+
If not provided default project_id is used.
|
1650
|
+
:param location: The location of the project.
|
1651
|
+
:param retry: Designation of what errors, if any, should be retried.
|
1652
|
+
:param timeout: The timeout for this request.
|
1653
|
+
:param metadata: Strings which should be sent along with the request as metadata.
|
1654
|
+
:param gcp_conn_id: The connection ID to use connecting to Google Cloud.
|
1655
|
+
:param impersonation_chain: Optional service account to impersonate using short-term
|
1656
|
+
credentials, or chained list of accounts required to get the access_token
|
1657
|
+
of the last account in the list, which will be impersonated in the request.
|
1658
|
+
If set as a string, the account must grant the originating account
|
1659
|
+
the Service Account Token Creator IAM role.
|
1660
|
+
If set as a sequence, the identities from the list must grant
|
1661
|
+
Service Account Token Creator IAM role to the directly preceding identity, with first
|
1662
|
+
account from the list granting this role to the originating account (templated).
|
1663
|
+
"""
|
1664
|
+
|
1665
|
+
template_fields: Sequence[str] = (
|
1666
|
+
"glossary_id",
|
1667
|
+
"location",
|
1668
|
+
"project_id",
|
1669
|
+
"gcp_conn_id",
|
1670
|
+
"impersonation_chain",
|
1671
|
+
)
|
1672
|
+
|
1673
|
+
def __init__(
|
1674
|
+
self,
|
1675
|
+
*,
|
1676
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1677
|
+
location: str,
|
1678
|
+
glossary_id: str,
|
1679
|
+
timeout: float | None = None,
|
1680
|
+
retry: Retry | _MethodDefault = DEFAULT,
|
1681
|
+
gcp_conn_id: str = "google_cloud_default",
|
1682
|
+
metadata: Sequence[tuple[str, str]] = (),
|
1683
|
+
impersonation_chain: str | Sequence[str] | None = None,
|
1684
|
+
**kwargs,
|
1685
|
+
) -> None:
|
1686
|
+
super().__init__(**kwargs)
|
1687
|
+
self.glossary_id = glossary_id
|
1688
|
+
self.project_id = project_id
|
1689
|
+
self.location = location
|
1690
|
+
self.metadata = metadata
|
1691
|
+
self.timeout = timeout
|
1692
|
+
self.retry = retry
|
1693
|
+
self.gcp_conn_id = gcp_conn_id
|
1694
|
+
self.impersonation_chain = impersonation_chain
|
1695
|
+
|
1696
|
+
def execute(self, context: Context) -> str:
|
1697
|
+
hook = TranslateHook(
|
1698
|
+
gcp_conn_id=self.gcp_conn_id,
|
1699
|
+
impersonation_chain=self.impersonation_chain,
|
1700
|
+
)
|
1701
|
+
project_id = self.project_id or hook.project_id
|
1702
|
+
try:
|
1703
|
+
result_operation = hook.delete_glossary(
|
1704
|
+
glossary_id=self.glossary_id,
|
1705
|
+
project_id=project_id,
|
1706
|
+
location=self.location,
|
1707
|
+
retry=self.retry,
|
1708
|
+
timeout=self.timeout,
|
1709
|
+
metadata=self.metadata,
|
1710
|
+
)
|
1711
|
+
except GoogleAPICallError as e:
|
1712
|
+
self.log.error("Error submitting delete_glossary operation ")
|
1713
|
+
raise AirflowException(e)
|
1714
|
+
self.log.info("Glossary delete started, glossary_id %s...", self.glossary_id)
|
1715
|
+
|
1716
|
+
result = hook.wait_for_operation_result(operation=result_operation)
|
1717
|
+
result = type(result).to_dict(result)
|
1718
|
+
self.log.info("Glossary deletion complete. The glossary_id: %s.", self.glossary_id)
|
1719
|
+
return result
|