apache-airflow-providers-google 12.0.0rc2__py3-none-any.whl → 14.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. airflow/providers/google/LICENSE +0 -52
  2. airflow/providers/google/__init__.py +1 -1
  3. airflow/providers/google/ads/hooks/ads.py +27 -13
  4. airflow/providers/google/ads/transfers/ads_to_gcs.py +18 -4
  5. airflow/providers/google/assets/bigquery.py +17 -0
  6. airflow/providers/google/cloud/_internal_client/secret_manager_client.py +2 -3
  7. airflow/providers/google/cloud/hooks/alloy_db.py +736 -8
  8. airflow/providers/google/cloud/hooks/automl.py +10 -4
  9. airflow/providers/google/cloud/hooks/bigquery.py +125 -22
  10. airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
  11. airflow/providers/google/cloud/hooks/bigtable.py +2 -3
  12. airflow/providers/google/cloud/hooks/cloud_batch.py +3 -4
  13. airflow/providers/google/cloud/hooks/cloud_build.py +4 -5
  14. airflow/providers/google/cloud/hooks/cloud_composer.py +3 -4
  15. airflow/providers/google/cloud/hooks/cloud_memorystore.py +3 -4
  16. airflow/providers/google/cloud/hooks/cloud_run.py +3 -4
  17. airflow/providers/google/cloud/hooks/cloud_sql.py +7 -3
  18. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +119 -7
  19. airflow/providers/google/cloud/hooks/compute.py +3 -3
  20. airflow/providers/google/cloud/hooks/datacatalog.py +3 -4
  21. airflow/providers/google/cloud/hooks/dataflow.py +12 -12
  22. airflow/providers/google/cloud/hooks/dataform.py +2 -3
  23. airflow/providers/google/cloud/hooks/datafusion.py +2 -2
  24. airflow/providers/google/cloud/hooks/dataplex.py +1032 -11
  25. airflow/providers/google/cloud/hooks/dataproc.py +4 -5
  26. airflow/providers/google/cloud/hooks/dataproc_metastore.py +3 -4
  27. airflow/providers/google/cloud/hooks/dlp.py +3 -4
  28. airflow/providers/google/cloud/hooks/gcs.py +7 -6
  29. airflow/providers/google/cloud/hooks/kms.py +2 -3
  30. airflow/providers/google/cloud/hooks/kubernetes_engine.py +8 -8
  31. airflow/providers/google/cloud/hooks/life_sciences.py +1 -1
  32. airflow/providers/google/cloud/hooks/managed_kafka.py +482 -0
  33. airflow/providers/google/cloud/hooks/natural_language.py +2 -3
  34. airflow/providers/google/cloud/hooks/os_login.py +2 -3
  35. airflow/providers/google/cloud/hooks/pubsub.py +6 -6
  36. airflow/providers/google/cloud/hooks/secret_manager.py +2 -3
  37. airflow/providers/google/cloud/hooks/spanner.py +2 -2
  38. airflow/providers/google/cloud/hooks/speech_to_text.py +2 -3
  39. airflow/providers/google/cloud/hooks/stackdriver.py +4 -4
  40. airflow/providers/google/cloud/hooks/tasks.py +3 -4
  41. airflow/providers/google/cloud/hooks/text_to_speech.py +2 -3
  42. airflow/providers/google/cloud/hooks/translate.py +236 -5
  43. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +9 -4
  44. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +3 -4
  45. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +4 -5
  46. airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +3 -4
  47. airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +2 -3
  48. airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +3 -4
  49. airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +1 -181
  50. airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +3 -4
  51. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +2 -3
  52. airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +3 -4
  53. airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +2 -3
  54. airflow/providers/google/cloud/hooks/video_intelligence.py +2 -3
  55. airflow/providers/google/cloud/hooks/vision.py +3 -4
  56. airflow/providers/google/cloud/hooks/workflows.py +2 -3
  57. airflow/providers/google/cloud/links/alloy_db.py +46 -0
  58. airflow/providers/google/cloud/links/bigquery.py +25 -0
  59. airflow/providers/google/cloud/links/dataplex.py +172 -2
  60. airflow/providers/google/cloud/links/kubernetes_engine.py +1 -2
  61. airflow/providers/google/cloud/links/managed_kafka.py +104 -0
  62. airflow/providers/google/cloud/links/translate.py +28 -0
  63. airflow/providers/google/cloud/log/gcs_task_handler.py +3 -3
  64. airflow/providers/google/cloud/log/stackdriver_task_handler.py +11 -10
  65. airflow/providers/google/cloud/openlineage/facets.py +67 -0
  66. airflow/providers/google/cloud/openlineage/mixins.py +438 -173
  67. airflow/providers/google/cloud/openlineage/utils.py +394 -61
  68. airflow/providers/google/cloud/operators/alloy_db.py +980 -69
  69. airflow/providers/google/cloud/operators/automl.py +83 -245
  70. airflow/providers/google/cloud/operators/bigquery.py +377 -74
  71. airflow/providers/google/cloud/operators/bigquery_dts.py +126 -13
  72. airflow/providers/google/cloud/operators/bigtable.py +1 -3
  73. airflow/providers/google/cloud/operators/cloud_base.py +1 -2
  74. airflow/providers/google/cloud/operators/cloud_batch.py +2 -4
  75. airflow/providers/google/cloud/operators/cloud_build.py +3 -5
  76. airflow/providers/google/cloud/operators/cloud_composer.py +5 -7
  77. airflow/providers/google/cloud/operators/cloud_memorystore.py +4 -6
  78. airflow/providers/google/cloud/operators/cloud_run.py +6 -5
  79. airflow/providers/google/cloud/operators/cloud_sql.py +20 -8
  80. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +62 -8
  81. airflow/providers/google/cloud/operators/compute.py +3 -4
  82. airflow/providers/google/cloud/operators/datacatalog.py +9 -11
  83. airflow/providers/google/cloud/operators/dataflow.py +1 -112
  84. airflow/providers/google/cloud/operators/dataform.py +3 -5
  85. airflow/providers/google/cloud/operators/datafusion.py +1 -1
  86. airflow/providers/google/cloud/operators/dataplex.py +2046 -7
  87. airflow/providers/google/cloud/operators/dataproc.py +102 -17
  88. airflow/providers/google/cloud/operators/dataproc_metastore.py +7 -9
  89. airflow/providers/google/cloud/operators/dlp.py +17 -19
  90. airflow/providers/google/cloud/operators/gcs.py +14 -17
  91. airflow/providers/google/cloud/operators/kubernetes_engine.py +2 -2
  92. airflow/providers/google/cloud/operators/managed_kafka.py +788 -0
  93. airflow/providers/google/cloud/operators/natural_language.py +3 -5
  94. airflow/providers/google/cloud/operators/pubsub.py +39 -7
  95. airflow/providers/google/cloud/operators/speech_to_text.py +3 -5
  96. airflow/providers/google/cloud/operators/stackdriver.py +3 -5
  97. airflow/providers/google/cloud/operators/tasks.py +4 -6
  98. airflow/providers/google/cloud/operators/text_to_speech.py +2 -4
  99. airflow/providers/google/cloud/operators/translate.py +414 -5
  100. airflow/providers/google/cloud/operators/translate_speech.py +2 -4
  101. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +9 -8
  102. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +4 -6
  103. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +6 -8
  104. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +4 -6
  105. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +4 -6
  106. airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +0 -322
  107. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +4 -6
  108. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +4 -6
  109. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +4 -6
  110. airflow/providers/google/cloud/operators/video_intelligence.py +3 -5
  111. airflow/providers/google/cloud/operators/vision.py +4 -6
  112. airflow/providers/google/cloud/operators/workflows.py +5 -7
  113. airflow/providers/google/cloud/secrets/secret_manager.py +1 -2
  114. airflow/providers/google/cloud/sensors/bigquery_dts.py +3 -5
  115. airflow/providers/google/cloud/sensors/bigtable.py +2 -3
  116. airflow/providers/google/cloud/sensors/cloud_composer.py +32 -8
  117. airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +39 -1
  118. airflow/providers/google/cloud/sensors/dataplex.py +4 -6
  119. airflow/providers/google/cloud/sensors/dataproc.py +2 -3
  120. airflow/providers/google/cloud/sensors/dataproc_metastore.py +1 -2
  121. airflow/providers/google/cloud/sensors/gcs.py +2 -4
  122. airflow/providers/google/cloud/sensors/pubsub.py +2 -3
  123. airflow/providers/google/cloud/sensors/workflows.py +3 -5
  124. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +5 -5
  125. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +10 -12
  126. airflow/providers/google/cloud/transfers/gcs_to_gcs.py +1 -1
  127. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +36 -4
  128. airflow/providers/google/cloud/transfers/mssql_to_gcs.py +27 -2
  129. airflow/providers/google/cloud/transfers/mysql_to_gcs.py +27 -2
  130. airflow/providers/google/cloud/transfers/postgres_to_gcs.py +27 -2
  131. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +34 -5
  132. airflow/providers/google/cloud/transfers/sql_to_gcs.py +15 -0
  133. airflow/providers/google/cloud/transfers/trino_to_gcs.py +25 -2
  134. airflow/providers/google/cloud/triggers/bigquery_dts.py +1 -2
  135. airflow/providers/google/cloud/triggers/cloud_batch.py +1 -2
  136. airflow/providers/google/cloud/triggers/cloud_build.py +1 -2
  137. airflow/providers/google/cloud/triggers/cloud_composer.py +13 -3
  138. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +102 -4
  139. airflow/providers/google/cloud/triggers/dataflow.py +2 -3
  140. airflow/providers/google/cloud/triggers/dataplex.py +1 -2
  141. airflow/providers/google/cloud/triggers/dataproc.py +2 -3
  142. airflow/providers/google/cloud/triggers/kubernetes_engine.py +1 -1
  143. airflow/providers/google/cloud/triggers/pubsub.py +1 -2
  144. airflow/providers/google/cloud/triggers/vertex_ai.py +7 -8
  145. airflow/providers/google/cloud/utils/credentials_provider.py +15 -8
  146. airflow/providers/google/cloud/utils/external_token_supplier.py +1 -0
  147. airflow/providers/google/common/auth_backend/google_openid.py +4 -4
  148. airflow/providers/google/common/consts.py +1 -2
  149. airflow/providers/google/common/hooks/base_google.py +8 -7
  150. airflow/providers/google/get_provider_info.py +186 -134
  151. airflow/providers/google/marketing_platform/hooks/analytics_admin.py +2 -3
  152. airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
  153. airflow/providers/google/marketing_platform/operators/analytics_admin.py +5 -7
  154. {apache_airflow_providers_google-12.0.0rc2.dist-info → apache_airflow_providers_google-14.0.0.dist-info}/METADATA +41 -58
  155. {apache_airflow_providers_google-12.0.0rc2.dist-info → apache_airflow_providers_google-14.0.0.dist-info}/RECORD +157 -159
  156. airflow/providers/google/cloud/example_dags/example_facebook_ads_to_gcs.py +0 -141
  157. airflow/providers/google/cloud/example_dags/example_looker.py +0 -64
  158. airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py +0 -194
  159. airflow/providers/google/cloud/example_dags/example_salesforce_to_gcs.py +0 -129
  160. airflow/providers/google/marketing_platform/example_dags/__init__.py +0 -16
  161. airflow/providers/google/marketing_platform/example_dags/example_display_video.py +0 -213
  162. {apache_airflow_providers_google-12.0.0rc2.dist-info → apache_airflow_providers_google-14.0.0.dist-info}/WHEEL +0 -0
  163. {apache_airflow_providers_google-12.0.0rc2.dist-info → apache_airflow_providers_google-14.0.0.dist-info}/entry_points.txt +0 -0
@@ -22,18 +22,16 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
26
- from google.protobuf.json_format import MessageToDict
27
-
28
25
  from airflow.providers.google.cloud.hooks.natural_language import CloudNaturalLanguageHook
29
26
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
27
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
28
+ from google.protobuf.json_format import MessageToDict
30
29
 
31
30
  if TYPE_CHECKING:
31
+ from airflow.utils.context import Context
32
32
  from google.api_core.retry import Retry
33
33
  from google.cloud.language_v1.types import Document, EncodingType
34
34
 
35
- from airflow.utils.context import Context
36
-
37
35
 
38
36
  MetaData = Sequence[tuple[str, str]]
39
37
 
@@ -28,6 +28,13 @@ from __future__ import annotations
28
28
  from collections.abc import Sequence
29
29
  from typing import TYPE_CHECKING, Any, Callable
30
30
 
31
+ from airflow.exceptions import AirflowException
32
+ from airflow.providers.google.cloud.hooks.pubsub import PubSubHook
33
+ from airflow.providers.google.cloud.links.pubsub import PubSubSubscriptionLink, PubSubTopicLink
34
+ from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
35
+ from airflow.providers.google.cloud.triggers.pubsub import PubsubPullTrigger
36
+ from airflow.providers.google.common.consts import GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME
37
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
31
38
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
32
39
  from google.cloud.pubsub_v1.types import (
33
40
  DeadLetterPolicy,
@@ -40,15 +47,9 @@ from google.cloud.pubsub_v1.types import (
40
47
  SchemaSettings,
41
48
  )
42
49
 
43
- from airflow.providers.google.cloud.hooks.pubsub import PubSubHook
44
- from airflow.providers.google.cloud.links.pubsub import PubSubSubscriptionLink, PubSubTopicLink
45
- from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
46
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
47
-
48
50
  if TYPE_CHECKING:
49
- from google.api_core.retry import Retry
50
-
51
51
  from airflow.utils.context import Context
52
+ from google.api_core.retry import Retry
52
53
 
53
54
 
54
55
  class PubSubCreateTopicOperator(GoogleCloudBaseOperator):
@@ -746,6 +747,9 @@ class PubSubPullOperator(GoogleCloudBaseOperator):
746
747
  If set as a sequence, the identities from the list must grant
747
748
  Service Account Token Creator IAM role to the directly preceding identity, with first
748
749
  account from the list granting this role to the originating account (templated).
750
+ :param deferrable: If True, run the task in the deferrable mode.
751
+ :param poll_interval: Time (seconds) to wait between two consecutive calls to check the job.
752
+ The default is 300 seconds.
749
753
  """
750
754
 
751
755
  template_fields: Sequence[str] = (
@@ -764,6 +768,8 @@ class PubSubPullOperator(GoogleCloudBaseOperator):
764
768
  messages_callback: Callable[[list[ReceivedMessage], Context], Any] | None = None,
765
769
  gcp_conn_id: str = "google_cloud_default",
766
770
  impersonation_chain: str | Sequence[str] | None = None,
771
+ deferrable: bool = False,
772
+ poll_interval: int = 300,
767
773
  **kwargs,
768
774
  ) -> None:
769
775
  super().__init__(**kwargs)
@@ -774,8 +780,23 @@ class PubSubPullOperator(GoogleCloudBaseOperator):
774
780
  self.ack_messages = ack_messages
775
781
  self.messages_callback = messages_callback
776
782
  self.impersonation_chain = impersonation_chain
783
+ self.deferrable = deferrable
784
+ self.poll_interval = poll_interval
777
785
 
778
786
  def execute(self, context: Context) -> list:
787
+ if self.deferrable:
788
+ self.defer(
789
+ trigger=PubsubPullTrigger(
790
+ subscription=self.subscription,
791
+ project_id=self.project_id,
792
+ max_messages=self.max_messages,
793
+ ack_messages=self.ack_messages,
794
+ gcp_conn_id=self.gcp_conn_id,
795
+ poke_interval=self.poll_interval,
796
+ impersonation_chain=self.impersonation_chain,
797
+ ),
798
+ method_name=GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME,
799
+ )
779
800
  hook = PubSubHook(
780
801
  gcp_conn_id=self.gcp_conn_id,
781
802
  impersonation_chain=self.impersonation_chain,
@@ -801,6 +822,17 @@ class PubSubPullOperator(GoogleCloudBaseOperator):
801
822
 
802
823
  return ret
803
824
 
825
+ def execute_complete(self, context: Context, event: dict[str, Any]):
826
+ """If messages_callback is provided, execute it; otherwise, return immediately with trigger event message."""
827
+ if event["status"] == "success":
828
+ self.log.info("Sensor pulls messages: %s", event["message"])
829
+ messages_callback = self.messages_callback or self._default_message_callback
830
+ _return_value = messages_callback(event["message"], context)
831
+ return _return_value
832
+
833
+ self.log.info("Sensor failed: %s", event["message"])
834
+ raise AirflowException(event["message"])
835
+
804
836
  def _default_message_callback(
805
837
  self,
806
838
  pulled_messages: list[ReceivedMessage],
@@ -22,21 +22,19 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
26
- from google.protobuf.json_format import MessageToDict
27
-
28
25
  from airflow.exceptions import AirflowException
29
26
  from airflow.providers.google.cloud.hooks.speech_to_text import CloudSpeechToTextHook, RecognitionAudio
30
27
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
31
28
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
32
29
  from airflow.providers.google.common.links.storage import FileDetailsLink
30
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
31
+ from google.protobuf.json_format import MessageToDict
33
32
 
34
33
  if TYPE_CHECKING:
34
+ from airflow.utils.context import Context
35
35
  from google.api_core.retry import Retry
36
36
  from google.cloud.speech_v1.types import RecognitionConfig
37
37
 
38
- from airflow.utils.context import Context
39
-
40
38
 
41
39
  class CloudSpeechToTextRecognizeSpeechOperator(GoogleCloudBaseOperator):
42
40
  """
@@ -20,9 +20,6 @@ from __future__ import annotations
20
20
  from collections.abc import Sequence
21
21
  from typing import TYPE_CHECKING
22
22
 
23
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
24
- from google.cloud.monitoring_v3 import AlertPolicy, NotificationChannel
25
-
26
23
  from airflow.providers.google.cloud.hooks.stackdriver import StackdriverHook
27
24
  from airflow.providers.google.cloud.links.stackdriver import (
28
25
  StackdriverNotificationsLink,
@@ -30,11 +27,12 @@ from airflow.providers.google.cloud.links.stackdriver import (
30
27
  )
31
28
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
32
29
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
30
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
31
+ from google.cloud.monitoring_v3 import AlertPolicy, NotificationChannel
33
32
 
34
33
  if TYPE_CHECKING:
35
- from google.api_core.retry import Retry
36
-
37
34
  from airflow.utils.context import Context
35
+ from google.api_core.retry import Retry
38
36
 
39
37
 
40
38
  class StackdriverListAlertPoliciesOperator(GoogleCloudBaseOperator):
@@ -22,21 +22,19 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
- from google.api_core.exceptions import AlreadyExists
26
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
27
- from google.cloud.tasks_v2.types import Queue, Task
28
-
29
25
  from airflow.providers.google.cloud.hooks.tasks import CloudTasksHook
30
26
  from airflow.providers.google.cloud.links.cloud_tasks import CloudTasksLink, CloudTasksQueueLink
31
27
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
32
28
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
29
+ from google.api_core.exceptions import AlreadyExists
30
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
31
+ from google.cloud.tasks_v2.types import Queue, Task
33
32
 
34
33
  if TYPE_CHECKING:
34
+ from airflow.utils.context import Context
35
35
  from google.api_core.retry import Retry
36
36
  from google.protobuf.field_mask_pb2 import FieldMask
37
37
 
38
- from airflow.utils.context import Context
39
-
40
38
 
41
39
  MetaData = Sequence[tuple[str, str]]
42
40
 
@@ -23,21 +23,19 @@ from collections.abc import Sequence
23
23
  from tempfile import NamedTemporaryFile
24
24
  from typing import TYPE_CHECKING
25
25
 
26
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
27
-
28
26
  from airflow.exceptions import AirflowException
29
27
  from airflow.providers.google.cloud.hooks.gcs import GCSHook
30
28
  from airflow.providers.google.cloud.hooks.text_to_speech import CloudTextToSpeechHook
31
29
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
32
30
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
33
31
  from airflow.providers.google.common.links.storage import FileDetailsLink
32
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
34
33
 
35
34
  if TYPE_CHECKING:
35
+ from airflow.utils.context import Context
36
36
  from google.api_core.retry import Retry
37
37
  from google.cloud.texttospeech_v1.types import AudioConfig, SynthesisInput, VoiceSelectionParams
38
38
 
39
- from airflow.utils.context import Context
40
-
41
39
 
42
40
  class CloudTextToSpeechSynthesizeOperator(GoogleCloudBaseOperator):
43
41
  """
@@ -22,23 +22,24 @@ from __future__ import annotations
22
22
  from collections.abc import MutableMapping, MutableSequence, Sequence
23
23
  from typing import TYPE_CHECKING, cast
24
24
 
25
- from google.api_core.exceptions import GoogleAPICallError
26
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
27
-
28
25
  from airflow.exceptions import AirflowException
29
26
  from airflow.providers.google.cloud.hooks.translate import CloudTranslateHook, TranslateHook
30
27
  from airflow.providers.google.cloud.links.translate import (
31
28
  TranslateResultByOutputConfigLink,
32
29
  TranslateTextBatchLink,
33
30
  TranslationDatasetsListLink,
31
+ TranslationGlossariesListLink,
34
32
  TranslationModelLink,
35
33
  TranslationModelsListLink,
36
34
  TranslationNativeDatasetLink,
37
35
  )
38
36
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
39
37
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
38
+ from google.api_core.exceptions import GoogleAPICallError
39
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
40
40
 
41
41
  if TYPE_CHECKING:
42
+ from airflow.utils.context import Context
42
43
  from google.api_core.retry import Retry
43
44
  from google.cloud.translate_v3.types import (
44
45
  BatchDocumentInputConfig,
@@ -52,8 +53,7 @@ if TYPE_CHECKING:
52
53
  TransliterationConfig,
53
54
  automl_translation,
54
55
  )
55
-
56
- from airflow.utils.context import Context
56
+ from google.cloud.translate_v3.types.translation_service import Glossary, GlossaryInputConfig
57
57
 
58
58
 
59
59
  class CloudTranslateTextOperator(GoogleCloudBaseOperator):
@@ -158,6 +158,7 @@ class TranslateTextOperator(GoogleCloudBaseOperator):
158
158
 
159
159
  :param project_id: Optional. The ID of the Google Cloud project that the
160
160
  service belongs to.
161
+ If not provided default project_id is used.
161
162
  :param location: optional. The ID of the Google Cloud location that the
162
163
  service belongs to. if not specified, 'global' is used.
163
164
  Non-global location is required for requests using AutoML models or custom glossaries.
@@ -1308,3 +1309,411 @@ class TranslateDocumentBatchOperator(GoogleCloudBaseOperator):
1308
1309
  result = hook.wait_for_operation_result(batch_document_translate_operation)
1309
1310
  self.log.info("Batch document translation job finished")
1310
1311
  return cast(dict, type(result).to_dict(result))
1312
+
1313
+
1314
+ class TranslateCreateGlossaryOperator(GoogleCloudBaseOperator):
1315
+ """
1316
+ Creates a Google Cloud Translation Glossary.
1317
+
1318
+ Creates a translation glossary, using API V3.
1319
+ For more information on how to use this operator, take a look at the guide:
1320
+ :ref:`howto/operator:TranslateCreateGlossaryOperator`.
1321
+
1322
+ :param glossary_id: User-specified id to built glossary resource name.
1323
+ :param input_config: The input configuration of examples to built glossary from.
1324
+ Total glossary must not exceed 10M Unicode codepoints.
1325
+ The headers should not be included into the input file table, as languages specified with the
1326
+ ``language_pair`` or ``language_codes_set`` params.
1327
+ :param language_pair: Pair of language codes to be used for glossary creation.
1328
+ Used to built unidirectional glossary. If specified, the ``language_codes_set`` should be empty.
1329
+ :param language_codes_set: Set of language codes to create the equivalent term sets glossary.
1330
+ Meant multiple languages mapping. If specified, the ``language_pair`` should be empty.
1331
+ :param project_id: ID of the Google Cloud project where glossary is located.
1332
+ If not provided default project_id is used.
1333
+ :param location: The location of the project.
1334
+ :param retry: Designation of what errors, if any, should be retried.
1335
+ :param timeout: The timeout for this request.
1336
+ :param metadata: Strings which should be sent along with the request as metadata.
1337
+ :param gcp_conn_id: The connection ID to use connecting to Google Cloud.
1338
+ :param impersonation_chain: Optional service account to impersonate using short-term
1339
+ credentials, or chained list of accounts required to get the access_token
1340
+ of the last account in the list, which will be impersonated in the request.
1341
+ If set as a string, the account must grant the originating account
1342
+ the Service Account Token Creator IAM role.
1343
+ If set as a sequence, the identities from the list must grant
1344
+ Service Account Token Creator IAM role to the directly preceding identity, with first
1345
+ account from the list granting this role to the originating account (templated).
1346
+ """
1347
+
1348
+ template_fields: Sequence[str] = (
1349
+ "glossary_id",
1350
+ "location",
1351
+ "project_id",
1352
+ "gcp_conn_id",
1353
+ "impersonation_chain",
1354
+ )
1355
+
1356
+ def __init__(
1357
+ self,
1358
+ *,
1359
+ project_id: str = PROVIDE_PROJECT_ID,
1360
+ location: str,
1361
+ glossary_id: str,
1362
+ input_config: GlossaryInputConfig | dict,
1363
+ language_pair: Glossary.LanguageCodePair | dict | None = None,
1364
+ language_codes_set: Glossary.LanguageCodesSet | MutableSequence[str] | None = None,
1365
+ timeout: float | None = None,
1366
+ retry: Retry | _MethodDefault = DEFAULT,
1367
+ gcp_conn_id: str = "google_cloud_default",
1368
+ metadata: Sequence[tuple[str, str]] = (),
1369
+ impersonation_chain: str | Sequence[str] | None = None,
1370
+ **kwargs,
1371
+ ) -> None:
1372
+ super().__init__(**kwargs)
1373
+ self.project_id = project_id
1374
+ self.location = location
1375
+ self.glossary_id = glossary_id
1376
+ self.input_config = input_config
1377
+ self.language_pair = language_pair
1378
+ self.language_codes_set = language_codes_set
1379
+ self.metadata = metadata
1380
+ self.timeout = timeout
1381
+ self.retry = retry
1382
+ self.gcp_conn_id = gcp_conn_id
1383
+ self.impersonation_chain = impersonation_chain
1384
+
1385
+ def execute(self, context: Context) -> str:
1386
+ hook = TranslateHook(
1387
+ gcp_conn_id=self.gcp_conn_id,
1388
+ impersonation_chain=self.impersonation_chain,
1389
+ )
1390
+ project_id = self.project_id or hook.project_id
1391
+ try:
1392
+ result_operation = hook.create_glossary(
1393
+ glossary_id=self.glossary_id,
1394
+ input_config=self.input_config,
1395
+ language_pair=self.language_pair,
1396
+ language_codes_set=self.language_codes_set,
1397
+ project_id=project_id,
1398
+ location=self.location,
1399
+ retry=self.retry,
1400
+ timeout=self.timeout,
1401
+ metadata=self.metadata,
1402
+ )
1403
+ except GoogleAPICallError as e:
1404
+ self.log.error("Error submitting create_glossary operation ")
1405
+ raise AirflowException(e)
1406
+ self.log.info("Glossary creation started, glossary_id %s...", self.glossary_id)
1407
+
1408
+ result = hook.wait_for_operation_result(operation=result_operation)
1409
+ result = type(result).to_dict(result)
1410
+
1411
+ glossary_id = hook.extract_object_id(result)
1412
+ self.xcom_push(context, key="glossary_id", value=glossary_id)
1413
+ self.log.info("Glossary creation complete. The glossary_id: %s.", glossary_id)
1414
+ return result
1415
+
1416
+
1417
+ class TranslateUpdateGlossaryOperator(GoogleCloudBaseOperator):
1418
+ """
1419
+ Update glossary item with values provided.
1420
+
1421
+ Updates the translation glossary, using translation API V3.
1422
+ Only ``display_name`` and ``input_config`` fields are allowed for update.
1423
+
1424
+ For more information on how to use this operator, take a look at the guide:
1425
+ :ref:`howto/operator:TranslateUpdateGlossaryOperator`.
1426
+
1427
+ :param glossary_id: User-specified id to built glossary resource name.
1428
+ :param input_config: The input configuration of examples to built glossary from.
1429
+ Total glossary must not exceed 10M Unicode codepoints.
1430
+ The headers should not be included into the input file table, as languages specified with the
1431
+ ``language_pair`` or ``language_codes_set`` params.
1432
+ :param language_pair: Pair of language codes to be used for glossary creation.
1433
+ Used to built unidirectional glossary. If specified, the ``language_codes_set`` should be empty.
1434
+ :param language_codes_set: Set of language codes to create the equivalent term sets glossary.
1435
+ Meant multiple languages mapping. If specified, the ``language_pair`` should be empty.
1436
+ :param project_id: ID of the Google Cloud project where glossary is located.
1437
+ If not provided default project_id is used.
1438
+ :param location: The location of the project.
1439
+ :param retry: Designation of what errors, if any, should be retried.
1440
+ :param timeout: The timeout for this request.
1441
+ :param metadata: Strings which should be sent along with the request as metadata.
1442
+ :param gcp_conn_id: The connection ID to use connecting to Google Cloud.
1443
+ :param impersonation_chain: Optional service account to impersonate using short-term
1444
+ credentials, or chained list of accounts required to get the access_token
1445
+ of the last account in the list, which will be impersonated in the request.
1446
+ If set as a string, the account must grant the originating account
1447
+ the Service Account Token Creator IAM role.
1448
+ If set as a sequence, the identities from the list must grant
1449
+ Service Account Token Creator IAM role to the directly preceding identity, with first
1450
+ account from the list granting this role to the originating account (templated).
1451
+ """
1452
+
1453
+ template_fields: Sequence[str] = (
1454
+ "glossary_id",
1455
+ "location",
1456
+ "project_id",
1457
+ "gcp_conn_id",
1458
+ "impersonation_chain",
1459
+ )
1460
+
1461
+ def __init__(
1462
+ self,
1463
+ *,
1464
+ project_id: str = PROVIDE_PROJECT_ID,
1465
+ location: str,
1466
+ glossary_id: str,
1467
+ new_display_name: str,
1468
+ new_input_config: GlossaryInputConfig | dict | None = None,
1469
+ timeout: float | None = None,
1470
+ retry: Retry | _MethodDefault = DEFAULT,
1471
+ gcp_conn_id: str = "google_cloud_default",
1472
+ metadata: Sequence[tuple[str, str]] = (),
1473
+ impersonation_chain: str | Sequence[str] | None = None,
1474
+ **kwargs,
1475
+ ) -> None:
1476
+ super().__init__(**kwargs)
1477
+ self.project_id = project_id
1478
+ self.location = location
1479
+ self.glossary_id = glossary_id
1480
+ self.new_display_name = new_display_name
1481
+ self.new_input_config = new_input_config
1482
+ self.metadata = metadata
1483
+ self.timeout = timeout
1484
+ self.retry = retry
1485
+ self.gcp_conn_id = gcp_conn_id
1486
+ self.impersonation_chain = impersonation_chain
1487
+
1488
+ def execute(self, context: Context) -> str:
1489
+ hook = TranslateHook(
1490
+ gcp_conn_id=self.gcp_conn_id,
1491
+ impersonation_chain=self.impersonation_chain,
1492
+ )
1493
+ project_id = self.project_id or hook.project_id
1494
+ glossary = hook.get_glossary(
1495
+ glossary_id=self.glossary_id,
1496
+ project_id=project_id,
1497
+ location=self.location,
1498
+ retry=self.retry,
1499
+ timeout=self.timeout,
1500
+ metadata=self.metadata,
1501
+ )
1502
+ try:
1503
+ result_operation = hook.update_glossary(
1504
+ glossary=glossary,
1505
+ new_display_name=self.new_display_name,
1506
+ new_input_config=self.new_input_config,
1507
+ retry=self.retry,
1508
+ timeout=self.timeout,
1509
+ metadata=self.metadata,
1510
+ )
1511
+ except GoogleAPICallError as e:
1512
+ self.log.error("Error submitting update_glossary operation ")
1513
+ raise AirflowException(e)
1514
+ self.log.info("Glossary update started, glossary_id %s...", self.glossary_id)
1515
+
1516
+ result = hook.wait_for_operation_result(operation=result_operation)
1517
+ result = type(result).to_dict(result)
1518
+ self.log.info("Glossary update complete. The glossary_id: %s.", self.glossary_id)
1519
+ return result
1520
+
1521
+
1522
+ class TranslateListGlossariesOperator(GoogleCloudBaseOperator):
1523
+ """
1524
+ Get a list of translation glossaries in a project.
1525
+
1526
+ List the translation glossaries, using translation API V3.
1527
+
1528
+ For more information on how to use this operator, take a look at the guide:
1529
+ :ref:`howto/operator:TranslateListGlossariesOperator`.
1530
+
1531
+ :param project_id: ID of the Google Cloud project where glossary is located.
1532
+ If not provided default project_id is used.
1533
+ :param page_size: Page size requested, if not set server use appropriate default.
1534
+ :param page_token: A token identifying a page of results the server should return.
1535
+ The first page is returned if ``page_token`` is empty or missing.
1536
+ :param filter_str: Filter specifying constraints of a list operation. Specify the constraint by the
1537
+ format of "key=value", where key must be ``src`` or ``tgt``, and the value must be a valid
1538
+ language code.
1539
+ For multiple restrictions, concatenate them by "AND" (uppercase only), such as:
1540
+ ``src=en-US AND tgt=zh-CN``. Notice that the exact match is used here, which means using 'en-US'
1541
+ and 'en' can lead to different results, which depends on the language code you used when you
1542
+ create the glossary.
1543
+ For the unidirectional glossaries, the ``src`` and ``tgt`` add restrictions
1544
+ on the source and target language code separately.
1545
+ For the equivalent term set glossaries, the ``src`` and/or ``tgt``
1546
+ add restrictions on the term set.
1547
+ For example: ``src=en-US AND tgt=zh-CN`` will only pick the unidirectional glossaries which
1548
+ exactly match the source language code as ``en-US`` and the target language code ``zh-CN``,
1549
+ but all equivalent term set glossaries which contain ``en-US`` and ``zh-CN`` in their language
1550
+ set will be picked.
1551
+ If missing, no filtering is performed.
1552
+ :param location: The location of the project.
1553
+ :param retry: Designation of what errors, if any, should be retried.
1554
+ :param timeout: The timeout for this request.
1555
+ :param metadata: Strings which should be sent along with the request as metadata.
1556
+ :param gcp_conn_id: The connection ID to use connecting to Google Cloud.
1557
+ :param impersonation_chain: Optional service account to impersonate using short-term
1558
+ credentials, or chained list of accounts required to get the access_token
1559
+ of the last account in the list, which will be impersonated in the request.
1560
+ If set as a string, the account must grant the originating account
1561
+ the Service Account Token Creator IAM role.
1562
+ If set as a sequence, the identities from the list must grant
1563
+ Service Account Token Creator IAM role to the directly preceding identity, with first
1564
+ account from the list granting this role to the originating account (templated).
1565
+ """
1566
+
1567
+ operator_extra_links = (TranslationGlossariesListLink(),)
1568
+
1569
+ template_fields: Sequence[str] = (
1570
+ "location",
1571
+ "project_id",
1572
+ "gcp_conn_id",
1573
+ "impersonation_chain",
1574
+ )
1575
+
1576
+ def __init__(
1577
+ self,
1578
+ *,
1579
+ project_id: str = PROVIDE_PROJECT_ID,
1580
+ location: str,
1581
+ page_size: int | None = None,
1582
+ page_token: str | None = None,
1583
+ filter_str: str | None = None,
1584
+ timeout: float | None = None,
1585
+ retry: Retry | _MethodDefault = DEFAULT,
1586
+ gcp_conn_id: str = "google_cloud_default",
1587
+ metadata: Sequence[tuple[str, str]] = (),
1588
+ impersonation_chain: str | Sequence[str] | None = None,
1589
+ **kwargs,
1590
+ ) -> None:
1591
+ super().__init__(**kwargs)
1592
+ self.project_id = project_id
1593
+ self.location = location
1594
+ self.page_size = page_size
1595
+ self.page_token = page_token
1596
+ self.filter_str = filter_str
1597
+ self.metadata = metadata
1598
+ self.timeout = timeout
1599
+ self.retry = retry
1600
+ self.gcp_conn_id = gcp_conn_id
1601
+ self.impersonation_chain = impersonation_chain
1602
+
1603
+ def execute(self, context: Context) -> Sequence[str]:
1604
+ hook = TranslateHook(
1605
+ gcp_conn_id=self.gcp_conn_id,
1606
+ impersonation_chain=self.impersonation_chain,
1607
+ )
1608
+ project_id = self.project_id or hook.project_id
1609
+ TranslationGlossariesListLink.persist(
1610
+ context=context,
1611
+ task_instance=self,
1612
+ project_id=project_id,
1613
+ )
1614
+ self.log.info("Requesting glossaries list")
1615
+ try:
1616
+ results_pager = hook.list_glossaries(
1617
+ project_id=project_id,
1618
+ location=self.location,
1619
+ page_size=self.page_size,
1620
+ page_token=self.page_token,
1621
+ filter_str=self.filter_str,
1622
+ retry=self.retry,
1623
+ timeout=self.timeout,
1624
+ metadata=self.metadata,
1625
+ )
1626
+ except GoogleAPICallError as e:
1627
+ self.log.error("Error submitting list_glossaries request")
1628
+ raise AirflowException(e)
1629
+
1630
+ result_ids = []
1631
+ for glossary_item in results_pager:
1632
+ glossary_item = type(glossary_item).to_dict(glossary_item)
1633
+ glossary_id = hook.extract_object_id(glossary_item)
1634
+ result_ids.append(glossary_id)
1635
+ self.log.info("Fetching the glossaries list complete. Glossary id-s: %s", result_ids)
1636
+ return result_ids
1637
+
1638
+
1639
+ class TranslateDeleteGlossaryOperator(GoogleCloudBaseOperator):
1640
+ """
1641
+ Delete a Google Cloud Translation Glossary.
1642
+
1643
+ Deletes a translation glossary, using API V3.
1644
+ For more information on how to use this operator, take a look at the guide:
1645
+ :ref:`howto/operator:TranslateDeleteGlossaryOperator`.
1646
+
1647
+ :param glossary_id: User-specified id to delete glossary resource item.
1648
+ :param project_id: ID of the Google Cloud project where glossary is located.
1649
+ If not provided default project_id is used.
1650
+ :param location: The location of the project.
1651
+ :param retry: Designation of what errors, if any, should be retried.
1652
+ :param timeout: The timeout for this request.
1653
+ :param metadata: Strings which should be sent along with the request as metadata.
1654
+ :param gcp_conn_id: The connection ID to use connecting to Google Cloud.
1655
+ :param impersonation_chain: Optional service account to impersonate using short-term
1656
+ credentials, or chained list of accounts required to get the access_token
1657
+ of the last account in the list, which will be impersonated in the request.
1658
+ If set as a string, the account must grant the originating account
1659
+ the Service Account Token Creator IAM role.
1660
+ If set as a sequence, the identities from the list must grant
1661
+ Service Account Token Creator IAM role to the directly preceding identity, with first
1662
+ account from the list granting this role to the originating account (templated).
1663
+ """
1664
+
1665
+ template_fields: Sequence[str] = (
1666
+ "glossary_id",
1667
+ "location",
1668
+ "project_id",
1669
+ "gcp_conn_id",
1670
+ "impersonation_chain",
1671
+ )
1672
+
1673
+ def __init__(
1674
+ self,
1675
+ *,
1676
+ project_id: str = PROVIDE_PROJECT_ID,
1677
+ location: str,
1678
+ glossary_id: str,
1679
+ timeout: float | None = None,
1680
+ retry: Retry | _MethodDefault = DEFAULT,
1681
+ gcp_conn_id: str = "google_cloud_default",
1682
+ metadata: Sequence[tuple[str, str]] = (),
1683
+ impersonation_chain: str | Sequence[str] | None = None,
1684
+ **kwargs,
1685
+ ) -> None:
1686
+ super().__init__(**kwargs)
1687
+ self.glossary_id = glossary_id
1688
+ self.project_id = project_id
1689
+ self.location = location
1690
+ self.metadata = metadata
1691
+ self.timeout = timeout
1692
+ self.retry = retry
1693
+ self.gcp_conn_id = gcp_conn_id
1694
+ self.impersonation_chain = impersonation_chain
1695
+
1696
+ def execute(self, context: Context) -> str:
1697
+ hook = TranslateHook(
1698
+ gcp_conn_id=self.gcp_conn_id,
1699
+ impersonation_chain=self.impersonation_chain,
1700
+ )
1701
+ project_id = self.project_id or hook.project_id
1702
+ try:
1703
+ result_operation = hook.delete_glossary(
1704
+ glossary_id=self.glossary_id,
1705
+ project_id=project_id,
1706
+ location=self.location,
1707
+ retry=self.retry,
1708
+ timeout=self.timeout,
1709
+ metadata=self.metadata,
1710
+ )
1711
+ except GoogleAPICallError as e:
1712
+ self.log.error("Error submitting delete_glossary operation ")
1713
+ raise AirflowException(e)
1714
+ self.log.info("Glossary delete started, glossary_id %s...", self.glossary_id)
1715
+
1716
+ result = hook.wait_for_operation_result(operation=result_operation)
1717
+ result = type(result).to_dict(result)
1718
+ self.log.info("Glossary deletion complete. The glossary_id: %s.", self.glossary_id)
1719
+ return result