apache-airflow-providers-google 14.0.0rc1__py3-none-any.whl → 14.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
  2. airflow/providers/google/3rd-party-licenses/NOTICE +15 -0
  3. airflow/providers/google/__init__.py +1 -1
  4. airflow/providers/google/_vendor/__init__.py +0 -0
  5. airflow/providers/google/_vendor/json_merge_patch.py +91 -0
  6. airflow/providers/google/ads/hooks/ads.py +12 -11
  7. airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
  8. airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
  9. airflow/providers/google/cloud/hooks/automl.py +7 -13
  10. airflow/providers/google/cloud/hooks/bigquery.py +21 -22
  11. airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
  12. airflow/providers/google/cloud/hooks/bigtable.py +3 -2
  13. airflow/providers/google/cloud/hooks/cloud_batch.py +4 -3
  14. airflow/providers/google/cloud/hooks/cloud_build.py +7 -13
  15. airflow/providers/google/cloud/hooks/cloud_composer.py +7 -12
  16. airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
  17. airflow/providers/google/cloud/hooks/cloud_run.py +4 -3
  18. airflow/providers/google/cloud/hooks/cloud_sql.py +1 -1
  19. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +8 -9
  20. airflow/providers/google/cloud/hooks/compute.py +3 -3
  21. airflow/providers/google/cloud/hooks/datacatalog.py +4 -3
  22. airflow/providers/google/cloud/hooks/dataflow.py +12 -12
  23. airflow/providers/google/cloud/hooks/dataform.py +3 -2
  24. airflow/providers/google/cloud/hooks/datafusion.py +2 -2
  25. airflow/providers/google/cloud/hooks/dataplex.py +11 -10
  26. airflow/providers/google/cloud/hooks/dataproc.py +5 -4
  27. airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
  28. airflow/providers/google/cloud/hooks/dlp.py +4 -3
  29. airflow/providers/google/cloud/hooks/gcs.py +19 -12
  30. airflow/providers/google/cloud/hooks/kms.py +3 -2
  31. airflow/providers/google/cloud/hooks/kubernetes_engine.py +20 -14
  32. airflow/providers/google/cloud/hooks/life_sciences.py +1 -1
  33. airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
  34. airflow/providers/google/cloud/hooks/natural_language.py +3 -2
  35. airflow/providers/google/cloud/hooks/os_login.py +3 -2
  36. airflow/providers/google/cloud/hooks/pubsub.py +6 -6
  37. airflow/providers/google/cloud/hooks/secret_manager.py +3 -2
  38. airflow/providers/google/cloud/hooks/spanner.py +2 -2
  39. airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
  40. airflow/providers/google/cloud/hooks/stackdriver.py +4 -4
  41. airflow/providers/google/cloud/hooks/tasks.py +4 -3
  42. airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
  43. airflow/providers/google/cloud/hooks/translate.py +6 -15
  44. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +6 -12
  45. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -12
  46. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +7 -13
  47. airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
  48. airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +4 -11
  49. airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +4 -3
  50. airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +6 -12
  51. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +6 -11
  52. airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
  53. airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
  54. airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
  55. airflow/providers/google/cloud/hooks/vision.py +4 -3
  56. airflow/providers/google/cloud/hooks/workflows.py +3 -2
  57. airflow/providers/google/cloud/links/base.py +7 -1
  58. airflow/providers/google/cloud/links/datafusion.py +8 -2
  59. airflow/providers/google/cloud/links/dataproc.py +8 -1
  60. airflow/providers/google/cloud/links/kubernetes_engine.py +2 -1
  61. airflow/providers/google/cloud/links/managed_kafka.py +30 -0
  62. airflow/providers/google/cloud/log/gcs_task_handler.py +15 -7
  63. airflow/providers/google/cloud/log/stackdriver_task_handler.py +8 -6
  64. airflow/providers/google/cloud/openlineage/utils.py +4 -2
  65. airflow/providers/google/cloud/operators/alloy_db.py +6 -5
  66. airflow/providers/google/cloud/operators/automl.py +11 -9
  67. airflow/providers/google/cloud/operators/bigquery.py +8 -6
  68. airflow/providers/google/cloud/operators/bigquery_dts.py +10 -8
  69. airflow/providers/google/cloud/operators/bigtable.py +3 -1
  70. airflow/providers/google/cloud/operators/cloud_base.py +2 -1
  71. airflow/providers/google/cloud/operators/cloud_batch.py +4 -2
  72. airflow/providers/google/cloud/operators/cloud_build.py +5 -3
  73. airflow/providers/google/cloud/operators/cloud_composer.py +7 -5
  74. airflow/providers/google/cloud/operators/cloud_memorystore.py +6 -4
  75. airflow/providers/google/cloud/operators/cloud_run.py +5 -3
  76. airflow/providers/google/cloud/operators/compute.py +5 -4
  77. airflow/providers/google/cloud/operators/datacatalog.py +11 -9
  78. airflow/providers/google/cloud/operators/dataform.py +5 -3
  79. airflow/providers/google/cloud/operators/datafusion.py +1 -1
  80. airflow/providers/google/cloud/operators/dataplex.py +20 -18
  81. airflow/providers/google/cloud/operators/dataproc.py +34 -5
  82. airflow/providers/google/cloud/operators/dataproc_metastore.py +18 -7
  83. airflow/providers/google/cloud/operators/dlp.py +19 -17
  84. airflow/providers/google/cloud/operators/gcs.py +5 -4
  85. airflow/providers/google/cloud/operators/kubernetes_engine.py +54 -5
  86. airflow/providers/google/cloud/operators/managed_kafka.py +271 -4
  87. airflow/providers/google/cloud/operators/natural_language.py +5 -3
  88. airflow/providers/google/cloud/operators/pubsub.py +10 -8
  89. airflow/providers/google/cloud/operators/speech_to_text.py +5 -3
  90. airflow/providers/google/cloud/operators/stackdriver.py +5 -3
  91. airflow/providers/google/cloud/operators/tasks.py +6 -4
  92. airflow/providers/google/cloud/operators/text_to_speech.py +4 -2
  93. airflow/providers/google/cloud/operators/translate.py +5 -3
  94. airflow/providers/google/cloud/operators/translate_speech.py +4 -2
  95. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +8 -6
  96. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +6 -4
  97. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +8 -6
  98. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +6 -4
  99. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +6 -4
  100. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +6 -4
  101. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +6 -4
  102. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +6 -4
  103. airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
  104. airflow/providers/google/cloud/operators/vision.py +6 -4
  105. airflow/providers/google/cloud/operators/workflows.py +7 -5
  106. airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
  107. airflow/providers/google/cloud/sensors/bigquery_dts.py +5 -3
  108. airflow/providers/google/cloud/sensors/bigtable.py +3 -2
  109. airflow/providers/google/cloud/sensors/cloud_composer.py +1 -1
  110. airflow/providers/google/cloud/sensors/dataplex.py +6 -4
  111. airflow/providers/google/cloud/sensors/dataproc.py +3 -2
  112. airflow/providers/google/cloud/sensors/dataproc_metastore.py +2 -1
  113. airflow/providers/google/cloud/sensors/gcs.py +4 -2
  114. airflow/providers/google/cloud/sensors/pubsub.py +3 -2
  115. airflow/providers/google/cloud/sensors/workflows.py +5 -3
  116. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +5 -3
  117. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +12 -10
  118. airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
  119. airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
  120. airflow/providers/google/cloud/triggers/cloud_build.py +2 -1
  121. airflow/providers/google/cloud/triggers/cloud_composer.py +1 -1
  122. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +3 -2
  123. airflow/providers/google/cloud/triggers/dataflow.py +3 -2
  124. airflow/providers/google/cloud/triggers/dataplex.py +2 -1
  125. airflow/providers/google/cloud/triggers/dataproc.py +3 -2
  126. airflow/providers/google/cloud/triggers/kubernetes_engine.py +1 -1
  127. airflow/providers/google/cloud/triggers/pubsub.py +2 -1
  128. airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
  129. airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
  130. airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
  131. airflow/providers/google/cloud/{example_dags/__init__.py → utils/validators.py} +27 -0
  132. airflow/providers/google/common/auth_backend/google_openid.py +14 -5
  133. airflow/providers/google/common/consts.py +2 -1
  134. airflow/providers/google/common/hooks/base_google.py +7 -8
  135. airflow/providers/google/common/hooks/operation_helpers.py +79 -0
  136. airflow/providers/google/get_provider_info.py +11 -6
  137. airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
  138. airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
  139. airflow/providers/google/marketing_platform/links/analytics_admin.py +10 -2
  140. airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -5
  141. {apache_airflow_providers_google-14.0.0rc1.dist-info → apache_airflow_providers_google-14.1.0rc1.dist-info}/METADATA +32 -35
  142. {apache_airflow_providers_google-14.0.0rc1.dist-info → apache_airflow_providers_google-14.1.0rc1.dist-info}/RECORD +144 -140
  143. {apache_airflow_providers_google-14.0.0rc1.dist-info → apache_airflow_providers_google-14.1.0rc1.dist-info}/WHEEL +1 -1
  144. airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
  145. {apache_airflow_providers_google-14.0.0rc1.dist-info → apache_airflow_providers_google-14.1.0rc1.dist-info}/entry_points.txt +0 -0
@@ -27,11 +27,12 @@ from __future__ import annotations
27
27
  from collections.abc import Sequence
28
28
  from typing import TYPE_CHECKING
29
29
 
30
- from airflow.providers.google.common.consts import CLIENT_INFO
31
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
32
30
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
33
31
  from google.cloud.oslogin_v1 import ImportSshPublicKeyResponse, OsLoginServiceClient
34
32
 
33
+ from airflow.providers.google.common.consts import CLIENT_INFO
34
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
35
+
35
36
  if TYPE_CHECKING:
36
37
  from google.api_core.retry import Retry
37
38
 
@@ -33,6 +33,12 @@ from functools import cached_property
33
33
  from typing import TYPE_CHECKING, Any
34
34
  from uuid import uuid4
35
35
 
36
+ from google.api_core.exceptions import AlreadyExists, GoogleAPICallError
37
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
38
+ from google.cloud.exceptions import NotFound
39
+ from google.cloud.pubsub_v1 import PublisherClient, SubscriberClient
40
+ from google.cloud.pubsub_v1.types import PublisherOptions
41
+ from google.pubsub_v1.services.subscriber.async_client import SubscriberAsyncClient
36
42
  from googleapiclient.errors import HttpError
37
43
 
38
44
  from airflow.exceptions import AirflowProviderDeprecationWarning
@@ -43,12 +49,6 @@ from airflow.providers.google.common.hooks.base_google import (
43
49
  GoogleBaseHook,
44
50
  )
45
51
  from airflow.version import version
46
- from google.api_core.exceptions import AlreadyExists, GoogleAPICallError
47
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
48
- from google.cloud.exceptions import NotFound
49
- from google.cloud.pubsub_v1 import PublisherClient, SubscriberClient
50
- from google.cloud.pubsub_v1.types import PublisherOptions
51
- from google.pubsub_v1.services.subscriber.async_client import SubscriberAsyncClient
52
52
 
53
53
  if TYPE_CHECKING:
54
54
  from google.api_core.retry import Retry
@@ -23,8 +23,6 @@ from collections.abc import Sequence
23
23
  from functools import cached_property
24
24
  from typing import TYPE_CHECKING
25
25
 
26
- from airflow.providers.google.common.consts import CLIENT_INFO
27
- from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
28
26
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
29
27
  from google.cloud.secretmanager_v1 import (
30
28
  AccessSecretVersionResponse,
@@ -34,6 +32,9 @@ from google.cloud.secretmanager_v1 import (
34
32
  SecretVersion,
35
33
  )
36
34
 
35
+ from airflow.providers.google.common.consts import CLIENT_INFO
36
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
37
+
37
38
  if TYPE_CHECKING:
38
39
  from google.api_core.retry import Retry
39
40
  from google.cloud.secretmanager_v1.services.secret_manager_service.pagers import ListSecretsPager
@@ -22,14 +22,14 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING, Callable, NamedTuple
24
24
 
25
+ from google.api_core.exceptions import AlreadyExists, GoogleAPICallError
26
+ from google.cloud.spanner_v1.client import Client
25
27
  from sqlalchemy import create_engine
26
28
 
27
29
  from airflow.exceptions import AirflowException
28
30
  from airflow.providers.common.sql.hooks.sql import DbApiHook
29
31
  from airflow.providers.google.common.consts import CLIENT_INFO
30
32
  from airflow.providers.google.common.hooks.base_google import GoogleBaseHook, get_field
31
- from google.api_core.exceptions import AlreadyExists, GoogleAPICallError
32
- from google.cloud.spanner_v1.client import Client
33
33
 
34
34
  if TYPE_CHECKING:
35
35
  from google.cloud.spanner_v1.database import Database
@@ -22,12 +22,13 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
- from airflow.providers.google.common.consts import CLIENT_INFO
26
- from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
27
25
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
28
26
  from google.cloud.speech_v1 import SpeechClient
29
27
  from google.cloud.speech_v1.types import RecognitionAudio, RecognitionConfig
30
28
 
29
+ from airflow.providers.google.common.consts import CLIENT_INFO
30
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
31
+
31
32
  if TYPE_CHECKING:
32
33
  from google.api_core.retry import Retry
33
34
 
@@ -24,15 +24,15 @@ import json
24
24
  from collections.abc import Sequence
25
25
  from typing import TYPE_CHECKING, Any
26
26
 
27
- from googleapiclient.errors import HttpError
28
-
29
- from airflow.exceptions import AirflowException
30
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
31
27
  from google.api_core.exceptions import InvalidArgument
32
28
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
33
29
  from google.cloud import monitoring_v3
34
30
  from google.cloud.monitoring_v3 import AlertPolicy, NotificationChannel
35
31
  from google.protobuf.field_mask_pb2 import FieldMask
32
+ from googleapiclient.errors import HttpError
33
+
34
+ from airflow.exceptions import AirflowException
35
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
36
36
 
37
37
  if TYPE_CHECKING:
38
38
  from google.api_core.retry import Retry
@@ -22,13 +22,14 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
- from airflow.exceptions import AirflowException
26
- from airflow.providers.google.common.consts import CLIENT_INFO
27
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
28
25
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
29
26
  from google.cloud.tasks_v2 import CloudTasksClient
30
27
  from google.cloud.tasks_v2.types import Queue, Task
31
28
 
29
+ from airflow.exceptions import AirflowException
30
+ from airflow.providers.google.common.consts import CLIENT_INFO
31
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
32
+
32
33
  if TYPE_CHECKING:
33
34
  from google.api_core.retry import Retry
34
35
  from google.protobuf.field_mask_pb2 import FieldMask
@@ -22,8 +22,6 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
- from airflow.providers.google.common.consts import CLIENT_INFO
26
- from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
27
25
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
28
26
  from google.cloud.texttospeech_v1 import TextToSpeechClient
29
27
  from google.cloud.texttospeech_v1.types import (
@@ -33,6 +31,9 @@ from google.cloud.texttospeech_v1.types import (
33
31
  VoiceSelectionParams,
34
32
  )
35
33
 
34
+ from airflow.providers.google.common.consts import CLIENT_INFO
35
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
36
+
36
37
  if TYPE_CHECKING:
37
38
  from google.api_core.retry import Retry
38
39
 
@@ -25,9 +25,6 @@ from typing import (
25
25
  cast,
26
26
  )
27
27
 
28
- from airflow.exceptions import AirflowException
29
- from airflow.providers.google.common.consts import CLIENT_INFO
30
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
31
28
  from google.api_core.exceptions import GoogleAPICallError
32
29
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
33
30
  from google.api_core.retry import Retry
@@ -35,9 +32,12 @@ from google.cloud.translate_v2 import Client
35
32
  from google.cloud.translate_v3 import TranslationServiceClient
36
33
  from google.cloud.translate_v3.types.translation_service import GlossaryInputConfig
37
34
 
38
- if TYPE_CHECKING:
39
- from proto import Message
35
+ from airflow.exceptions import AirflowException
36
+ from airflow.providers.google.common.consts import CLIENT_INFO
37
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
38
+ from airflow.providers.google.common.hooks.operation_helpers import OperationHelper
40
39
 
40
+ if TYPE_CHECKING:
41
41
  from google.api_core.operation import Operation
42
42
  from google.cloud.translate_v3.services.translation_service import pagers
43
43
  from google.cloud.translate_v3.types import (
@@ -155,7 +155,7 @@ class CloudTranslateHook(GoogleBaseHook):
155
155
  )
156
156
 
157
157
 
158
- class TranslateHook(GoogleBaseHook):
158
+ class TranslateHook(GoogleBaseHook, OperationHelper):
159
159
  """
160
160
  Hook for Google Cloud translation (Advanced) using client version V3.
161
161
 
@@ -221,15 +221,6 @@ class TranslateHook(GoogleBaseHook):
221
221
  error = operation.exception(timeout=timeout)
222
222
  raise AirflowException(error)
223
223
 
224
- @staticmethod
225
- def wait_for_operation_result(operation: Operation, timeout: int | None = None) -> Message:
226
- """Wait for long-lasting operation to complete."""
227
- try:
228
- return operation.result(timeout=timeout)
229
- except GoogleAPICallError:
230
- error = operation.exception(timeout=timeout)
231
- raise AirflowException(error)
232
-
233
224
  @staticmethod
234
225
  def extract_object_id(obj: dict) -> str:
235
226
  """Return unique id of the object."""
@@ -23,9 +23,6 @@ import warnings
23
23
  from collections.abc import Sequence
24
24
  from typing import TYPE_CHECKING
25
25
 
26
- from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
27
- from airflow.providers.google.common.deprecated import deprecated
28
- from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
29
26
  from google.api_core.client_options import ClientOptions
30
27
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
31
28
  from google.cloud.aiplatform import (
@@ -39,6 +36,11 @@ from google.cloud.aiplatform import (
39
36
  )
40
37
  from google.cloud.aiplatform_v1 import JobServiceClient, PipelineServiceClient
41
38
 
39
+ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
40
+ from airflow.providers.google.common.deprecated import deprecated
41
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
42
+ from airflow.providers.google.common.hooks.operation_helpers import OperationHelper
43
+
42
44
  if TYPE_CHECKING:
43
45
  from google.api_core.operation import Operation
44
46
  from google.api_core.retry import Retry
@@ -46,7 +48,7 @@ if TYPE_CHECKING:
46
48
  from google.cloud.aiplatform_v1.types import TrainingPipeline
47
49
 
48
50
 
49
- class AutoMLHook(GoogleBaseHook):
51
+ class AutoMLHook(GoogleBaseHook, OperationHelper):
50
52
  """Hook for Google Cloud Vertex AI Auto ML APIs."""
51
53
 
52
54
  def __init__(
@@ -252,14 +254,6 @@ class AutoMLHook(GoogleBaseHook):
252
254
  """Return unique id of the Training pipeline."""
253
255
  return resource_name.rpartition("/")[-1]
254
256
 
255
- def wait_for_operation(self, operation: Operation, timeout: float | None = None):
256
- """Wait for long-lasting operation to complete."""
257
- try:
258
- return operation.result(timeout=timeout)
259
- except Exception:
260
- error = operation.exception(timeout=timeout)
261
- raise AirflowException(error)
262
-
263
257
  def cancel_auto_ml_job(self) -> None:
264
258
  """Cancel Auto ML Job for training pipeline."""
265
259
  if self._job:
@@ -23,21 +23,23 @@ import asyncio
23
23
  from collections.abc import Sequence
24
24
  from typing import TYPE_CHECKING
25
25
 
26
- from airflow.exceptions import AirflowException
27
- from airflow.providers.google.common.consts import CLIENT_INFO
28
- from airflow.providers.google.common.hooks.base_google import GoogleBaseAsyncHook, GoogleBaseHook
29
26
  from google.api_core.client_options import ClientOptions
30
27
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
31
28
  from google.cloud.aiplatform import BatchPredictionJob, Model, explain
32
29
  from google.cloud.aiplatform_v1 import JobServiceAsyncClient, JobServiceClient, JobState, types
33
30
 
31
+ from airflow.exceptions import AirflowException
32
+ from airflow.providers.google.common.consts import CLIENT_INFO
33
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseAsyncHook, GoogleBaseHook
34
+ from airflow.providers.google.common.hooks.operation_helpers import OperationHelper
35
+
34
36
  if TYPE_CHECKING:
35
37
  from google.api_core.operation import Operation
36
38
  from google.api_core.retry import AsyncRetry, Retry
37
39
  from google.cloud.aiplatform_v1.services.job_service.pagers import ListBatchPredictionJobsPager
38
40
 
39
41
 
40
- class BatchPredictionJobHook(GoogleBaseHook):
42
+ class BatchPredictionJobHook(GoogleBaseHook, OperationHelper):
41
43
  """Hook for Google Cloud Vertex AI Batch Prediction Job APIs."""
42
44
 
43
45
  def __init__(
@@ -64,14 +66,6 @@ class BatchPredictionJobHook(GoogleBaseHook):
64
66
  credentials=self.get_credentials(), client_info=self.client_info, client_options=client_options
65
67
  )
66
68
 
67
- def wait_for_operation(self, operation: Operation, timeout: float | None = None):
68
- """Wait for long-lasting operation to complete."""
69
- try:
70
- return operation.result(timeout=timeout)
71
- except Exception:
72
- error = operation.exception(timeout=timeout)
73
- raise AirflowException(error)
74
-
75
69
  @staticmethod
76
70
  def extract_batch_prediction_job_id(obj: dict) -> str:
77
71
  """Return unique id of the batch_prediction_job."""
@@ -23,10 +23,6 @@ import asyncio
23
23
  from collections.abc import Sequence
24
24
  from typing import TYPE_CHECKING, Any
25
25
 
26
- from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
27
- from airflow.providers.google.common.consts import CLIENT_INFO
28
- from airflow.providers.google.common.deprecated import deprecated
29
- from airflow.providers.google.common.hooks.base_google import GoogleBaseAsyncHook, GoogleBaseHook
30
26
  from google.api_core.client_options import ClientOptions
31
27
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
32
28
  from google.cloud.aiplatform import (
@@ -46,6 +42,12 @@ from google.cloud.aiplatform_v1 import (
46
42
  types,
47
43
  )
48
44
 
45
+ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
46
+ from airflow.providers.google.common.consts import CLIENT_INFO
47
+ from airflow.providers.google.common.deprecated import deprecated
48
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseAsyncHook, GoogleBaseHook
49
+ from airflow.providers.google.common.hooks.operation_helpers import OperationHelper
50
+
49
51
  if TYPE_CHECKING:
50
52
  from google.api_core.operation import Operation
51
53
  from google.api_core.retry import AsyncRetry, Retry
@@ -58,7 +60,7 @@ if TYPE_CHECKING:
58
60
  from google.cloud.aiplatform_v1.types import CustomJob, PipelineJob, TrainingPipeline
59
61
 
60
62
 
61
- class CustomJobHook(GoogleBaseHook):
63
+ class CustomJobHook(GoogleBaseHook, OperationHelper):
62
64
  """Hook for Google Cloud Vertex AI Custom Job APIs."""
63
65
 
64
66
  def __init__(
@@ -276,14 +278,6 @@ class CustomJobHook(GoogleBaseHook):
276
278
  """Return a unique Custom Job id from a serialized TrainingPipeline proto."""
277
279
  return training_pipeline["training_task_metadata"]["backingCustomJob"].rpartition("/")[-1]
278
280
 
279
- def wait_for_operation(self, operation: Operation, timeout: float | None = None):
280
- """Wait for long-lasting operation to complete."""
281
- try:
282
- return operation.result(timeout=timeout)
283
- except Exception:
284
- error = operation.exception(timeout=timeout)
285
- raise AirflowException(error)
286
-
287
281
  def cancel_job(self) -> None:
288
282
  """Cancel Job for training pipeline."""
289
283
  if self._job:
@@ -22,13 +22,14 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
- from airflow.exceptions import AirflowException
26
- from airflow.providers.google.common.consts import CLIENT_INFO
27
- from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
28
25
  from google.api_core.client_options import ClientOptions
29
26
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
30
27
  from google.cloud.aiplatform_v1 import DatasetServiceClient
31
28
 
29
+ from airflow.providers.google.common.consts import CLIENT_INFO
30
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
31
+ from airflow.providers.google.common.hooks.operation_helpers import OperationHelper
32
+
32
33
  if TYPE_CHECKING:
33
34
  from google.api_core.operation import Operation
34
35
  from google.api_core.retry import Retry
@@ -41,7 +42,7 @@ if TYPE_CHECKING:
41
42
  from google.protobuf.field_mask_pb2 import FieldMask
42
43
 
43
44
 
44
- class DatasetHook(GoogleBaseHook):
45
+ class DatasetHook(GoogleBaseHook, OperationHelper):
45
46
  """Hook for Google Cloud Vertex AI Dataset APIs."""
46
47
 
47
48
  def get_dataset_service_client(self, region: str | None = None) -> DatasetServiceClient:
@@ -55,14 +56,6 @@ class DatasetHook(GoogleBaseHook):
55
56
  credentials=self.get_credentials(), client_info=CLIENT_INFO, client_options=client_options
56
57
  )
57
58
 
58
- def wait_for_operation(self, operation: Operation, timeout: float | None = None):
59
- """Wait for long-lasting operation to complete."""
60
- try:
61
- return operation.result(timeout=timeout)
62
- except Exception:
63
- error = operation.exception(timeout=timeout)
64
- raise AirflowException(error)
65
-
66
59
  @staticmethod
67
60
  def extract_dataset_id(obj: dict) -> str:
68
61
  """Return unique id of the dataset."""
@@ -22,12 +22,13 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
- from airflow.exceptions import AirflowException
26
- from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
27
25
  from google.api_core.client_options import ClientOptions
28
26
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
29
27
  from google.cloud.aiplatform_v1 import EndpointServiceClient
30
28
 
29
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
30
+ from airflow.providers.google.common.hooks.operation_helpers import OperationHelper
31
+
31
32
  if TYPE_CHECKING:
32
33
  from google.api_core.operation import Operation
33
34
  from google.api_core.retry import Retry
@@ -36,7 +37,7 @@ if TYPE_CHECKING:
36
37
  from google.protobuf.field_mask_pb2 import FieldMask
37
38
 
38
39
 
39
- class EndpointServiceHook(GoogleBaseHook):
40
+ class EndpointServiceHook(GoogleBaseHook, OperationHelper):
40
41
  """Hook for Google Cloud Vertex AI Endpoint Service APIs."""
41
42
 
42
43
  def get_endpoint_service_client(self, region: str | None = None) -> EndpointServiceClient:
@@ -50,14 +51,6 @@ class EndpointServiceHook(GoogleBaseHook):
50
51
  credentials=self.get_credentials(), client_info=self.client_info, client_options=client_options
51
52
  )
52
53
 
53
- def wait_for_operation(self, operation: Operation, timeout: float | None = None):
54
- """Wait for long-lasting operation to complete."""
55
- try:
56
- return operation.result(timeout=timeout)
57
- except Exception:
58
- error = operation.exception(timeout=timeout)
59
- raise AirflowException(error)
60
-
61
54
  @staticmethod
62
55
  def extract_endpoint_id(obj: dict) -> str:
63
56
  """Return unique id of the endpoint."""
@@ -18,14 +18,15 @@
18
18
 
19
19
  from __future__ import annotations
20
20
 
21
- from airflow.exceptions import AirflowException
22
- from airflow.providers.google.common.consts import CLIENT_INFO
23
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
24
21
  from google.api_core.client_options import ClientOptions
25
22
  from google.cloud.aiplatform_v1beta1 import (
26
23
  FeatureOnlineStoreAdminServiceClient,
27
24
  )
28
25
 
26
+ from airflow.exceptions import AirflowException
27
+ from airflow.providers.google.common.consts import CLIENT_INFO
28
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
29
+
29
30
 
30
31
  class FeatureStoreHook(GoogleBaseHook):
31
32
  """
@@ -29,21 +29,23 @@ import asyncio
29
29
  from collections.abc import Sequence
30
30
  from typing import TYPE_CHECKING
31
31
 
32
- from airflow.exceptions import AirflowException
33
- from airflow.providers.google.common.consts import CLIENT_INFO
34
- from airflow.providers.google.common.hooks.base_google import GoogleBaseAsyncHook, GoogleBaseHook
35
32
  from google.api_core.client_options import ClientOptions
36
33
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
37
34
  from google.cloud.aiplatform import CustomJob, HyperparameterTuningJob, gapic, hyperparameter_tuning
38
35
  from google.cloud.aiplatform_v1 import JobServiceAsyncClient, JobServiceClient, JobState, types
39
36
 
37
+ from airflow.exceptions import AirflowException
38
+ from airflow.providers.google.common.consts import CLIENT_INFO
39
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseAsyncHook, GoogleBaseHook
40
+ from airflow.providers.google.common.hooks.operation_helpers import OperationHelper
41
+
40
42
  if TYPE_CHECKING:
41
43
  from google.api_core.operation import Operation
42
44
  from google.api_core.retry import AsyncRetry, Retry
43
45
  from google.cloud.aiplatform_v1.services.job_service.pagers import ListHyperparameterTuningJobsPager
44
46
 
45
47
 
46
- class HyperparameterTuningJobHook(GoogleBaseHook):
48
+ class HyperparameterTuningJobHook(GoogleBaseHook, OperationHelper):
47
49
  """Hook for Google Cloud Vertex AI Hyperparameter Tuning Job APIs."""
48
50
 
49
51
  def __init__(
@@ -133,14 +135,6 @@ class HyperparameterTuningJobHook(GoogleBaseHook):
133
135
  """Return unique id of the hyperparameter_tuning_job."""
134
136
  return obj["name"].rpartition("/")[-1]
135
137
 
136
- def wait_for_operation(self, operation: Operation, timeout: float | None = None):
137
- """Wait for long-lasting operation to complete."""
138
- try:
139
- return operation.result(timeout=timeout)
140
- except Exception:
141
- error = operation.exception(timeout=timeout)
142
- raise AirflowException(error)
143
-
144
138
  def cancel_hyperparameter_tuning_job(self) -> None:
145
139
  """Cancel HyperparameterTuningJob."""
146
140
  if self._hyperparameter_tuning_job:
@@ -23,12 +23,13 @@ from __future__ import annotations
23
23
  from collections.abc import Sequence
24
24
  from typing import TYPE_CHECKING
25
25
 
26
- from airflow.exceptions import AirflowException
27
- from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
28
26
  from google.api_core.client_options import ClientOptions
29
27
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
30
28
  from google.cloud.aiplatform_v1 import ModelServiceClient
31
29
 
30
+ from airflow.exceptions import AirflowException
31
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
32
+
32
33
  if TYPE_CHECKING:
33
34
  from google.api_core.operation import Operation
34
35
  from google.api_core.retry import Retry
@@ -38,8 +39,10 @@ if TYPE_CHECKING:
38
39
  )
39
40
  from google.cloud.aiplatform_v1.types import Model, model_service
40
41
 
42
+ from airflow.providers.google.common.hooks.operation_helpers import OperationHelper
43
+
41
44
 
42
- class ModelServiceHook(GoogleBaseHook):
45
+ class ModelServiceHook(GoogleBaseHook, OperationHelper):
43
46
  """Hook for Google Cloud Vertex AI Endpoint Service APIs."""
44
47
 
45
48
  def get_model_service_client(self, region: str | None = None) -> ModelServiceClient:
@@ -58,14 +61,6 @@ class ModelServiceHook(GoogleBaseHook):
58
61
  """Return unique id of the model."""
59
62
  return obj["model"].rpartition("/")[-1]
60
63
 
61
- def wait_for_operation(self, operation: Operation, timeout: float | None = None):
62
- """Wait for long-lasting operation to complete."""
63
- try:
64
- return operation.result(timeout=timeout)
65
- except Exception:
66
- error = operation.exception(timeout=timeout)
67
- raise AirflowException(error)
68
-
69
64
  @GoogleBaseHook.fallback_to_default_project_id
70
65
  def delete_model(
71
66
  self,
@@ -29,9 +29,6 @@ import asyncio
29
29
  from collections.abc import Sequence
30
30
  from typing import TYPE_CHECKING, Any
31
31
 
32
- from airflow.exceptions import AirflowException
33
- from airflow.providers.google.common.consts import CLIENT_INFO
34
- from airflow.providers.google.common.hooks.base_google import GoogleBaseAsyncHook, GoogleBaseHook
35
32
  from google.api_core.client_options import ClientOptions
36
33
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
37
34
  from google.cloud.aiplatform import PipelineJob
@@ -42,6 +39,11 @@ from google.cloud.aiplatform_v1 import (
42
39
  types,
43
40
  )
44
41
 
42
+ from airflow.exceptions import AirflowException
43
+ from airflow.providers.google.common.consts import CLIENT_INFO
44
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseAsyncHook, GoogleBaseHook
45
+ from airflow.providers.google.common.hooks.operation_helpers import OperationHelper
46
+
45
47
  if TYPE_CHECKING:
46
48
  from google.api_core.operation import Operation
47
49
  from google.api_core.retry import AsyncRetry, Retry
@@ -50,7 +52,7 @@ if TYPE_CHECKING:
50
52
  from google.cloud.aiplatform_v1.services.pipeline_service.pagers import ListPipelineJobsPager
51
53
 
52
54
 
53
- class PipelineJobHook(GoogleBaseHook):
55
+ class PipelineJobHook(GoogleBaseHook, OperationHelper):
54
56
  """Hook for Google Cloud Vertex AI Pipeline Job APIs."""
55
57
 
56
58
  def __init__(
@@ -111,14 +113,6 @@ class PipelineJobHook(GoogleBaseHook):
111
113
  failure_policy=failure_policy,
112
114
  )
113
115
 
114
- def wait_for_operation(self, operation: Operation, timeout: float | None = None):
115
- """Wait for long-lasting operation to complete."""
116
- try:
117
- return operation.result(timeout=timeout)
118
- except Exception:
119
- error = operation.exception(timeout=timeout)
120
- raise AirflowException(error)
121
-
122
116
  def cancel_pipeline_job(self) -> None:
123
117
  """Cancel PipelineJob."""
124
118
  if self._pipeline_job:
@@ -20,12 +20,13 @@ from __future__ import annotations
20
20
  from collections.abc import Sequence
21
21
  from typing import TYPE_CHECKING
22
22
 
23
- from airflow.providers.google.common.consts import CLIENT_INFO
24
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
25
23
  from google.api_core.client_options import ClientOptions
26
24
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
27
25
  from google.cloud.aiplatform_v1 import PredictionServiceClient
28
26
 
27
+ from airflow.providers.google.common.consts import CLIENT_INFO
28
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
29
+
29
30
  if TYPE_CHECKING:
30
31
  from google.api_core.retry import Retry
31
32
  from google.cloud.aiplatform_v1.types import PredictResponse
@@ -22,8 +22,6 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
- from airflow.providers.google.common.consts import CLIENT_INFO
26
- from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
27
25
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
28
26
  from google.cloud.videointelligence_v1 import (
29
27
  Feature,
@@ -31,6 +29,9 @@ from google.cloud.videointelligence_v1 import (
31
29
  VideoIntelligenceServiceClient,
32
30
  )
33
31
 
32
+ from airflow.providers.google.common.consts import CLIENT_INFO
33
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
34
+
34
35
  if TYPE_CHECKING:
35
36
  from google.api_core.operation import Operation
36
37
  from google.api_core.retry import Retry
@@ -24,9 +24,6 @@ from copy import deepcopy
24
24
  from functools import cached_property
25
25
  from typing import TYPE_CHECKING, Any, Callable
26
26
 
27
- from airflow.exceptions import AirflowException
28
- from airflow.providers.google.common.consts import CLIENT_INFO
29
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
30
27
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
31
28
  from google.cloud.vision_v1 import (
32
29
  AnnotateImageRequest,
@@ -39,6 +36,10 @@ from google.cloud.vision_v1 import (
39
36
  )
40
37
  from google.protobuf.json_format import MessageToDict
41
38
 
39
+ from airflow.exceptions import AirflowException
40
+ from airflow.providers.google.common.consts import CLIENT_INFO
41
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
42
+
42
43
  if TYPE_CHECKING:
43
44
  from google.api_core.retry import Retry
44
45
  from google.protobuf import field_mask_pb2
@@ -19,12 +19,13 @@ from __future__ import annotations
19
19
  from collections.abc import Sequence
20
20
  from typing import TYPE_CHECKING
21
21
 
22
- from airflow.providers.google.common.consts import CLIENT_INFO
23
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
24
22
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
25
23
  from google.cloud.workflows.executions_v1beta import Execution, ExecutionsClient
26
24
  from google.cloud.workflows_v1beta import Workflow, WorkflowsClient
27
25
 
26
+ from airflow.providers.google.common.consts import CLIENT_INFO
27
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
28
+
28
29
  if TYPE_CHECKING:
29
30
  from google.api_core.operation import Operation
30
31
  from google.api_core.retry import Retry