apache-airflow-providers-google 14.0.0__py3-none-any.whl → 14.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
  2. airflow/providers/google/3rd-party-licenses/NOTICE +15 -0
  3. airflow/providers/google/__init__.py +1 -1
  4. airflow/providers/google/_vendor/__init__.py +0 -0
  5. airflow/providers/google/_vendor/json_merge_patch.py +91 -0
  6. airflow/providers/google/ads/hooks/ads.py +12 -11
  7. airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
  8. airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
  9. airflow/providers/google/cloud/hooks/automl.py +7 -13
  10. airflow/providers/google/cloud/hooks/bigquery.py +21 -22
  11. airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
  12. airflow/providers/google/cloud/hooks/bigtable.py +3 -2
  13. airflow/providers/google/cloud/hooks/cloud_batch.py +4 -3
  14. airflow/providers/google/cloud/hooks/cloud_build.py +7 -13
  15. airflow/providers/google/cloud/hooks/cloud_composer.py +7 -12
  16. airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
  17. airflow/providers/google/cloud/hooks/cloud_run.py +4 -3
  18. airflow/providers/google/cloud/hooks/cloud_sql.py +1 -1
  19. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +8 -9
  20. airflow/providers/google/cloud/hooks/compute.py +3 -3
  21. airflow/providers/google/cloud/hooks/datacatalog.py +4 -3
  22. airflow/providers/google/cloud/hooks/dataflow.py +12 -12
  23. airflow/providers/google/cloud/hooks/dataform.py +3 -2
  24. airflow/providers/google/cloud/hooks/datafusion.py +2 -2
  25. airflow/providers/google/cloud/hooks/dataplex.py +11 -10
  26. airflow/providers/google/cloud/hooks/dataproc.py +5 -4
  27. airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
  28. airflow/providers/google/cloud/hooks/dlp.py +4 -3
  29. airflow/providers/google/cloud/hooks/gcs.py +19 -12
  30. airflow/providers/google/cloud/hooks/kms.py +3 -2
  31. airflow/providers/google/cloud/hooks/kubernetes_engine.py +20 -14
  32. airflow/providers/google/cloud/hooks/life_sciences.py +1 -1
  33. airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
  34. airflow/providers/google/cloud/hooks/natural_language.py +3 -2
  35. airflow/providers/google/cloud/hooks/os_login.py +3 -2
  36. airflow/providers/google/cloud/hooks/pubsub.py +6 -6
  37. airflow/providers/google/cloud/hooks/secret_manager.py +3 -2
  38. airflow/providers/google/cloud/hooks/spanner.py +2 -2
  39. airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
  40. airflow/providers/google/cloud/hooks/stackdriver.py +4 -4
  41. airflow/providers/google/cloud/hooks/tasks.py +4 -3
  42. airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
  43. airflow/providers/google/cloud/hooks/translate.py +6 -15
  44. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +6 -12
  45. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -12
  46. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +7 -13
  47. airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
  48. airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +4 -11
  49. airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +4 -3
  50. airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +6 -12
  51. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +6 -11
  52. airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
  53. airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
  54. airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
  55. airflow/providers/google/cloud/hooks/vision.py +4 -3
  56. airflow/providers/google/cloud/hooks/workflows.py +3 -2
  57. airflow/providers/google/cloud/links/base.py +7 -1
  58. airflow/providers/google/cloud/links/datafusion.py +8 -2
  59. airflow/providers/google/cloud/links/dataproc.py +8 -1
  60. airflow/providers/google/cloud/links/kubernetes_engine.py +2 -1
  61. airflow/providers/google/cloud/links/managed_kafka.py +30 -0
  62. airflow/providers/google/cloud/log/gcs_task_handler.py +15 -7
  63. airflow/providers/google/cloud/log/stackdriver_task_handler.py +8 -6
  64. airflow/providers/google/cloud/openlineage/utils.py +4 -2
  65. airflow/providers/google/cloud/operators/alloy_db.py +6 -5
  66. airflow/providers/google/cloud/operators/automl.py +11 -9
  67. airflow/providers/google/cloud/operators/bigquery.py +8 -6
  68. airflow/providers/google/cloud/operators/bigquery_dts.py +10 -8
  69. airflow/providers/google/cloud/operators/bigtable.py +3 -1
  70. airflow/providers/google/cloud/operators/cloud_base.py +2 -1
  71. airflow/providers/google/cloud/operators/cloud_batch.py +4 -2
  72. airflow/providers/google/cloud/operators/cloud_build.py +5 -3
  73. airflow/providers/google/cloud/operators/cloud_composer.py +7 -5
  74. airflow/providers/google/cloud/operators/cloud_memorystore.py +6 -4
  75. airflow/providers/google/cloud/operators/cloud_run.py +5 -3
  76. airflow/providers/google/cloud/operators/compute.py +5 -4
  77. airflow/providers/google/cloud/operators/datacatalog.py +11 -9
  78. airflow/providers/google/cloud/operators/dataform.py +5 -3
  79. airflow/providers/google/cloud/operators/datafusion.py +1 -1
  80. airflow/providers/google/cloud/operators/dataplex.py +20 -18
  81. airflow/providers/google/cloud/operators/dataproc.py +34 -5
  82. airflow/providers/google/cloud/operators/dataproc_metastore.py +18 -7
  83. airflow/providers/google/cloud/operators/dlp.py +19 -17
  84. airflow/providers/google/cloud/operators/gcs.py +5 -4
  85. airflow/providers/google/cloud/operators/kubernetes_engine.py +54 -5
  86. airflow/providers/google/cloud/operators/managed_kafka.py +271 -4
  87. airflow/providers/google/cloud/operators/natural_language.py +5 -3
  88. airflow/providers/google/cloud/operators/pubsub.py +10 -8
  89. airflow/providers/google/cloud/operators/speech_to_text.py +5 -3
  90. airflow/providers/google/cloud/operators/stackdriver.py +5 -3
  91. airflow/providers/google/cloud/operators/tasks.py +6 -4
  92. airflow/providers/google/cloud/operators/text_to_speech.py +4 -2
  93. airflow/providers/google/cloud/operators/translate.py +5 -3
  94. airflow/providers/google/cloud/operators/translate_speech.py +4 -2
  95. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +8 -6
  96. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +6 -4
  97. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +8 -6
  98. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +6 -4
  99. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +6 -4
  100. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +6 -4
  101. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +6 -4
  102. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +6 -4
  103. airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
  104. airflow/providers/google/cloud/operators/vision.py +6 -4
  105. airflow/providers/google/cloud/operators/workflows.py +7 -5
  106. airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
  107. airflow/providers/google/cloud/sensors/bigquery_dts.py +5 -3
  108. airflow/providers/google/cloud/sensors/bigtable.py +3 -2
  109. airflow/providers/google/cloud/sensors/cloud_composer.py +1 -1
  110. airflow/providers/google/cloud/sensors/dataplex.py +6 -4
  111. airflow/providers/google/cloud/sensors/dataproc.py +3 -2
  112. airflow/providers/google/cloud/sensors/dataproc_metastore.py +2 -1
  113. airflow/providers/google/cloud/sensors/gcs.py +4 -2
  114. airflow/providers/google/cloud/sensors/pubsub.py +3 -2
  115. airflow/providers/google/cloud/sensors/workflows.py +5 -3
  116. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +5 -3
  117. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +12 -10
  118. airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
  119. airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
  120. airflow/providers/google/cloud/triggers/cloud_build.py +2 -1
  121. airflow/providers/google/cloud/triggers/cloud_composer.py +1 -1
  122. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +3 -2
  123. airflow/providers/google/cloud/triggers/dataflow.py +3 -2
  124. airflow/providers/google/cloud/triggers/dataplex.py +2 -1
  125. airflow/providers/google/cloud/triggers/dataproc.py +3 -2
  126. airflow/providers/google/cloud/triggers/kubernetes_engine.py +1 -1
  127. airflow/providers/google/cloud/triggers/pubsub.py +2 -1
  128. airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
  129. airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
  130. airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
  131. airflow/providers/google/cloud/{example_dags/__init__.py → utils/validators.py} +27 -0
  132. airflow/providers/google/common/auth_backend/google_openid.py +14 -5
  133. airflow/providers/google/common/consts.py +2 -1
  134. airflow/providers/google/common/hooks/base_google.py +7 -8
  135. airflow/providers/google/common/hooks/operation_helpers.py +79 -0
  136. airflow/providers/google/get_provider_info.py +11 -6
  137. airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
  138. airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
  139. airflow/providers/google/marketing_platform/links/analytics_admin.py +10 -2
  140. airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -5
  141. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0.dist-info}/METADATA +32 -35
  142. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0.dist-info}/RECORD +144 -140
  143. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0.dist-info}/WHEEL +1 -1
  144. airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
  145. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,14 @@
1
+ BSD 3-Clause "New" or "Revised" License
2
+
3
+ Copyright (c) 2015, Open Data Services Coop
4
+ All rights reserved.
5
+
6
+ Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
7
+
8
+ 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
9
+
10
+ 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
11
+
12
+ 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
13
+
14
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,15 @@
1
+ Apache Airflow
2
+ Copyright 2016-2025 The Apache Software Foundation
3
+
4
+ This product includes software developed at The Apache Software
5
+ Foundation (http://www.apache.org/).
6
+
7
+ =======================================================================
8
+
9
+ json-merge-patch:
10
+ -----------------
11
+
12
+ This product contains a modified portion of 'json-merge-patch' developed by Open Data Services Coop
13
+ https://github.com/OpenDataServices/json-merge-patch).
14
+
15
+ Copyright (c) 2015, Open Data Services Coop
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
29
29
 
30
30
  __all__ = ["__version__"]
31
31
 
32
- __version__ = "14.0.0"
32
+ __version__ = "14.1.0"
33
33
 
34
34
  if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
35
35
  "2.9.0"
File without changes
@@ -0,0 +1,91 @@
1
+ # BSD 3-Clause "New" or "Revised" License
2
+ #
3
+ # Copyright (c) 2015, Open Data Services Coop
4
+ # All rights reserved.
5
+ #
6
+ # Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
7
+ #
8
+ # 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
9
+ #
10
+ # 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
11
+ #
12
+ # 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
13
+ #
14
+ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
15
+
16
+ from collections import OrderedDict
17
+ import sys
18
+
19
+ def merge(*objs, **kw):
20
+ result = objs[0]
21
+ for obj in objs[1:]:
22
+ result = merge_obj(result, obj, kw.get('position'))
23
+ return result
24
+
25
+ def move_to_start(result, key):
26
+ result_copy = result.copy()
27
+ result.clear()
28
+ result[key] = result_copy.pop(key)
29
+ result.update(result_copy)
30
+
31
+ def merge_obj(result, obj, position=None):
32
+ if not isinstance(result, dict):
33
+ result = OrderedDict() if position else {}
34
+
35
+ if not isinstance(obj, dict):
36
+ return obj
37
+
38
+ if position:
39
+ if position not in ('first', 'last'):
40
+ raise ValueError("position can either be first or last")
41
+ if not isinstance(result, OrderedDict) or not isinstance(obj, OrderedDict):
42
+ raise ValueError("If using position all dicts need to be OrderedDicts")
43
+
44
+ for key, value in obj.items():
45
+ if isinstance(value, dict):
46
+ target = result.get(key)
47
+ if isinstance(target, dict):
48
+ merge_obj(target, value, position)
49
+ continue
50
+ result[key] = OrderedDict() if position else {}
51
+ if position and position == 'first':
52
+ if sys.version_info >= (3, 2):
53
+ result.move_to_end(key, False)
54
+ else:
55
+ move_to_start(result, key)
56
+ merge_obj(result[key], value, position)
57
+ continue
58
+ if value is None:
59
+ result.pop(key, None)
60
+ continue
61
+ if key not in result and position == 'first':
62
+ result[key] = value
63
+ if sys.version_info >= (3, 2):
64
+ result.move_to_end(key, False)
65
+ else:
66
+ move_to_start(result, key)
67
+ else:
68
+ result[key] = value
69
+
70
+ return result
71
+
72
+ def create_patch(source, target):
73
+ return create_patch_obj(source, target)
74
+
75
+ def create_patch_obj(source, target):
76
+ if not isinstance(target, dict) or not isinstance(source, dict):
77
+ return target
78
+
79
+ result = {}
80
+
81
+ for key in set(source.keys()) - set(target.keys()):
82
+ result[key] = None
83
+
84
+ for key, value in target.items():
85
+ if key not in source:
86
+ result[key] = value
87
+ continue
88
+ if value == source[key]:
89
+ continue
90
+ result[key] = create_patch_obj(source[key], value)
91
+ return result
@@ -24,18 +24,19 @@ from functools import cached_property
24
24
  from tempfile import NamedTemporaryFile
25
25
  from typing import IO, TYPE_CHECKING, Any, Literal
26
26
 
27
- from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
28
- from airflow.hooks.base import BaseHook
29
- from airflow.providers.google.common.hooks.base_google import get_field
30
27
  from google.ads.googleads.client import GoogleAdsClient
31
28
  from google.ads.googleads.errors import GoogleAdsException
32
29
  from google.auth.exceptions import GoogleAuthError
33
30
 
31
+ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
32
+ from airflow.hooks.base import BaseHook
33
+ from airflow.providers.google.common.hooks.base_google import get_field
34
+
34
35
  if TYPE_CHECKING:
35
- from google.ads.googleads.v18.services.services.customer_service import CustomerServiceClient
36
- from google.ads.googleads.v18.services.services.google_ads_service import GoogleAdsServiceClient
37
- from google.ads.googleads.v18.services.types.google_ads_service import GoogleAdsRow
38
- from google.api_core.page_iterator import GRPCIterator
36
+ from google.ads.googleads.v19.services.services.customer_service import CustomerServiceClient
37
+ from google.ads.googleads.v19.services.services.google_ads_service import GoogleAdsServiceClient
38
+ from google.ads.googleads.v19.services.services.google_ads_service.pagers import SearchPager
39
+ from google.ads.googleads.v19.services.types.google_ads_service import GoogleAdsRow
39
40
 
40
41
 
41
42
  class GoogleAdsHook(BaseHook):
@@ -177,7 +178,7 @@ class GoogleAdsHook(BaseHook):
177
178
  """
178
179
  try:
179
180
  accessible_customers = self._get_customer_service.list_accessible_customers()
180
- return accessible_customers.resource_names
181
+ return list(accessible_customers.resource_names)
181
182
  except GoogleAdsException as ex:
182
183
  for error in ex.failure.errors:
183
184
  self.log.error('\tError with message "%s".', error.message)
@@ -305,11 +306,11 @@ class GoogleAdsHook(BaseHook):
305
306
 
306
307
  return self._extract_rows(iterators)
307
308
 
308
- def _extract_rows(self, iterators: list[GRPCIterator]) -> list[GoogleAdsRow]:
309
+ def _extract_rows(self, iterators: list[SearchPager]) -> list[GoogleAdsRow]:
309
310
  """
310
- Convert Google Page Iterator (GRPCIterator) objects to Google Ads Rows.
311
+ Convert Google Page Iterator (SearchPager) objects to Google Ads Rows.
311
312
 
312
- :param iterators: List of Google Page Iterator (GRPCIterator) objects
313
+ :param iterators: List of Google Page Iterator (SearchPager) objects
313
314
  :return: API response for all clients in the form of Google Ads Row object(s)
314
315
  """
315
316
  try:
@@ -20,11 +20,12 @@ import re
20
20
  from functools import cached_property
21
21
  from typing import TYPE_CHECKING
22
22
 
23
- from airflow.providers.google.common.consts import CLIENT_INFO
24
- from airflow.utils.log.logging_mixin import LoggingMixin
25
23
  from google.api_core.exceptions import InvalidArgument, NotFound, PermissionDenied
26
24
  from google.cloud.secretmanager_v1 import SecretManagerServiceClient
27
25
 
26
+ from airflow.providers.google.common.consts import CLIENT_INFO
27
+ from airflow.utils.log.logging_mixin import LoggingMixin
28
+
28
29
  if TYPE_CHECKING:
29
30
  import google
30
31
 
@@ -24,16 +24,15 @@ from copy import deepcopy
24
24
  from typing import TYPE_CHECKING
25
25
 
26
26
  import tenacity
27
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
28
+ from google.cloud import alloydb_v1
27
29
 
28
30
  from airflow.exceptions import AirflowException
29
31
  from airflow.providers.google.common.consts import CLIENT_INFO
30
32
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
31
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
32
- from google.cloud import alloydb_v1
33
33
 
34
34
  if TYPE_CHECKING:
35
35
  import proto
36
-
37
36
  from google.api_core.operation import Operation
38
37
  from google.api_core.retry import Retry
39
38
  from google.protobuf.field_mask_pb2 import FieldMask
@@ -29,10 +29,6 @@ from collections.abc import Sequence
29
29
  from functools import cached_property
30
30
  from typing import TYPE_CHECKING
31
31
 
32
- from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
33
- from airflow.providers.google.common.consts import CLIENT_INFO
34
- from airflow.providers.google.common.deprecated import deprecated
35
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
36
32
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
37
33
  from google.cloud.automl_v1beta1 import (
38
34
  AutoMlClient,
@@ -47,6 +43,12 @@ from google.cloud.automl_v1beta1 import (
47
43
  PredictResponse,
48
44
  )
49
45
 
46
+ from airflow.exceptions import AirflowProviderDeprecationWarning
47
+ from airflow.providers.google.common.consts import CLIENT_INFO
48
+ from airflow.providers.google.common.deprecated import deprecated
49
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
50
+ from airflow.providers.google.common.hooks.operation_helpers import OperationHelper
51
+
50
52
  if TYPE_CHECKING:
51
53
  from google.api_core.operation import Operation
52
54
  from google.api_core.retry import Retry
@@ -64,7 +66,7 @@ if TYPE_CHECKING:
64
66
  "airflow.providers.google.cloud.hooks.translate.TranslateHook",
65
67
  category=AirflowProviderDeprecationWarning,
66
68
  )
67
- class CloudAutoMLHook(GoogleBaseHook):
69
+ class CloudAutoMLHook(GoogleBaseHook, OperationHelper):
68
70
  """
69
71
  Google Cloud AutoML hook.
70
72
 
@@ -100,14 +102,6 @@ class CloudAutoMLHook(GoogleBaseHook):
100
102
  self._client = AutoMlClient(credentials=self.get_credentials(), client_info=CLIENT_INFO)
101
103
  return self._client
102
104
 
103
- def wait_for_operation(self, operation: Operation, timeout: float | None = None):
104
- """Wait for long-lasting operation to complete."""
105
- try:
106
- return operation.result(timeout=timeout)
107
- except Exception:
108
- error = operation.exception(timeout=timeout)
109
- raise AirflowException(error)
110
-
111
105
  @cached_property
112
106
  def prediction_client(self) -> PredictionServiceClient:
113
107
  """
@@ -33,6 +33,26 @@ from typing import TYPE_CHECKING, Any, NoReturn, Union, cast
33
33
 
34
34
  from aiohttp import ClientSession as ClientSession
35
35
  from gcloud.aio.bigquery import Job, Table as Table_async
36
+ from google.cloud.bigquery import (
37
+ DEFAULT_RETRY,
38
+ Client,
39
+ CopyJob,
40
+ ExtractJob,
41
+ LoadJob,
42
+ QueryJob,
43
+ SchemaField,
44
+ UnknownJob,
45
+ )
46
+ from google.cloud.bigquery.dataset import AccessEntry, Dataset, DatasetListItem, DatasetReference
47
+ from google.cloud.bigquery.retry import DEFAULT_JOB_RETRY
48
+ from google.cloud.bigquery.table import (
49
+ Row,
50
+ RowIterator,
51
+ Table,
52
+ TableListItem,
53
+ TableReference,
54
+ )
55
+ from google.cloud.exceptions import NotFound
36
56
  from googleapiclient.discovery import build
37
57
  from pandas_gbq import read_gbq
38
58
  from pandas_gbq.gbq import GbqConnector # noqa: F401 used in ``airflow.contrib.hooks.bigquery``
@@ -55,30 +75,9 @@ from airflow.providers.google.common.hooks.base_google import (
55
75
  from airflow.utils.hashlib_wrapper import md5
56
76
  from airflow.utils.helpers import convert_camel_to_snake
57
77
  from airflow.utils.log.logging_mixin import LoggingMixin
58
- from google.cloud.bigquery import (
59
- DEFAULT_RETRY,
60
- Client,
61
- CopyJob,
62
- ExtractJob,
63
- LoadJob,
64
- QueryJob,
65
- SchemaField,
66
- UnknownJob,
67
- )
68
- from google.cloud.bigquery.dataset import AccessEntry, Dataset, DatasetListItem, DatasetReference
69
- from google.cloud.bigquery.retry import DEFAULT_JOB_RETRY
70
- from google.cloud.bigquery.table import (
71
- Row,
72
- RowIterator,
73
- Table,
74
- TableListItem,
75
- TableReference,
76
- )
77
- from google.cloud.exceptions import NotFound
78
78
 
79
79
  if TYPE_CHECKING:
80
80
  import pandas as pd
81
-
82
81
  from google.api_core.page_iterator import HTTPIterator
83
82
  from google.api_core.retry import Retry
84
83
 
@@ -121,7 +120,7 @@ class BigQueryHook(GoogleBaseHook, DbApiHook):
121
120
  from wtforms import validators
122
121
  from wtforms.fields.simple import BooleanField, StringField
123
122
 
124
- from airflow.www.validators import ValidJson
123
+ from airflow.providers.google.cloud.utils.validators import ValidJson
125
124
 
126
125
  connection_form_widgets = super().get_connection_form_widgets()
127
126
  connection_form_widgets["use_legacy_sql"] = BooleanField(lazy_gettext("Use Legacy SQL"), default=True)
@@ -23,12 +23,6 @@ from collections.abc import Sequence
23
23
  from copy import copy
24
24
  from typing import TYPE_CHECKING
25
25
 
26
- from airflow.providers.google.common.consts import CLIENT_INFO
27
- from airflow.providers.google.common.hooks.base_google import (
28
- PROVIDE_PROJECT_ID,
29
- GoogleBaseAsyncHook,
30
- GoogleBaseHook,
31
- )
32
26
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
33
27
  from google.cloud.bigquery_datatransfer_v1 import DataTransferServiceAsyncClient, DataTransferServiceClient
34
28
  from google.cloud.bigquery_datatransfer_v1.types import (
@@ -37,11 +31,17 @@ from google.cloud.bigquery_datatransfer_v1.types import (
37
31
  TransferRun,
38
32
  )
39
33
 
40
- if TYPE_CHECKING:
41
- from googleapiclient.discovery import Resource
34
+ from airflow.providers.google.common.consts import CLIENT_INFO
35
+ from airflow.providers.google.common.hooks.base_google import (
36
+ PROVIDE_PROJECT_ID,
37
+ GoogleBaseAsyncHook,
38
+ GoogleBaseHook,
39
+ )
42
40
 
41
+ if TYPE_CHECKING:
43
42
  from google.api_core.retry import Retry
44
43
  from google.api_core.retry_async import AsyncRetry
44
+ from googleapiclient.discovery import Resource
45
45
 
46
46
 
47
47
  def get_object_id(obj: dict) -> str:
@@ -22,13 +22,14 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
- from airflow.providers.google.common.consts import CLIENT_INFO
26
- from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
27
25
  from google.cloud.bigtable import Client, enums
28
26
  from google.cloud.bigtable.cluster import Cluster
29
27
  from google.cloud.bigtable.instance import Instance
30
28
  from google.cloud.bigtable.table import ClusterState, Table
31
29
 
30
+ from airflow.providers.google.common.consts import CLIENT_INFO
31
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
32
+
32
33
  if TYPE_CHECKING:
33
34
  import enum
34
35
 
@@ -23,9 +23,6 @@ import time
23
23
  from collections.abc import Iterable, Sequence
24
24
  from typing import TYPE_CHECKING
25
25
 
26
- from airflow.exceptions import AirflowException
27
- from airflow.providers.google.common.consts import CLIENT_INFO
28
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
29
26
  from google.cloud.batch import ListJobsRequest, ListTasksRequest
30
27
  from google.cloud.batch_v1 import (
31
28
  BatchServiceAsyncClient,
@@ -36,6 +33,10 @@ from google.cloud.batch_v1 import (
36
33
  Task,
37
34
  )
38
35
 
36
+ from airflow.exceptions import AirflowException
37
+ from airflow.providers.google.common.consts import CLIENT_INFO
38
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
39
+
39
40
  if TYPE_CHECKING:
40
41
  from google.api_core import operation
41
42
  from google.cloud.batch_v1.services.batch_service import pagers
@@ -22,15 +22,17 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
- from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
26
- from airflow.providers.google.common.consts import CLIENT_INFO
27
- from airflow.providers.google.common.deprecated import deprecated
28
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
29
25
  from google.api_core.client_options import ClientOptions
30
26
  from google.api_core.exceptions import AlreadyExists
31
27
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
32
28
  from google.cloud.devtools.cloudbuild_v1 import CloudBuildAsyncClient, CloudBuildClient, GetBuildRequest
33
29
 
30
+ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
31
+ from airflow.providers.google.common.consts import CLIENT_INFO
32
+ from airflow.providers.google.common.deprecated import deprecated
33
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
34
+ from airflow.providers.google.common.hooks.operation_helpers import OperationHelper
35
+
34
36
  if TYPE_CHECKING:
35
37
  from google.api_core.operation import Operation
36
38
  from google.api_core.retry import Retry
@@ -41,7 +43,7 @@ if TYPE_CHECKING:
41
43
  TIME_TO_SLEEP_IN_SECONDS = 5
42
44
 
43
45
 
44
- class CloudBuildHook(GoogleBaseHook):
46
+ class CloudBuildHook(GoogleBaseHook, OperationHelper):
45
47
  """
46
48
  Hook for the Google Cloud Build Service.
47
49
 
@@ -79,14 +81,6 @@ class CloudBuildHook(GoogleBaseHook):
79
81
  except Exception:
80
82
  raise AirflowException("Could not retrieve Build ID from Operation.")
81
83
 
82
- def wait_for_operation(self, operation: Operation, timeout: float | None = None):
83
- """Wait for long-lasting operation to complete."""
84
- try:
85
- return operation.result(timeout=timeout)
86
- except Exception:
87
- error = operation.exception(timeout=timeout)
88
- raise AirflowException(error)
89
-
90
84
  def get_conn(self, location: str = "global") -> CloudBuildClient:
91
85
  """
92
86
  Retrieve the connection to Google Cloud Build.
@@ -22,9 +22,6 @@ import time
22
22
  from collections.abc import MutableSequence, Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
- from airflow.exceptions import AirflowException
26
- from airflow.providers.google.common.consts import CLIENT_INFO
27
- from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
28
25
  from google.api_core.client_options import ClientOptions
29
26
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
30
27
  from google.cloud.orchestration.airflow.service_v1 import (
@@ -34,6 +31,10 @@ from google.cloud.orchestration.airflow.service_v1 import (
34
31
  PollAirflowCommandResponse,
35
32
  )
36
33
 
34
+ from airflow.exceptions import AirflowException
35
+ from airflow.providers.google.common.consts import CLIENT_INFO
36
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
37
+
37
38
  if TYPE_CHECKING:
38
39
  from google.api_core.operation import Operation
39
40
  from google.api_core.operation_async import AsyncOperation
@@ -51,8 +52,10 @@ if TYPE_CHECKING:
51
52
  )
52
53
  from google.protobuf.field_mask_pb2 import FieldMask
53
54
 
55
+ from airflow.providers.google.common.hooks.operation_helpers import OperationHelper
54
56
 
55
- class CloudComposerHook(GoogleBaseHook):
57
+
58
+ class CloudComposerHook(GoogleBaseHook, OperationHelper):
56
59
  """Hook for Google Cloud Composer APIs."""
57
60
 
58
61
  client_options = ClientOptions(api_endpoint="composer.googleapis.com:443")
@@ -73,14 +76,6 @@ class CloudComposerHook(GoogleBaseHook):
73
76
  client_options=self.client_options,
74
77
  )
75
78
 
76
- def wait_for_operation(self, operation: Operation, timeout: float | None = None):
77
- """Wait for long-lasting operation to complete."""
78
- try:
79
- return operation.result(timeout=timeout)
80
- except Exception:
81
- error = operation.exception(timeout=timeout)
82
- raise AirflowException(error)
83
-
84
79
  def get_operation(self, operation_name):
85
80
  return self.get_environment_client().transport.operations_client.get_operation(name=operation_name)
86
81
 
@@ -31,9 +31,6 @@ from __future__ import annotations
31
31
  from collections.abc import Sequence
32
32
  from typing import TYPE_CHECKING
33
33
 
34
- from airflow import version
35
- from airflow.exceptions import AirflowException
36
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
37
34
  from google.api_core import path_template
38
35
  from google.api_core.exceptions import NotFound
39
36
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
@@ -47,6 +44,10 @@ from google.cloud.redis_v1 import (
47
44
  OutputConfig,
48
45
  )
49
46
 
47
+ from airflow import version
48
+ from airflow.exceptions import AirflowException
49
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
50
+
50
51
  if TYPE_CHECKING:
51
52
  from google.api_core.retry import Retry
52
53
  from google.protobuf.field_mask_pb2 import FieldMask
@@ -21,9 +21,6 @@ import itertools
21
21
  from collections.abc import Iterable, Sequence
22
22
  from typing import TYPE_CHECKING, Any
23
23
 
24
- from airflow.exceptions import AirflowException
25
- from airflow.providers.google.common.consts import CLIENT_INFO
26
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
27
24
  from google.cloud.run_v2 import (
28
25
  CreateJobRequest,
29
26
  CreateServiceRequest,
@@ -43,6 +40,10 @@ from google.cloud.run_v2 import (
43
40
  )
44
41
  from google.longrunning import operations_pb2 # type: ignore[attr-defined]
45
42
 
43
+ from airflow.exceptions import AirflowException
44
+ from airflow.providers.google.common.consts import CLIENT_INFO
45
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
46
+
46
47
  if TYPE_CHECKING:
47
48
  from google.api_core import operation
48
49
  from google.api_core.operation_async import AsyncOperation
@@ -64,10 +64,10 @@ from airflow.providers.google.common.hooks.base_google import (
64
64
  from airflow.utils.log.logging_mixin import LoggingMixin
65
65
 
66
66
  if TYPE_CHECKING:
67
+ from google.cloud.secretmanager_v1 import AccessSecretVersionResponse
67
68
  from requests import Session
68
69
 
69
70
  from airflow.providers.common.sql.hooks.sql import DbApiHook
70
- from google.cloud.secretmanager_v1 import AccessSecretVersionResponse
71
71
 
72
72
  UNIX_PATH_MAX = 108
73
73
 
@@ -36,6 +36,13 @@ from copy import deepcopy
36
36
  from datetime import timedelta
37
37
  from typing import TYPE_CHECKING, Any
38
38
 
39
+ from google.api_core import protobuf_helpers
40
+ from google.cloud.storage_transfer_v1 import (
41
+ ListTransferJobsRequest,
42
+ StorageTransferServiceAsyncClient,
43
+ TransferJob,
44
+ TransferOperation,
45
+ )
39
46
  from googleapiclient.discovery import Resource, build
40
47
  from googleapiclient.errors import HttpError
41
48
 
@@ -46,21 +53,13 @@ from airflow.providers.google.common.hooks.base_google import (
46
53
  GoogleBaseAsyncHook,
47
54
  GoogleBaseHook,
48
55
  )
49
- from google.api_core import protobuf_helpers
50
- from google.cloud.storage_transfer_v1 import (
51
- ListTransferJobsRequest,
52
- StorageTransferServiceAsyncClient,
53
- TransferJob,
54
- TransferOperation,
55
- )
56
56
 
57
57
  if TYPE_CHECKING:
58
- from proto import Message
59
-
60
58
  from google.cloud.storage_transfer_v1.services.storage_transfer_service.pagers import (
61
59
  ListTransferJobsAsyncPager,
62
60
  )
63
61
  from google.longrunning import operations_pb2 # type: ignore[attr-defined]
62
+ from proto import Message
64
63
 
65
64
  log = logging.getLogger(__name__)
66
65
 
@@ -23,13 +23,13 @@ import time
23
23
  from collections.abc import Sequence
24
24
  from typing import TYPE_CHECKING, Any
25
25
 
26
+ from google.cloud.compute_v1.services.instance_group_managers import InstanceGroupManagersClient
27
+ from google.cloud.compute_v1.services.instance_templates import InstanceTemplatesClient
28
+ from google.cloud.compute_v1.services.instances import InstancesClient
26
29
  from googleapiclient.discovery import build
27
30
 
28
31
  from airflow.exceptions import AirflowException
29
32
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
30
- from google.cloud.compute_v1.services.instance_group_managers import InstanceGroupManagersClient
31
- from google.cloud.compute_v1.services.instance_templates import InstanceTemplatesClient
32
- from google.cloud.compute_v1.services.instances import InstancesClient
33
33
 
34
34
  if TYPE_CHECKING:
35
35
  from google.api_core.retry import Retry
@@ -19,9 +19,6 @@ from __future__ import annotations
19
19
  from collections.abc import Sequence
20
20
  from typing import TYPE_CHECKING
21
21
 
22
- from airflow.exceptions import AirflowException
23
- from airflow.providers.google.common.consts import CLIENT_INFO
24
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
25
22
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
26
23
  from google.cloud import datacatalog
27
24
  from google.cloud.datacatalog import (
@@ -35,6 +32,10 @@ from google.cloud.datacatalog import (
35
32
  TagTemplateField,
36
33
  )
37
34
 
35
+ from airflow.exceptions import AirflowException
36
+ from airflow.providers.google.common.consts import CLIENT_INFO
37
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
38
+
38
39
  if TYPE_CHECKING:
39
40
  from google.api_core.retry import Retry
40
41
  from google.protobuf.field_mask_pb2 import FieldMask