apache-airflow-providers-google 14.0.0__py3-none-any.whl → 14.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
- airflow/providers/google/3rd-party-licenses/NOTICE +15 -0
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/_vendor/__init__.py +0 -0
- airflow/providers/google/_vendor/json_merge_patch.py +91 -0
- airflow/providers/google/ads/hooks/ads.py +12 -11
- airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
- airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
- airflow/providers/google/cloud/hooks/automl.py +7 -13
- airflow/providers/google/cloud/hooks/bigquery.py +21 -22
- airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
- airflow/providers/google/cloud/hooks/bigtable.py +3 -2
- airflow/providers/google/cloud/hooks/cloud_batch.py +4 -3
- airflow/providers/google/cloud/hooks/cloud_build.py +7 -13
- airflow/providers/google/cloud/hooks/cloud_composer.py +7 -12
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
- airflow/providers/google/cloud/hooks/cloud_run.py +4 -3
- airflow/providers/google/cloud/hooks/cloud_sql.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +8 -9
- airflow/providers/google/cloud/hooks/compute.py +3 -3
- airflow/providers/google/cloud/hooks/datacatalog.py +4 -3
- airflow/providers/google/cloud/hooks/dataflow.py +12 -12
- airflow/providers/google/cloud/hooks/dataform.py +3 -2
- airflow/providers/google/cloud/hooks/datafusion.py +2 -2
- airflow/providers/google/cloud/hooks/dataplex.py +11 -10
- airflow/providers/google/cloud/hooks/dataproc.py +5 -4
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/hooks/dlp.py +4 -3
- airflow/providers/google/cloud/hooks/gcs.py +19 -12
- airflow/providers/google/cloud/hooks/kms.py +3 -2
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +20 -14
- airflow/providers/google/cloud/hooks/life_sciences.py +1 -1
- airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
- airflow/providers/google/cloud/hooks/natural_language.py +3 -2
- airflow/providers/google/cloud/hooks/os_login.py +3 -2
- airflow/providers/google/cloud/hooks/pubsub.py +6 -6
- airflow/providers/google/cloud/hooks/secret_manager.py +3 -2
- airflow/providers/google/cloud/hooks/spanner.py +2 -2
- airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
- airflow/providers/google/cloud/hooks/stackdriver.py +4 -4
- airflow/providers/google/cloud/hooks/tasks.py +4 -3
- airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
- airflow/providers/google/cloud/hooks/translate.py +6 -15
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +7 -13
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +4 -11
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +4 -3
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +6 -11
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
- airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
- airflow/providers/google/cloud/hooks/vision.py +4 -3
- airflow/providers/google/cloud/hooks/workflows.py +3 -2
- airflow/providers/google/cloud/links/base.py +7 -1
- airflow/providers/google/cloud/links/datafusion.py +8 -2
- airflow/providers/google/cloud/links/dataproc.py +8 -1
- airflow/providers/google/cloud/links/kubernetes_engine.py +2 -1
- airflow/providers/google/cloud/links/managed_kafka.py +30 -0
- airflow/providers/google/cloud/log/gcs_task_handler.py +15 -7
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +8 -6
- airflow/providers/google/cloud/openlineage/utils.py +4 -2
- airflow/providers/google/cloud/operators/alloy_db.py +6 -5
- airflow/providers/google/cloud/operators/automl.py +11 -9
- airflow/providers/google/cloud/operators/bigquery.py +8 -6
- airflow/providers/google/cloud/operators/bigquery_dts.py +10 -8
- airflow/providers/google/cloud/operators/bigtable.py +3 -1
- airflow/providers/google/cloud/operators/cloud_base.py +2 -1
- airflow/providers/google/cloud/operators/cloud_batch.py +4 -2
- airflow/providers/google/cloud/operators/cloud_build.py +5 -3
- airflow/providers/google/cloud/operators/cloud_composer.py +7 -5
- airflow/providers/google/cloud/operators/cloud_memorystore.py +6 -4
- airflow/providers/google/cloud/operators/cloud_run.py +5 -3
- airflow/providers/google/cloud/operators/compute.py +5 -4
- airflow/providers/google/cloud/operators/datacatalog.py +11 -9
- airflow/providers/google/cloud/operators/dataform.py +5 -3
- airflow/providers/google/cloud/operators/datafusion.py +1 -1
- airflow/providers/google/cloud/operators/dataplex.py +20 -18
- airflow/providers/google/cloud/operators/dataproc.py +34 -5
- airflow/providers/google/cloud/operators/dataproc_metastore.py +18 -7
- airflow/providers/google/cloud/operators/dlp.py +19 -17
- airflow/providers/google/cloud/operators/gcs.py +5 -4
- airflow/providers/google/cloud/operators/kubernetes_engine.py +54 -5
- airflow/providers/google/cloud/operators/managed_kafka.py +271 -4
- airflow/providers/google/cloud/operators/natural_language.py +5 -3
- airflow/providers/google/cloud/operators/pubsub.py +10 -8
- airflow/providers/google/cloud/operators/speech_to_text.py +5 -3
- airflow/providers/google/cloud/operators/stackdriver.py +5 -3
- airflow/providers/google/cloud/operators/tasks.py +6 -4
- airflow/providers/google/cloud/operators/text_to_speech.py +4 -2
- airflow/providers/google/cloud/operators/translate.py +5 -3
- airflow/providers/google/cloud/operators/translate_speech.py +4 -2
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +8 -6
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +6 -4
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +8 -6
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +6 -4
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +6 -4
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +6 -4
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +6 -4
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +6 -4
- airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
- airflow/providers/google/cloud/operators/vision.py +6 -4
- airflow/providers/google/cloud/operators/workflows.py +7 -5
- airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
- airflow/providers/google/cloud/sensors/bigquery_dts.py +5 -3
- airflow/providers/google/cloud/sensors/bigtable.py +3 -2
- airflow/providers/google/cloud/sensors/cloud_composer.py +1 -1
- airflow/providers/google/cloud/sensors/dataplex.py +6 -4
- airflow/providers/google/cloud/sensors/dataproc.py +3 -2
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +2 -1
- airflow/providers/google/cloud/sensors/gcs.py +4 -2
- airflow/providers/google/cloud/sensors/pubsub.py +3 -2
- airflow/providers/google/cloud/sensors/workflows.py +5 -3
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +5 -3
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +12 -10
- airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_composer.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +3 -2
- airflow/providers/google/cloud/triggers/dataflow.py +3 -2
- airflow/providers/google/cloud/triggers/dataplex.py +2 -1
- airflow/providers/google/cloud/triggers/dataproc.py +3 -2
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +1 -1
- airflow/providers/google/cloud/triggers/pubsub.py +2 -1
- airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
- airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
- airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
- airflow/providers/google/cloud/{example_dags/__init__.py → utils/validators.py} +27 -0
- airflow/providers/google/common/auth_backend/google_openid.py +14 -5
- airflow/providers/google/common/consts.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +7 -8
- airflow/providers/google/common/hooks/operation_helpers.py +79 -0
- airflow/providers/google/get_provider_info.py +11 -6
- airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
- airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +10 -2
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -5
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0.dist-info}/METADATA +32 -35
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0.dist-info}/RECORD +144 -140
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0.dist-info}/WHEEL +1 -1
- airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-14.1.0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,14 @@
|
|
1
|
+
BSD 3-Clause "New" or "Revised" License
|
2
|
+
|
3
|
+
Copyright (c) 2015, Open Data Services Coop
|
4
|
+
All rights reserved.
|
5
|
+
|
6
|
+
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
7
|
+
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
9
|
+
|
10
|
+
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
11
|
+
|
12
|
+
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
13
|
+
|
14
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
@@ -0,0 +1,15 @@
|
|
1
|
+
Apache Airflow
|
2
|
+
Copyright 2016-2025 The Apache Software Foundation
|
3
|
+
|
4
|
+
This product includes software developed at The Apache Software
|
5
|
+
Foundation (http://www.apache.org/).
|
6
|
+
|
7
|
+
=======================================================================
|
8
|
+
|
9
|
+
json-merge-patch:
|
10
|
+
-----------------
|
11
|
+
|
12
|
+
This product contains a modified portion of 'json-merge-patch' developed by Open Data Services Coop
|
13
|
+
https://github.com/OpenDataServices/json-merge-patch).
|
14
|
+
|
15
|
+
Copyright (c) 2015, Open Data Services Coop
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
29
29
|
|
30
30
|
__all__ = ["__version__"]
|
31
31
|
|
32
|
-
__version__ = "14.
|
32
|
+
__version__ = "14.1.0"
|
33
33
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
35
35
|
"2.9.0"
|
File without changes
|
@@ -0,0 +1,91 @@
|
|
1
|
+
# BSD 3-Clause "New" or "Revised" License
|
2
|
+
#
|
3
|
+
# Copyright (c) 2015, Open Data Services Coop
|
4
|
+
# All rights reserved.
|
5
|
+
#
|
6
|
+
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
7
|
+
#
|
8
|
+
# 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
9
|
+
#
|
10
|
+
# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
11
|
+
#
|
12
|
+
# 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
13
|
+
#
|
14
|
+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
15
|
+
|
16
|
+
from collections import OrderedDict
|
17
|
+
import sys
|
18
|
+
|
19
|
+
def merge(*objs, **kw):
|
20
|
+
result = objs[0]
|
21
|
+
for obj in objs[1:]:
|
22
|
+
result = merge_obj(result, obj, kw.get('position'))
|
23
|
+
return result
|
24
|
+
|
25
|
+
def move_to_start(result, key):
|
26
|
+
result_copy = result.copy()
|
27
|
+
result.clear()
|
28
|
+
result[key] = result_copy.pop(key)
|
29
|
+
result.update(result_copy)
|
30
|
+
|
31
|
+
def merge_obj(result, obj, position=None):
|
32
|
+
if not isinstance(result, dict):
|
33
|
+
result = OrderedDict() if position else {}
|
34
|
+
|
35
|
+
if not isinstance(obj, dict):
|
36
|
+
return obj
|
37
|
+
|
38
|
+
if position:
|
39
|
+
if position not in ('first', 'last'):
|
40
|
+
raise ValueError("position can either be first or last")
|
41
|
+
if not isinstance(result, OrderedDict) or not isinstance(obj, OrderedDict):
|
42
|
+
raise ValueError("If using position all dicts need to be OrderedDicts")
|
43
|
+
|
44
|
+
for key, value in obj.items():
|
45
|
+
if isinstance(value, dict):
|
46
|
+
target = result.get(key)
|
47
|
+
if isinstance(target, dict):
|
48
|
+
merge_obj(target, value, position)
|
49
|
+
continue
|
50
|
+
result[key] = OrderedDict() if position else {}
|
51
|
+
if position and position == 'first':
|
52
|
+
if sys.version_info >= (3, 2):
|
53
|
+
result.move_to_end(key, False)
|
54
|
+
else:
|
55
|
+
move_to_start(result, key)
|
56
|
+
merge_obj(result[key], value, position)
|
57
|
+
continue
|
58
|
+
if value is None:
|
59
|
+
result.pop(key, None)
|
60
|
+
continue
|
61
|
+
if key not in result and position == 'first':
|
62
|
+
result[key] = value
|
63
|
+
if sys.version_info >= (3, 2):
|
64
|
+
result.move_to_end(key, False)
|
65
|
+
else:
|
66
|
+
move_to_start(result, key)
|
67
|
+
else:
|
68
|
+
result[key] = value
|
69
|
+
|
70
|
+
return result
|
71
|
+
|
72
|
+
def create_patch(source, target):
|
73
|
+
return create_patch_obj(source, target)
|
74
|
+
|
75
|
+
def create_patch_obj(source, target):
|
76
|
+
if not isinstance(target, dict) or not isinstance(source, dict):
|
77
|
+
return target
|
78
|
+
|
79
|
+
result = {}
|
80
|
+
|
81
|
+
for key in set(source.keys()) - set(target.keys()):
|
82
|
+
result[key] = None
|
83
|
+
|
84
|
+
for key, value in target.items():
|
85
|
+
if key not in source:
|
86
|
+
result[key] = value
|
87
|
+
continue
|
88
|
+
if value == source[key]:
|
89
|
+
continue
|
90
|
+
result[key] = create_patch_obj(source[key], value)
|
91
|
+
return result
|
@@ -24,18 +24,19 @@ from functools import cached_property
|
|
24
24
|
from tempfile import NamedTemporaryFile
|
25
25
|
from typing import IO, TYPE_CHECKING, Any, Literal
|
26
26
|
|
27
|
-
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
28
|
-
from airflow.hooks.base import BaseHook
|
29
|
-
from airflow.providers.google.common.hooks.base_google import get_field
|
30
27
|
from google.ads.googleads.client import GoogleAdsClient
|
31
28
|
from google.ads.googleads.errors import GoogleAdsException
|
32
29
|
from google.auth.exceptions import GoogleAuthError
|
33
30
|
|
31
|
+
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
32
|
+
from airflow.hooks.base import BaseHook
|
33
|
+
from airflow.providers.google.common.hooks.base_google import get_field
|
34
|
+
|
34
35
|
if TYPE_CHECKING:
|
35
|
-
from google.ads.googleads.
|
36
|
-
from google.ads.googleads.
|
37
|
-
from google.ads.googleads.
|
38
|
-
from google.
|
36
|
+
from google.ads.googleads.v19.services.services.customer_service import CustomerServiceClient
|
37
|
+
from google.ads.googleads.v19.services.services.google_ads_service import GoogleAdsServiceClient
|
38
|
+
from google.ads.googleads.v19.services.services.google_ads_service.pagers import SearchPager
|
39
|
+
from google.ads.googleads.v19.services.types.google_ads_service import GoogleAdsRow
|
39
40
|
|
40
41
|
|
41
42
|
class GoogleAdsHook(BaseHook):
|
@@ -177,7 +178,7 @@ class GoogleAdsHook(BaseHook):
|
|
177
178
|
"""
|
178
179
|
try:
|
179
180
|
accessible_customers = self._get_customer_service.list_accessible_customers()
|
180
|
-
return accessible_customers.resource_names
|
181
|
+
return list(accessible_customers.resource_names)
|
181
182
|
except GoogleAdsException as ex:
|
182
183
|
for error in ex.failure.errors:
|
183
184
|
self.log.error('\tError with message "%s".', error.message)
|
@@ -305,11 +306,11 @@ class GoogleAdsHook(BaseHook):
|
|
305
306
|
|
306
307
|
return self._extract_rows(iterators)
|
307
308
|
|
308
|
-
def _extract_rows(self, iterators: list[
|
309
|
+
def _extract_rows(self, iterators: list[SearchPager]) -> list[GoogleAdsRow]:
|
309
310
|
"""
|
310
|
-
Convert Google Page Iterator (
|
311
|
+
Convert Google Page Iterator (SearchPager) objects to Google Ads Rows.
|
311
312
|
|
312
|
-
:param iterators: List of Google Page Iterator (
|
313
|
+
:param iterators: List of Google Page Iterator (SearchPager) objects
|
313
314
|
:return: API response for all clients in the form of Google Ads Row object(s)
|
314
315
|
"""
|
315
316
|
try:
|
@@ -20,11 +20,12 @@ import re
|
|
20
20
|
from functools import cached_property
|
21
21
|
from typing import TYPE_CHECKING
|
22
22
|
|
23
|
-
from airflow.providers.google.common.consts import CLIENT_INFO
|
24
|
-
from airflow.utils.log.logging_mixin import LoggingMixin
|
25
23
|
from google.api_core.exceptions import InvalidArgument, NotFound, PermissionDenied
|
26
24
|
from google.cloud.secretmanager_v1 import SecretManagerServiceClient
|
27
25
|
|
26
|
+
from airflow.providers.google.common.consts import CLIENT_INFO
|
27
|
+
from airflow.utils.log.logging_mixin import LoggingMixin
|
28
|
+
|
28
29
|
if TYPE_CHECKING:
|
29
30
|
import google
|
30
31
|
|
@@ -24,16 +24,15 @@ from copy import deepcopy
|
|
24
24
|
from typing import TYPE_CHECKING
|
25
25
|
|
26
26
|
import tenacity
|
27
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
28
|
+
from google.cloud import alloydb_v1
|
27
29
|
|
28
30
|
from airflow.exceptions import AirflowException
|
29
31
|
from airflow.providers.google.common.consts import CLIENT_INFO
|
30
32
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
31
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
32
|
-
from google.cloud import alloydb_v1
|
33
33
|
|
34
34
|
if TYPE_CHECKING:
|
35
35
|
import proto
|
36
|
-
|
37
36
|
from google.api_core.operation import Operation
|
38
37
|
from google.api_core.retry import Retry
|
39
38
|
from google.protobuf.field_mask_pb2 import FieldMask
|
@@ -29,10 +29,6 @@ from collections.abc import Sequence
|
|
29
29
|
from functools import cached_property
|
30
30
|
from typing import TYPE_CHECKING
|
31
31
|
|
32
|
-
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
33
|
-
from airflow.providers.google.common.consts import CLIENT_INFO
|
34
|
-
from airflow.providers.google.common.deprecated import deprecated
|
35
|
-
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
36
32
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
37
33
|
from google.cloud.automl_v1beta1 import (
|
38
34
|
AutoMlClient,
|
@@ -47,6 +43,12 @@ from google.cloud.automl_v1beta1 import (
|
|
47
43
|
PredictResponse,
|
48
44
|
)
|
49
45
|
|
46
|
+
from airflow.exceptions import AirflowProviderDeprecationWarning
|
47
|
+
from airflow.providers.google.common.consts import CLIENT_INFO
|
48
|
+
from airflow.providers.google.common.deprecated import deprecated
|
49
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
50
|
+
from airflow.providers.google.common.hooks.operation_helpers import OperationHelper
|
51
|
+
|
50
52
|
if TYPE_CHECKING:
|
51
53
|
from google.api_core.operation import Operation
|
52
54
|
from google.api_core.retry import Retry
|
@@ -64,7 +66,7 @@ if TYPE_CHECKING:
|
|
64
66
|
"airflow.providers.google.cloud.hooks.translate.TranslateHook",
|
65
67
|
category=AirflowProviderDeprecationWarning,
|
66
68
|
)
|
67
|
-
class CloudAutoMLHook(GoogleBaseHook):
|
69
|
+
class CloudAutoMLHook(GoogleBaseHook, OperationHelper):
|
68
70
|
"""
|
69
71
|
Google Cloud AutoML hook.
|
70
72
|
|
@@ -100,14 +102,6 @@ class CloudAutoMLHook(GoogleBaseHook):
|
|
100
102
|
self._client = AutoMlClient(credentials=self.get_credentials(), client_info=CLIENT_INFO)
|
101
103
|
return self._client
|
102
104
|
|
103
|
-
def wait_for_operation(self, operation: Operation, timeout: float | None = None):
|
104
|
-
"""Wait for long-lasting operation to complete."""
|
105
|
-
try:
|
106
|
-
return operation.result(timeout=timeout)
|
107
|
-
except Exception:
|
108
|
-
error = operation.exception(timeout=timeout)
|
109
|
-
raise AirflowException(error)
|
110
|
-
|
111
105
|
@cached_property
|
112
106
|
def prediction_client(self) -> PredictionServiceClient:
|
113
107
|
"""
|
@@ -33,6 +33,26 @@ from typing import TYPE_CHECKING, Any, NoReturn, Union, cast
|
|
33
33
|
|
34
34
|
from aiohttp import ClientSession as ClientSession
|
35
35
|
from gcloud.aio.bigquery import Job, Table as Table_async
|
36
|
+
from google.cloud.bigquery import (
|
37
|
+
DEFAULT_RETRY,
|
38
|
+
Client,
|
39
|
+
CopyJob,
|
40
|
+
ExtractJob,
|
41
|
+
LoadJob,
|
42
|
+
QueryJob,
|
43
|
+
SchemaField,
|
44
|
+
UnknownJob,
|
45
|
+
)
|
46
|
+
from google.cloud.bigquery.dataset import AccessEntry, Dataset, DatasetListItem, DatasetReference
|
47
|
+
from google.cloud.bigquery.retry import DEFAULT_JOB_RETRY
|
48
|
+
from google.cloud.bigquery.table import (
|
49
|
+
Row,
|
50
|
+
RowIterator,
|
51
|
+
Table,
|
52
|
+
TableListItem,
|
53
|
+
TableReference,
|
54
|
+
)
|
55
|
+
from google.cloud.exceptions import NotFound
|
36
56
|
from googleapiclient.discovery import build
|
37
57
|
from pandas_gbq import read_gbq
|
38
58
|
from pandas_gbq.gbq import GbqConnector # noqa: F401 used in ``airflow.contrib.hooks.bigquery``
|
@@ -55,30 +75,9 @@ from airflow.providers.google.common.hooks.base_google import (
|
|
55
75
|
from airflow.utils.hashlib_wrapper import md5
|
56
76
|
from airflow.utils.helpers import convert_camel_to_snake
|
57
77
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
58
|
-
from google.cloud.bigquery import (
|
59
|
-
DEFAULT_RETRY,
|
60
|
-
Client,
|
61
|
-
CopyJob,
|
62
|
-
ExtractJob,
|
63
|
-
LoadJob,
|
64
|
-
QueryJob,
|
65
|
-
SchemaField,
|
66
|
-
UnknownJob,
|
67
|
-
)
|
68
|
-
from google.cloud.bigquery.dataset import AccessEntry, Dataset, DatasetListItem, DatasetReference
|
69
|
-
from google.cloud.bigquery.retry import DEFAULT_JOB_RETRY
|
70
|
-
from google.cloud.bigquery.table import (
|
71
|
-
Row,
|
72
|
-
RowIterator,
|
73
|
-
Table,
|
74
|
-
TableListItem,
|
75
|
-
TableReference,
|
76
|
-
)
|
77
|
-
from google.cloud.exceptions import NotFound
|
78
78
|
|
79
79
|
if TYPE_CHECKING:
|
80
80
|
import pandas as pd
|
81
|
-
|
82
81
|
from google.api_core.page_iterator import HTTPIterator
|
83
82
|
from google.api_core.retry import Retry
|
84
83
|
|
@@ -121,7 +120,7 @@ class BigQueryHook(GoogleBaseHook, DbApiHook):
|
|
121
120
|
from wtforms import validators
|
122
121
|
from wtforms.fields.simple import BooleanField, StringField
|
123
122
|
|
124
|
-
from airflow.
|
123
|
+
from airflow.providers.google.cloud.utils.validators import ValidJson
|
125
124
|
|
126
125
|
connection_form_widgets = super().get_connection_form_widgets()
|
127
126
|
connection_form_widgets["use_legacy_sql"] = BooleanField(lazy_gettext("Use Legacy SQL"), default=True)
|
@@ -23,12 +23,6 @@ from collections.abc import Sequence
|
|
23
23
|
from copy import copy
|
24
24
|
from typing import TYPE_CHECKING
|
25
25
|
|
26
|
-
from airflow.providers.google.common.consts import CLIENT_INFO
|
27
|
-
from airflow.providers.google.common.hooks.base_google import (
|
28
|
-
PROVIDE_PROJECT_ID,
|
29
|
-
GoogleBaseAsyncHook,
|
30
|
-
GoogleBaseHook,
|
31
|
-
)
|
32
26
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
33
27
|
from google.cloud.bigquery_datatransfer_v1 import DataTransferServiceAsyncClient, DataTransferServiceClient
|
34
28
|
from google.cloud.bigquery_datatransfer_v1.types import (
|
@@ -37,11 +31,17 @@ from google.cloud.bigquery_datatransfer_v1.types import (
|
|
37
31
|
TransferRun,
|
38
32
|
)
|
39
33
|
|
40
|
-
|
41
|
-
|
34
|
+
from airflow.providers.google.common.consts import CLIENT_INFO
|
35
|
+
from airflow.providers.google.common.hooks.base_google import (
|
36
|
+
PROVIDE_PROJECT_ID,
|
37
|
+
GoogleBaseAsyncHook,
|
38
|
+
GoogleBaseHook,
|
39
|
+
)
|
42
40
|
|
41
|
+
if TYPE_CHECKING:
|
43
42
|
from google.api_core.retry import Retry
|
44
43
|
from google.api_core.retry_async import AsyncRetry
|
44
|
+
from googleapiclient.discovery import Resource
|
45
45
|
|
46
46
|
|
47
47
|
def get_object_id(obj: dict) -> str:
|
@@ -22,13 +22,14 @@ from __future__ import annotations
|
|
22
22
|
from collections.abc import Sequence
|
23
23
|
from typing import TYPE_CHECKING
|
24
24
|
|
25
|
-
from airflow.providers.google.common.consts import CLIENT_INFO
|
26
|
-
from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
|
27
25
|
from google.cloud.bigtable import Client, enums
|
28
26
|
from google.cloud.bigtable.cluster import Cluster
|
29
27
|
from google.cloud.bigtable.instance import Instance
|
30
28
|
from google.cloud.bigtable.table import ClusterState, Table
|
31
29
|
|
30
|
+
from airflow.providers.google.common.consts import CLIENT_INFO
|
31
|
+
from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
|
32
|
+
|
32
33
|
if TYPE_CHECKING:
|
33
34
|
import enum
|
34
35
|
|
@@ -23,9 +23,6 @@ import time
|
|
23
23
|
from collections.abc import Iterable, Sequence
|
24
24
|
from typing import TYPE_CHECKING
|
25
25
|
|
26
|
-
from airflow.exceptions import AirflowException
|
27
|
-
from airflow.providers.google.common.consts import CLIENT_INFO
|
28
|
-
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
29
26
|
from google.cloud.batch import ListJobsRequest, ListTasksRequest
|
30
27
|
from google.cloud.batch_v1 import (
|
31
28
|
BatchServiceAsyncClient,
|
@@ -36,6 +33,10 @@ from google.cloud.batch_v1 import (
|
|
36
33
|
Task,
|
37
34
|
)
|
38
35
|
|
36
|
+
from airflow.exceptions import AirflowException
|
37
|
+
from airflow.providers.google.common.consts import CLIENT_INFO
|
38
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
39
|
+
|
39
40
|
if TYPE_CHECKING:
|
40
41
|
from google.api_core import operation
|
41
42
|
from google.cloud.batch_v1.services.batch_service import pagers
|
@@ -22,15 +22,17 @@ from __future__ import annotations
|
|
22
22
|
from collections.abc import Sequence
|
23
23
|
from typing import TYPE_CHECKING
|
24
24
|
|
25
|
-
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
26
|
-
from airflow.providers.google.common.consts import CLIENT_INFO
|
27
|
-
from airflow.providers.google.common.deprecated import deprecated
|
28
|
-
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
29
25
|
from google.api_core.client_options import ClientOptions
|
30
26
|
from google.api_core.exceptions import AlreadyExists
|
31
27
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
32
28
|
from google.cloud.devtools.cloudbuild_v1 import CloudBuildAsyncClient, CloudBuildClient, GetBuildRequest
|
33
29
|
|
30
|
+
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
31
|
+
from airflow.providers.google.common.consts import CLIENT_INFO
|
32
|
+
from airflow.providers.google.common.deprecated import deprecated
|
33
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
34
|
+
from airflow.providers.google.common.hooks.operation_helpers import OperationHelper
|
35
|
+
|
34
36
|
if TYPE_CHECKING:
|
35
37
|
from google.api_core.operation import Operation
|
36
38
|
from google.api_core.retry import Retry
|
@@ -41,7 +43,7 @@ if TYPE_CHECKING:
|
|
41
43
|
TIME_TO_SLEEP_IN_SECONDS = 5
|
42
44
|
|
43
45
|
|
44
|
-
class CloudBuildHook(GoogleBaseHook):
|
46
|
+
class CloudBuildHook(GoogleBaseHook, OperationHelper):
|
45
47
|
"""
|
46
48
|
Hook for the Google Cloud Build Service.
|
47
49
|
|
@@ -79,14 +81,6 @@ class CloudBuildHook(GoogleBaseHook):
|
|
79
81
|
except Exception:
|
80
82
|
raise AirflowException("Could not retrieve Build ID from Operation.")
|
81
83
|
|
82
|
-
def wait_for_operation(self, operation: Operation, timeout: float | None = None):
|
83
|
-
"""Wait for long-lasting operation to complete."""
|
84
|
-
try:
|
85
|
-
return operation.result(timeout=timeout)
|
86
|
-
except Exception:
|
87
|
-
error = operation.exception(timeout=timeout)
|
88
|
-
raise AirflowException(error)
|
89
|
-
|
90
84
|
def get_conn(self, location: str = "global") -> CloudBuildClient:
|
91
85
|
"""
|
92
86
|
Retrieve the connection to Google Cloud Build.
|
@@ -22,9 +22,6 @@ import time
|
|
22
22
|
from collections.abc import MutableSequence, Sequence
|
23
23
|
from typing import TYPE_CHECKING
|
24
24
|
|
25
|
-
from airflow.exceptions import AirflowException
|
26
|
-
from airflow.providers.google.common.consts import CLIENT_INFO
|
27
|
-
from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
|
28
25
|
from google.api_core.client_options import ClientOptions
|
29
26
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
30
27
|
from google.cloud.orchestration.airflow.service_v1 import (
|
@@ -34,6 +31,10 @@ from google.cloud.orchestration.airflow.service_v1 import (
|
|
34
31
|
PollAirflowCommandResponse,
|
35
32
|
)
|
36
33
|
|
34
|
+
from airflow.exceptions import AirflowException
|
35
|
+
from airflow.providers.google.common.consts import CLIENT_INFO
|
36
|
+
from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
|
37
|
+
|
37
38
|
if TYPE_CHECKING:
|
38
39
|
from google.api_core.operation import Operation
|
39
40
|
from google.api_core.operation_async import AsyncOperation
|
@@ -51,8 +52,10 @@ if TYPE_CHECKING:
|
|
51
52
|
)
|
52
53
|
from google.protobuf.field_mask_pb2 import FieldMask
|
53
54
|
|
55
|
+
from airflow.providers.google.common.hooks.operation_helpers import OperationHelper
|
54
56
|
|
55
|
-
|
57
|
+
|
58
|
+
class CloudComposerHook(GoogleBaseHook, OperationHelper):
|
56
59
|
"""Hook for Google Cloud Composer APIs."""
|
57
60
|
|
58
61
|
client_options = ClientOptions(api_endpoint="composer.googleapis.com:443")
|
@@ -73,14 +76,6 @@ class CloudComposerHook(GoogleBaseHook):
|
|
73
76
|
client_options=self.client_options,
|
74
77
|
)
|
75
78
|
|
76
|
-
def wait_for_operation(self, operation: Operation, timeout: float | None = None):
|
77
|
-
"""Wait for long-lasting operation to complete."""
|
78
|
-
try:
|
79
|
-
return operation.result(timeout=timeout)
|
80
|
-
except Exception:
|
81
|
-
error = operation.exception(timeout=timeout)
|
82
|
-
raise AirflowException(error)
|
83
|
-
|
84
79
|
def get_operation(self, operation_name):
|
85
80
|
return self.get_environment_client().transport.operations_client.get_operation(name=operation_name)
|
86
81
|
|
@@ -31,9 +31,6 @@ from __future__ import annotations
|
|
31
31
|
from collections.abc import Sequence
|
32
32
|
from typing import TYPE_CHECKING
|
33
33
|
|
34
|
-
from airflow import version
|
35
|
-
from airflow.exceptions import AirflowException
|
36
|
-
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
37
34
|
from google.api_core import path_template
|
38
35
|
from google.api_core.exceptions import NotFound
|
39
36
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
@@ -47,6 +44,10 @@ from google.cloud.redis_v1 import (
|
|
47
44
|
OutputConfig,
|
48
45
|
)
|
49
46
|
|
47
|
+
from airflow import version
|
48
|
+
from airflow.exceptions import AirflowException
|
49
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
50
|
+
|
50
51
|
if TYPE_CHECKING:
|
51
52
|
from google.api_core.retry import Retry
|
52
53
|
from google.protobuf.field_mask_pb2 import FieldMask
|
@@ -21,9 +21,6 @@ import itertools
|
|
21
21
|
from collections.abc import Iterable, Sequence
|
22
22
|
from typing import TYPE_CHECKING, Any
|
23
23
|
|
24
|
-
from airflow.exceptions import AirflowException
|
25
|
-
from airflow.providers.google.common.consts import CLIENT_INFO
|
26
|
-
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
27
24
|
from google.cloud.run_v2 import (
|
28
25
|
CreateJobRequest,
|
29
26
|
CreateServiceRequest,
|
@@ -43,6 +40,10 @@ from google.cloud.run_v2 import (
|
|
43
40
|
)
|
44
41
|
from google.longrunning import operations_pb2 # type: ignore[attr-defined]
|
45
42
|
|
43
|
+
from airflow.exceptions import AirflowException
|
44
|
+
from airflow.providers.google.common.consts import CLIENT_INFO
|
45
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
46
|
+
|
46
47
|
if TYPE_CHECKING:
|
47
48
|
from google.api_core import operation
|
48
49
|
from google.api_core.operation_async import AsyncOperation
|
@@ -64,10 +64,10 @@ from airflow.providers.google.common.hooks.base_google import (
|
|
64
64
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
65
65
|
|
66
66
|
if TYPE_CHECKING:
|
67
|
+
from google.cloud.secretmanager_v1 import AccessSecretVersionResponse
|
67
68
|
from requests import Session
|
68
69
|
|
69
70
|
from airflow.providers.common.sql.hooks.sql import DbApiHook
|
70
|
-
from google.cloud.secretmanager_v1 import AccessSecretVersionResponse
|
71
71
|
|
72
72
|
UNIX_PATH_MAX = 108
|
73
73
|
|
@@ -36,6 +36,13 @@ from copy import deepcopy
|
|
36
36
|
from datetime import timedelta
|
37
37
|
from typing import TYPE_CHECKING, Any
|
38
38
|
|
39
|
+
from google.api_core import protobuf_helpers
|
40
|
+
from google.cloud.storage_transfer_v1 import (
|
41
|
+
ListTransferJobsRequest,
|
42
|
+
StorageTransferServiceAsyncClient,
|
43
|
+
TransferJob,
|
44
|
+
TransferOperation,
|
45
|
+
)
|
39
46
|
from googleapiclient.discovery import Resource, build
|
40
47
|
from googleapiclient.errors import HttpError
|
41
48
|
|
@@ -46,21 +53,13 @@ from airflow.providers.google.common.hooks.base_google import (
|
|
46
53
|
GoogleBaseAsyncHook,
|
47
54
|
GoogleBaseHook,
|
48
55
|
)
|
49
|
-
from google.api_core import protobuf_helpers
|
50
|
-
from google.cloud.storage_transfer_v1 import (
|
51
|
-
ListTransferJobsRequest,
|
52
|
-
StorageTransferServiceAsyncClient,
|
53
|
-
TransferJob,
|
54
|
-
TransferOperation,
|
55
|
-
)
|
56
56
|
|
57
57
|
if TYPE_CHECKING:
|
58
|
-
from proto import Message
|
59
|
-
|
60
58
|
from google.cloud.storage_transfer_v1.services.storage_transfer_service.pagers import (
|
61
59
|
ListTransferJobsAsyncPager,
|
62
60
|
)
|
63
61
|
from google.longrunning import operations_pb2 # type: ignore[attr-defined]
|
62
|
+
from proto import Message
|
64
63
|
|
65
64
|
log = logging.getLogger(__name__)
|
66
65
|
|
@@ -23,13 +23,13 @@ import time
|
|
23
23
|
from collections.abc import Sequence
|
24
24
|
from typing import TYPE_CHECKING, Any
|
25
25
|
|
26
|
+
from google.cloud.compute_v1.services.instance_group_managers import InstanceGroupManagersClient
|
27
|
+
from google.cloud.compute_v1.services.instance_templates import InstanceTemplatesClient
|
28
|
+
from google.cloud.compute_v1.services.instances import InstancesClient
|
26
29
|
from googleapiclient.discovery import build
|
27
30
|
|
28
31
|
from airflow.exceptions import AirflowException
|
29
32
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
30
|
-
from google.cloud.compute_v1.services.instance_group_managers import InstanceGroupManagersClient
|
31
|
-
from google.cloud.compute_v1.services.instance_templates import InstanceTemplatesClient
|
32
|
-
from google.cloud.compute_v1.services.instances import InstancesClient
|
33
33
|
|
34
34
|
if TYPE_CHECKING:
|
35
35
|
from google.api_core.retry import Retry
|
@@ -19,9 +19,6 @@ from __future__ import annotations
|
|
19
19
|
from collections.abc import Sequence
|
20
20
|
from typing import TYPE_CHECKING
|
21
21
|
|
22
|
-
from airflow.exceptions import AirflowException
|
23
|
-
from airflow.providers.google.common.consts import CLIENT_INFO
|
24
|
-
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
25
22
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
26
23
|
from google.cloud import datacatalog
|
27
24
|
from google.cloud.datacatalog import (
|
@@ -35,6 +32,10 @@ from google.cloud.datacatalog import (
|
|
35
32
|
TagTemplateField,
|
36
33
|
)
|
37
34
|
|
35
|
+
from airflow.exceptions import AirflowException
|
36
|
+
from airflow.providers.google.common.consts import CLIENT_INFO
|
37
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
38
|
+
|
38
39
|
if TYPE_CHECKING:
|
39
40
|
from google.api_core.retry import Retry
|
40
41
|
from google.protobuf.field_mask_pb2 import FieldMask
|