apache-airflow-providers-google 10.14.0rc2__py3-none-any.whl → 10.15.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/cloud/hooks/automl.py +13 -13
- airflow/providers/google/cloud/hooks/bigquery.py +193 -246
- airflow/providers/google/cloud/hooks/bigquery_dts.py +6 -6
- airflow/providers/google/cloud/hooks/bigtable.py +8 -8
- airflow/providers/google/cloud/hooks/cloud_batch.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_build.py +19 -20
- airflow/providers/google/cloud/hooks/cloud_composer.py +4 -4
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +10 -10
- airflow/providers/google/cloud/hooks/cloud_run.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_sql.py +17 -17
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +3 -3
- airflow/providers/google/cloud/hooks/compute.py +16 -16
- airflow/providers/google/cloud/hooks/compute_ssh.py +1 -1
- airflow/providers/google/cloud/hooks/datacatalog.py +22 -22
- airflow/providers/google/cloud/hooks/dataflow.py +48 -49
- airflow/providers/google/cloud/hooks/dataform.py +16 -16
- airflow/providers/google/cloud/hooks/datafusion.py +15 -15
- airflow/providers/google/cloud/hooks/datapipeline.py +3 -3
- airflow/providers/google/cloud/hooks/dataplex.py +19 -19
- airflow/providers/google/cloud/hooks/dataprep.py +8 -8
- airflow/providers/google/cloud/hooks/dataproc.py +88 -0
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +13 -13
- airflow/providers/google/cloud/hooks/datastore.py +3 -3
- airflow/providers/google/cloud/hooks/dlp.py +25 -25
- airflow/providers/google/cloud/hooks/gcs.py +25 -23
- airflow/providers/google/cloud/hooks/gdm.py +3 -3
- airflow/providers/google/cloud/hooks/kms.py +3 -3
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +63 -48
- airflow/providers/google/cloud/hooks/life_sciences.py +13 -12
- airflow/providers/google/cloud/hooks/looker.py +7 -7
- airflow/providers/google/cloud/hooks/mlengine.py +12 -12
- airflow/providers/google/cloud/hooks/natural_language.py +2 -2
- airflow/providers/google/cloud/hooks/os_login.py +1 -1
- airflow/providers/google/cloud/hooks/pubsub.py +9 -9
- airflow/providers/google/cloud/hooks/secret_manager.py +1 -1
- airflow/providers/google/cloud/hooks/spanner.py +11 -11
- airflow/providers/google/cloud/hooks/speech_to_text.py +1 -1
- airflow/providers/google/cloud/hooks/stackdriver.py +7 -7
- airflow/providers/google/cloud/hooks/tasks.py +11 -11
- airflow/providers/google/cloud/hooks/text_to_speech.py +1 -1
- airflow/providers/google/cloud/hooks/translate.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +13 -13
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -6
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +45 -50
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +13 -13
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +9 -9
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +128 -11
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +10 -10
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +8 -8
- airflow/providers/google/cloud/hooks/video_intelligence.py +2 -2
- airflow/providers/google/cloud/hooks/vision.py +1 -1
- airflow/providers/google/cloud/hooks/workflows.py +10 -10
- airflow/providers/google/cloud/links/datafusion.py +12 -5
- airflow/providers/google/cloud/operators/bigquery.py +9 -11
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +3 -1
- airflow/providers/google/cloud/operators/dataflow.py +16 -16
- airflow/providers/google/cloud/operators/datafusion.py +9 -1
- airflow/providers/google/cloud/operators/dataproc.py +298 -65
- airflow/providers/google/cloud/operators/kubernetes_engine.py +6 -6
- airflow/providers/google/cloud/operators/life_sciences.py +10 -9
- airflow/providers/google/cloud/operators/mlengine.py +96 -96
- airflow/providers/google/cloud/operators/pubsub.py +2 -0
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +33 -3
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +59 -2
- airflow/providers/google/cloud/secrets/secret_manager.py +8 -7
- airflow/providers/google/cloud/sensors/bigquery.py +20 -16
- airflow/providers/google/cloud/sensors/cloud_composer.py +11 -8
- airflow/providers/google/cloud/sensors/gcs.py +8 -7
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +4 -4
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/mysql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/presto_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/trino_to_gcs.py +1 -1
- airflow/providers/google/cloud/triggers/bigquery.py +12 -12
- airflow/providers/google/cloud/triggers/bigquery_dts.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +3 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_run.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +6 -6
- airflow/providers/google/cloud/triggers/dataflow.py +3 -1
- airflow/providers/google/cloud/triggers/datafusion.py +2 -2
- airflow/providers/google/cloud/triggers/dataplex.py +2 -2
- airflow/providers/google/cloud/triggers/dataproc.py +2 -2
- airflow/providers/google/cloud/triggers/gcs.py +12 -8
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +2 -2
- airflow/providers/google/cloud/triggers/mlengine.py +2 -2
- airflow/providers/google/cloud/triggers/pubsub.py +1 -1
- airflow/providers/google/cloud/triggers/vertex_ai.py +99 -0
- airflow/providers/google/cloud/utils/bigquery.py +2 -2
- airflow/providers/google/cloud/utils/credentials_provider.py +2 -2
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/field_validator.py +2 -2
- airflow/providers/google/cloud/utils/helpers.py +2 -2
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +1 -1
- airflow/providers/google/cloud/utils/mlengine_prediction_summary.py +1 -1
- airflow/providers/google/common/auth_backend/google_openid.py +2 -2
- airflow/providers/google/common/hooks/base_google.py +29 -22
- airflow/providers/google/common/hooks/discovery_api.py +2 -2
- airflow/providers/google/common/utils/id_token_credentials.py +5 -5
- airflow/providers/google/firebase/hooks/firestore.py +3 -3
- airflow/providers/google/get_provider_info.py +7 -2
- airflow/providers/google/leveldb/hooks/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/analytics.py +11 -14
- airflow/providers/google/marketing_platform/hooks/campaign_manager.py +11 -11
- airflow/providers/google/marketing_platform/hooks/display_video.py +13 -13
- airflow/providers/google/marketing_platform/hooks/search_ads.py +4 -4
- airflow/providers/google/marketing_platform/operators/analytics.py +37 -32
- airflow/providers/google/suite/hooks/calendar.py +2 -2
- airflow/providers/google/suite/hooks/drive.py +7 -7
- airflow/providers/google/suite/hooks/sheets.py +8 -8
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/METADATA +11 -11
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/RECORD +121 -120
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/entry_points.txt +0 -0
@@ -79,7 +79,7 @@ class PubsubPullTrigger(BaseTrigger):
|
|
79
79
|
self.hook = PubSubAsyncHook()
|
80
80
|
|
81
81
|
def serialize(self) -> tuple[str, dict[str, Any]]:
|
82
|
-
"""
|
82
|
+
"""Serialize PubsubPullTrigger arguments and classpath."""
|
83
83
|
return (
|
84
84
|
"airflow.providers.google.cloud.triggers.pubsub.PubsubPullTrigger",
|
85
85
|
{
|
@@ -0,0 +1,99 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
from __future__ import annotations
|
18
|
+
|
19
|
+
from typing import Any, AsyncIterator, Sequence
|
20
|
+
|
21
|
+
from google.cloud.aiplatform_v1 import HyperparameterTuningJob, JobState
|
22
|
+
|
23
|
+
from airflow.exceptions import AirflowException
|
24
|
+
from airflow.providers.google.cloud.hooks.vertex_ai.hyperparameter_tuning_job import (
|
25
|
+
HyperparameterTuningJobAsyncHook,
|
26
|
+
)
|
27
|
+
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
28
|
+
|
29
|
+
|
30
|
+
class CreateHyperparameterTuningJobTrigger(BaseTrigger):
|
31
|
+
"""CreateHyperparameterTuningJobTrigger run on the trigger worker to perform create operation."""
|
32
|
+
|
33
|
+
statuses_success = {
|
34
|
+
JobState.JOB_STATE_PAUSED,
|
35
|
+
JobState.JOB_STATE_SUCCEEDED,
|
36
|
+
}
|
37
|
+
|
38
|
+
def __init__(
|
39
|
+
self,
|
40
|
+
conn_id: str,
|
41
|
+
project_id: str,
|
42
|
+
location: str,
|
43
|
+
job_id: str,
|
44
|
+
poll_interval: int,
|
45
|
+
impersonation_chain: str | Sequence[str] | None = None,
|
46
|
+
):
|
47
|
+
super().__init__()
|
48
|
+
self.conn_id = conn_id
|
49
|
+
self.project_id = project_id
|
50
|
+
self.location = location
|
51
|
+
self.job_id = job_id
|
52
|
+
self.poll_interval = poll_interval
|
53
|
+
self.impersonation_chain = impersonation_chain
|
54
|
+
|
55
|
+
def serialize(self) -> tuple[str, dict[str, Any]]:
|
56
|
+
return (
|
57
|
+
"airflow.providers.google.cloud.triggers.vertex_ai.CreateHyperparameterTuningJobTrigger",
|
58
|
+
{
|
59
|
+
"conn_id": self.conn_id,
|
60
|
+
"project_id": self.project_id,
|
61
|
+
"location": self.location,
|
62
|
+
"job_id": self.job_id,
|
63
|
+
"poll_interval": self.poll_interval,
|
64
|
+
"impersonation_chain": self.impersonation_chain,
|
65
|
+
},
|
66
|
+
)
|
67
|
+
|
68
|
+
async def run(self) -> AsyncIterator[TriggerEvent]:
|
69
|
+
hook = self._get_async_hook()
|
70
|
+
try:
|
71
|
+
job = await hook.wait_hyperparameter_tuning_job(
|
72
|
+
project_id=self.project_id,
|
73
|
+
location=self.location,
|
74
|
+
job_id=self.job_id,
|
75
|
+
poll_interval=self.poll_interval,
|
76
|
+
)
|
77
|
+
except AirflowException as ex:
|
78
|
+
yield TriggerEvent(
|
79
|
+
{
|
80
|
+
"status": "error",
|
81
|
+
"message": str(ex),
|
82
|
+
}
|
83
|
+
)
|
84
|
+
return
|
85
|
+
|
86
|
+
status = "success" if job.state in self.statuses_success else "error"
|
87
|
+
message = f"Hyperparameter tuning job {job.name} completed with status {job.state.name}"
|
88
|
+
yield TriggerEvent(
|
89
|
+
{
|
90
|
+
"status": status,
|
91
|
+
"message": message,
|
92
|
+
"job": HyperparameterTuningJob.to_dict(job),
|
93
|
+
}
|
94
|
+
)
|
95
|
+
|
96
|
+
def _get_async_hook(self) -> HyperparameterTuningJobAsyncHook:
|
97
|
+
return HyperparameterTuningJobAsyncHook(
|
98
|
+
gcp_conn_id=self.conn_id, impersonation_chain=self.impersonation_chain
|
99
|
+
)
|
@@ -21,7 +21,7 @@ from typing import Any
|
|
21
21
|
|
22
22
|
def bq_cast(string_field: str, bq_type: str) -> None | int | float | bool | str:
|
23
23
|
"""
|
24
|
-
|
24
|
+
Cast a BigQuery row to the appropriate data types.
|
25
25
|
|
26
26
|
This is useful because BigQuery returns all fields as strings.
|
27
27
|
"""
|
@@ -41,7 +41,7 @@ def bq_cast(string_field: str, bq_type: str) -> None | int | float | bool | str:
|
|
41
41
|
|
42
42
|
def convert_job_id(job_id: str | list[str], project_id: str, location: str | None) -> Any:
|
43
43
|
"""
|
44
|
-
|
44
|
+
Convert job_id to path: project_id:location:job_id.
|
45
45
|
|
46
46
|
:param project_id: Required. The ID of the Google Cloud project where workspace located.
|
47
47
|
:param location: Optional. The ID of the Google Cloud region where workspace located.
|
@@ -358,7 +358,7 @@ class _CredentialProvider(LoggingMixin):
|
|
358
358
|
|
359
359
|
|
360
360
|
def get_credentials_and_project_id(*args, **kwargs) -> tuple[google.auth.credentials.Credentials, str]:
|
361
|
-
"""
|
361
|
+
"""Return the Credentials object for Google API and the associated project_id."""
|
362
362
|
return _CredentialProvider(*args, **kwargs).get_credentials_and_project()
|
363
363
|
|
364
364
|
|
@@ -398,7 +398,7 @@ def _get_target_principal_and_delegates(
|
|
398
398
|
|
399
399
|
def _get_project_id_from_service_account_email(service_account_email: str) -> str:
|
400
400
|
"""
|
401
|
-
|
401
|
+
Extract project_id from service account's email address.
|
402
402
|
|
403
403
|
:param service_account_email: email of the service account.
|
404
404
|
|
@@ -45,7 +45,7 @@ def make_initialization_workspace_flow(
|
|
45
45
|
without_installation: bool = False,
|
46
46
|
) -> tuple:
|
47
47
|
"""
|
48
|
-
|
48
|
+
Create flow which simulates the initialization of the default project.
|
49
49
|
|
50
50
|
:param project_id: Required. The ID of the Google Cloud project where workspace located.
|
51
51
|
:param region: Required. The ID of the Google Cloud region where workspace located.
|
@@ -309,7 +309,7 @@ class GcpBodyFieldValidator(LoggingMixin):
|
|
309
309
|
|
310
310
|
def _validate_field(self, validation_spec, dictionary_to_validate, parent=None, force_optional=False):
|
311
311
|
"""
|
312
|
-
|
312
|
+
Validate if field is OK.
|
313
313
|
|
314
314
|
:param validation_spec: specification of the field
|
315
315
|
:param dictionary_to_validate: dictionary where the field should be present
|
@@ -413,7 +413,7 @@ class GcpBodyFieldValidator(LoggingMixin):
|
|
413
413
|
|
414
414
|
def validate(self, body_to_validate: dict) -> None:
|
415
415
|
"""
|
416
|
-
|
416
|
+
Validate if the body (dictionary) follows specification that the validator was instantiated with.
|
417
417
|
|
418
418
|
Raises ValidationSpecificationException or ValidationFieldException in case of problems
|
419
419
|
with specification or the body not conforming to the specification respectively.
|
@@ -19,12 +19,12 @@ from __future__ import annotations
|
|
19
19
|
|
20
20
|
|
21
21
|
def normalize_directory_path(source_object: str | None) -> str | None:
|
22
|
-
"""
|
22
|
+
"""Make sure dir path ends with a slash."""
|
23
23
|
return source_object + "/" if source_object and not source_object.endswith("/") else source_object
|
24
24
|
|
25
25
|
|
26
26
|
def resource_path_to_dict(resource_name: str) -> dict[str, str]:
|
27
|
-
"""
|
27
|
+
"""Convert a path-like GCP resource name into a dictionary.
|
28
28
|
|
29
29
|
For example, the path `projects/my-project/locations/my-location/instances/my-instance` will be converted
|
30
30
|
to a dict:
|
@@ -57,7 +57,7 @@ def create_evaluate_ops(
|
|
57
57
|
py_interpreter="python3",
|
58
58
|
) -> tuple[MLEngineStartBatchPredictionJobOperator, BeamRunPythonPipelineOperator, PythonOperator]:
|
59
59
|
r"""
|
60
|
-
|
60
|
+
Create Operators needed for model evaluation and returns.
|
61
61
|
|
62
62
|
This function is deprecated. All the functionality of legacy MLEngine and new features are available
|
63
63
|
on the Vertex AI platform.
|
@@ -151,7 +151,7 @@ def MakeSummary(pcoll, metric_fn, metric_keys):
|
|
151
151
|
|
152
152
|
|
153
153
|
def run(argv=None):
|
154
|
-
"""
|
154
|
+
"""Obtain prediction summary."""
|
155
155
|
parser = argparse.ArgumentParser()
|
156
156
|
parser.add_argument(
|
157
157
|
"--prediction_path",
|
@@ -52,7 +52,7 @@ def create_client_session():
|
|
52
52
|
|
53
53
|
|
54
54
|
def init_app(_):
|
55
|
-
"""
|
55
|
+
"""Initialize authentication."""
|
56
56
|
|
57
57
|
|
58
58
|
def _get_id_token_from_request(request) -> str | None:
|
@@ -110,7 +110,7 @@ T = TypeVar("T", bound=Callable)
|
|
110
110
|
|
111
111
|
|
112
112
|
def requires_authentication(function: T):
|
113
|
-
"""Decorator for
|
113
|
+
"""Decorator for function that require authentication."""
|
114
114
|
|
115
115
|
@wraps(function)
|
116
116
|
def decorated(*args, **kwargs):
|
@@ -24,7 +24,6 @@ import json
|
|
24
24
|
import logging
|
25
25
|
import os
|
26
26
|
import tempfile
|
27
|
-
import warnings
|
28
27
|
from contextlib import ExitStack, contextmanager
|
29
28
|
from subprocess import check_output
|
30
29
|
from typing import TYPE_CHECKING, Any, Callable, Generator, Sequence, TypeVar, cast
|
@@ -36,6 +35,7 @@ import google_auth_httplib2
|
|
36
35
|
import requests
|
37
36
|
import tenacity
|
38
37
|
from asgiref.sync import sync_to_async
|
38
|
+
from deprecated import deprecated
|
39
39
|
from gcloud.aio.auth.token import Token
|
40
40
|
from google.api_core.exceptions import Forbidden, ResourceExhausted, TooManyRequests
|
41
41
|
from google.auth import _cloud_sdk, compute_engine # type: ignore[attr-defined]
|
@@ -79,6 +79,8 @@ INVALID_REASONS = [
|
|
79
79
|
|
80
80
|
def is_soft_quota_exception(exception: Exception):
|
81
81
|
"""
|
82
|
+
Check for quota violation errors.
|
83
|
+
|
82
84
|
API for Google services does not have a standardized way to report quota violation errors.
|
83
85
|
|
84
86
|
The function has been adapted by trial and error to the following services:
|
@@ -100,6 +102,8 @@ def is_soft_quota_exception(exception: Exception):
|
|
100
102
|
|
101
103
|
def is_operation_in_progress_exception(exception: Exception) -> bool:
|
102
104
|
"""
|
105
|
+
Handle operation in-progress exceptions.
|
106
|
+
|
103
107
|
Some calls return 429 (too many requests!) or 409 errors (Conflict) in case of operation in progress.
|
104
108
|
|
105
109
|
* Google Cloud SQL
|
@@ -195,7 +199,7 @@ class GoogleBaseHook(BaseHook):
|
|
195
199
|
|
196
200
|
@classmethod
|
197
201
|
def get_connection_form_widgets(cls) -> dict[str, Any]:
|
198
|
-
"""
|
202
|
+
"""Return connection widgets to add to connection form."""
|
199
203
|
from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget
|
200
204
|
from flask_babel import lazy_gettext
|
201
205
|
from wtforms import IntegerField, PasswordField, StringField
|
@@ -228,7 +232,7 @@ class GoogleBaseHook(BaseHook):
|
|
228
232
|
|
229
233
|
@classmethod
|
230
234
|
def get_ui_field_behaviour(cls) -> dict[str, Any]:
|
231
|
-
"""
|
235
|
+
"""Return custom field behaviour."""
|
232
236
|
return {
|
233
237
|
"hidden_fields": ["host", "schema", "login", "password", "port", "extra"],
|
234
238
|
"relabeling": {},
|
@@ -249,7 +253,7 @@ class GoogleBaseHook(BaseHook):
|
|
249
253
|
self._cached_project_id: str | None = None
|
250
254
|
|
251
255
|
def get_credentials_and_project_id(self) -> tuple[google.auth.credentials.Credentials, str | None]:
|
252
|
-
"""
|
256
|
+
"""Return the Credentials object for Google API and the associated project_id."""
|
253
257
|
if self._cached_credentials is not None:
|
254
258
|
return self._cached_credentials, self._cached_project_id
|
255
259
|
|
@@ -299,12 +303,12 @@ class GoogleBaseHook(BaseHook):
|
|
299
303
|
return credentials, project_id
|
300
304
|
|
301
305
|
def get_credentials(self) -> google.auth.credentials.Credentials:
|
302
|
-
"""
|
306
|
+
"""Return the Credentials object for Google API."""
|
303
307
|
credentials, _ = self.get_credentials_and_project_id()
|
304
308
|
return credentials
|
305
309
|
|
306
310
|
def _get_access_token(self) -> str:
|
307
|
-
"""
|
311
|
+
"""Return a valid access token from Google API Credentials."""
|
308
312
|
credentials = self.get_credentials()
|
309
313
|
auth_req = google.auth.transport.requests.Request()
|
310
314
|
# credentials.token is None
|
@@ -315,7 +319,7 @@ class GoogleBaseHook(BaseHook):
|
|
315
319
|
@functools.lru_cache(maxsize=None)
|
316
320
|
def _get_credentials_email(self) -> str:
|
317
321
|
"""
|
318
|
-
|
322
|
+
Return the email address associated with the currently logged in account.
|
319
323
|
|
320
324
|
If a service account is used, it returns the service account.
|
321
325
|
If user authentication (e.g. gcloud auth) is used, it returns the e-mail account of that user.
|
@@ -341,7 +345,7 @@ class GoogleBaseHook(BaseHook):
|
|
341
345
|
return oauth2_client.tokeninfo().execute()["email"]
|
342
346
|
|
343
347
|
def _authorize(self) -> google_auth_httplib2.AuthorizedHttp:
|
344
|
-
"""
|
348
|
+
"""Return an authorized HTTP object to be used to build a Google cloud service hook connection."""
|
345
349
|
credentials = self.get_credentials()
|
346
350
|
http = build_http()
|
347
351
|
http = set_user_agent(http, "airflow/" + version.version)
|
@@ -350,7 +354,7 @@ class GoogleBaseHook(BaseHook):
|
|
350
354
|
|
351
355
|
def _get_field(self, f: str, default: Any = None) -> Any:
|
352
356
|
"""
|
353
|
-
|
357
|
+
Fetch a field from extras, and returns it.
|
354
358
|
|
355
359
|
This is some Airflow magic. The google_cloud_platform hook type adds
|
356
360
|
custom UI elements to the hook page, which allow admins to specify
|
@@ -390,6 +394,10 @@ class GoogleBaseHook(BaseHook):
|
|
390
394
|
)
|
391
395
|
|
392
396
|
@property
|
397
|
+
@deprecated(
|
398
|
+
reason="Please use `airflow.providers.google.common.consts.CLIENT_INFO`.",
|
399
|
+
category=AirflowProviderDeprecationWarning,
|
400
|
+
)
|
393
401
|
def client_info(self) -> ClientInfo:
|
394
402
|
"""
|
395
403
|
Return client information used to generate a user-agent for API calls.
|
@@ -400,11 +408,6 @@ class GoogleBaseHook(BaseHook):
|
|
400
408
|
the Google Cloud. It is not supported by The Google APIs Python Client that use Discovery
|
401
409
|
based APIs.
|
402
410
|
"""
|
403
|
-
warnings.warn(
|
404
|
-
"This method is deprecated, please use `airflow.providers.google.common.consts.CLIENT_INFO`.",
|
405
|
-
AirflowProviderDeprecationWarning,
|
406
|
-
stacklevel=2,
|
407
|
-
)
|
408
411
|
return CLIENT_INFO
|
409
412
|
|
410
413
|
@property
|
@@ -420,7 +423,7 @@ class GoogleBaseHook(BaseHook):
|
|
420
423
|
|
421
424
|
@staticmethod
|
422
425
|
def quota_retry(*args, **kwargs) -> Callable:
|
423
|
-
"""
|
426
|
+
"""Provide a mechanism to repeat requests in response to exceeding a temporary quota limit."""
|
424
427
|
|
425
428
|
def decorator(fun: Callable):
|
426
429
|
default_kwargs = {
|
@@ -436,7 +439,7 @@ class GoogleBaseHook(BaseHook):
|
|
436
439
|
|
437
440
|
@staticmethod
|
438
441
|
def operation_in_progress_retry(*args, **kwargs) -> Callable[[T], T]:
|
439
|
-
"""
|
442
|
+
"""Provide a mechanism to repeat requests in response to operation in progress (HTTP 409) limit."""
|
440
443
|
|
441
444
|
def decorator(fun: T):
|
442
445
|
default_kwargs = {
|
@@ -453,7 +456,7 @@ class GoogleBaseHook(BaseHook):
|
|
453
456
|
@staticmethod
|
454
457
|
def fallback_to_default_project_id(func: Callable[..., RT]) -> Callable[..., RT]:
|
455
458
|
"""
|
456
|
-
|
459
|
+
Provide fallback for Google Cloud project id. To be used as a decorator.
|
457
460
|
|
458
461
|
If the project is None it will be replaced with the project_id from the
|
459
462
|
service account the Hook is authenticated with. Project id can be specified
|
@@ -486,7 +489,7 @@ class GoogleBaseHook(BaseHook):
|
|
486
489
|
@staticmethod
|
487
490
|
def provide_gcp_credential_file(func: T) -> T:
|
488
491
|
"""
|
489
|
-
|
492
|
+
Provide a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
|
490
493
|
|
491
494
|
It is recommended to use ``provide_gcp_credential_file_as_context`` context
|
492
495
|
manager to limit the scope when authorization data is available. Using context
|
@@ -503,7 +506,7 @@ class GoogleBaseHook(BaseHook):
|
|
503
506
|
@contextmanager
|
504
507
|
def provide_gcp_credential_file_as_context(self) -> Generator[str | None, None, None]:
|
505
508
|
"""
|
506
|
-
|
509
|
+
Provide a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
|
507
510
|
|
508
511
|
See:
|
509
512
|
`Application Default Credentials (ADC)
|
@@ -539,7 +542,7 @@ class GoogleBaseHook(BaseHook):
|
|
539
542
|
@contextmanager
|
540
543
|
def provide_authorized_gcloud(self) -> Generator[None, None, None]:
|
541
544
|
"""
|
542
|
-
|
545
|
+
Provide a separate gcloud configuration with current credentials.
|
543
546
|
|
544
547
|
The gcloud tool allows you to login to Google Cloud only - ``gcloud auth login`` and
|
545
548
|
for the needs of Application Default Credentials ``gcloud auth application-default login``.
|
@@ -691,11 +694,15 @@ class GoogleBaseAsyncHook(BaseHook):
|
|
691
694
|
return self._sync_hook
|
692
695
|
|
693
696
|
async def get_token(self, *, session: ClientSession | None = None) -> _CredentialsToken:
|
694
|
-
"""
|
697
|
+
"""Return a Token instance for use in [gcloud-aio](https://talkiq.github.io/gcloud-aio/) clients."""
|
695
698
|
sync_hook = await self.get_sync_hook()
|
696
699
|
return await _CredentialsToken.from_hook(sync_hook, session=session)
|
697
700
|
|
698
701
|
async def service_file_as_context(self) -> Any:
|
699
|
-
"""
|
702
|
+
"""
|
703
|
+
Provide a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
|
704
|
+
|
705
|
+
This is the async equivalent of the non-async GoogleBaseHook's `provide_gcp_credential_file_as_context` method.
|
706
|
+
"""
|
700
707
|
sync_hook = await self.get_sync_hook()
|
701
708
|
return await sync_to_async(sync_hook.provide_gcp_credential_file_as_context)()
|
@@ -66,7 +66,7 @@ class GoogleDiscoveryApiHook(GoogleBaseHook):
|
|
66
66
|
|
67
67
|
def get_conn(self) -> Resource:
|
68
68
|
"""
|
69
|
-
|
69
|
+
Create an authenticated api client for the given api service name and credentials.
|
70
70
|
|
71
71
|
:return: the authenticated api service.
|
72
72
|
"""
|
@@ -84,7 +84,7 @@ class GoogleDiscoveryApiHook(GoogleBaseHook):
|
|
84
84
|
|
85
85
|
def query(self, endpoint: str, data: dict, paginate: bool = False, num_retries: int = 0) -> dict:
|
86
86
|
"""
|
87
|
-
|
87
|
+
Create a dynamic API call to any Google API registered in Google's API Client Library and queries it.
|
88
88
|
|
89
89
|
:param endpoint: The client libraries path to the api call's executing method.
|
90
90
|
For example: 'analyticsreporting.reports.batchGet'
|
@@ -81,7 +81,7 @@ def _load_credentials_from_file(
|
|
81
81
|
filename: str, target_audience: str | None
|
82
82
|
) -> google_auth_credentials.Credentials | None:
|
83
83
|
"""
|
84
|
-
|
84
|
+
Load credentials from a file.
|
85
85
|
|
86
86
|
The credentials file must be a service account key or a stored authorized user credential.
|
87
87
|
|
@@ -129,7 +129,7 @@ def _load_credentials_from_file(
|
|
129
129
|
def _get_explicit_environ_credentials(
|
130
130
|
target_audience: str | None,
|
131
131
|
) -> google_auth_credentials.Credentials | None:
|
132
|
-
"""
|
132
|
+
"""Get credentials from the GOOGLE_APPLICATION_CREDENTIALS environment variable."""
|
133
133
|
explicit_file = os.environ.get(environment_vars.CREDENTIALS)
|
134
134
|
|
135
135
|
if explicit_file is None:
|
@@ -145,7 +145,7 @@ def _get_explicit_environ_credentials(
|
|
145
145
|
def _get_gcloud_sdk_credentials(
|
146
146
|
target_audience: str | None,
|
147
147
|
) -> google_auth_credentials.Credentials | None:
|
148
|
-
"""
|
148
|
+
"""Get the credentials and project ID from the Cloud SDK."""
|
149
149
|
from google.auth import _cloud_sdk # type: ignore[attr-defined]
|
150
150
|
|
151
151
|
# Check if application default credentials exist.
|
@@ -162,7 +162,7 @@ def _get_gcloud_sdk_credentials(
|
|
162
162
|
def _get_gce_credentials(
|
163
163
|
target_audience: str | None, request: google.auth.transport.Request | None = None
|
164
164
|
) -> google_auth_credentials.Credentials | None:
|
165
|
-
"""
|
165
|
+
"""Get credentials and project ID from the GCE Metadata Service."""
|
166
166
|
# Ping requires a transport, but we want application default credentials
|
167
167
|
# to require no arguments. So, we'll use the _http_client transport which
|
168
168
|
# uses http.client. This is only acceptable because the metadata server
|
@@ -191,7 +191,7 @@ def _get_gce_credentials(
|
|
191
191
|
def get_default_id_token_credentials(
|
192
192
|
target_audience: str | None, request: google.auth.transport.Request = None
|
193
193
|
) -> google_auth_credentials.Credentials:
|
194
|
-
"""
|
194
|
+
"""Get the default ID Token credentials for the current environment.
|
195
195
|
|
196
196
|
`Application Default Credentials`_ provides an easy way to obtain credentials to call Google APIs for
|
197
197
|
server-to-server or local applications.
|
@@ -65,7 +65,7 @@ class CloudFirestoreHook(GoogleBaseHook):
|
|
65
65
|
|
66
66
|
def get_conn(self):
|
67
67
|
"""
|
68
|
-
|
68
|
+
Retrieve the connection to Cloud Firestore.
|
69
69
|
|
70
70
|
:return: Google Cloud Firestore services object.
|
71
71
|
"""
|
@@ -86,7 +86,7 @@ class CloudFirestoreHook(GoogleBaseHook):
|
|
86
86
|
self, body: dict, database_id: str = "(default)", project_id: str | None = None
|
87
87
|
) -> None:
|
88
88
|
"""
|
89
|
-
|
89
|
+
Start a export with the specified configuration.
|
90
90
|
|
91
91
|
:param database_id: The Database ID.
|
92
92
|
:param body: The request body.
|
@@ -110,7 +110,7 @@ class CloudFirestoreHook(GoogleBaseHook):
|
|
110
110
|
|
111
111
|
def _wait_for_operation_to_complete(self, operation_name: str) -> None:
|
112
112
|
"""
|
113
|
-
|
113
|
+
Wait for the named operation to complete - checks status of the asynchronous call.
|
114
114
|
|
115
115
|
:param operation_name: The name of the operation.
|
116
116
|
:return: The response returned by the operation.
|
@@ -28,8 +28,9 @@ def get_provider_info():
|
|
28
28
|
"name": "Google",
|
29
29
|
"description": "Google services including:\n\n - `Google Ads <https://ads.google.com/>`__\n - `Google Cloud (GCP) <https://cloud.google.com/>`__\n - `Google Firebase <https://firebase.google.com/>`__\n - `Google LevelDB <https://github.com/google/leveldb/>`__\n - `Google Marketing Platform <https://marketingplatform.google.com/>`__\n - `Google Workspace <https://workspace.google.com/>`__ (formerly Google Suite)\n",
|
30
30
|
"state": "ready",
|
31
|
-
"source-date-epoch":
|
31
|
+
"source-date-epoch": 1707636385,
|
32
32
|
"versions": [
|
33
|
+
"10.15.0",
|
33
34
|
"10.14.0",
|
34
35
|
"10.13.1",
|
35
36
|
"10.13.0",
|
@@ -92,7 +93,7 @@ def get_provider_info():
|
|
92
93
|
"gcsfs>=2023.10.0",
|
93
94
|
"google-ads>=22.1.0",
|
94
95
|
"google-analytics-admin",
|
95
|
-
"google-api-core>=2.11.0",
|
96
|
+
"google-api-core>=2.11.0,!=2.16.0",
|
96
97
|
"google-api-python-client>=1.6.0",
|
97
98
|
"google-auth>=1.0.0",
|
98
99
|
"google-auth-httplib2>=0.0.1",
|
@@ -1215,6 +1216,10 @@ def get_provider_info():
|
|
1215
1216
|
"integration-name": "Google Cloud",
|
1216
1217
|
"python-modules": ["airflow.providers.google.cloud.triggers.cloud_batch"],
|
1217
1218
|
},
|
1219
|
+
{
|
1220
|
+
"integration-name": "Google Vertex AI",
|
1221
|
+
"python-modules": ["airflow.providers.google.cloud.triggers.vertex_ai"],
|
1222
|
+
},
|
1218
1223
|
],
|
1219
1224
|
"transfers": [
|
1220
1225
|
{
|
@@ -53,7 +53,7 @@ class LevelDBHook(BaseHook):
|
|
53
53
|
|
54
54
|
def get_conn(self, name: str = "/tmp/testdb/", create_if_missing: bool = False, **kwargs) -> DB:
|
55
55
|
"""
|
56
|
-
|
56
|
+
Create `Plyvel DB <https://plyvel.readthedocs.io/en/latest/api.html#DB>`__.
|
57
57
|
|
58
58
|
:param name: path to create database e.g. `/tmp/testdb/`)
|
59
59
|
:param create_if_missing: whether a new database should be created if needed
|
@@ -66,7 +66,7 @@ class LevelDBHook(BaseHook):
|
|
66
66
|
return self.db
|
67
67
|
|
68
68
|
def close_conn(self) -> None:
|
69
|
-
"""
|
69
|
+
"""Close connection."""
|
70
70
|
db = self.db
|
71
71
|
if db is not None:
|
72
72
|
db.close()
|
@@ -17,9 +17,9 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
import warnings
|
21
20
|
from typing import Any
|
22
21
|
|
22
|
+
from deprecated import deprecated
|
23
23
|
from googleapiclient.discovery import Resource, build
|
24
24
|
from googleapiclient.http import MediaFileUpload
|
25
25
|
|
@@ -27,18 +27,15 @@ from airflow.exceptions import AirflowProviderDeprecationWarning
|
|
27
27
|
from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
|
28
28
|
|
29
29
|
|
30
|
+
@deprecated(
|
31
|
+
reason="The `GoogleAnalyticsHook` class is deprecated, please use `GoogleAnalyticsAdminHook` instead.",
|
32
|
+
category=AirflowProviderDeprecationWarning,
|
33
|
+
)
|
30
34
|
class GoogleAnalyticsHook(GoogleBaseHook):
|
31
35
|
"""Hook for Google Analytics 360."""
|
32
36
|
|
33
37
|
def __init__(self, api_version: str = "v3", *args, **kwargs):
|
34
38
|
super().__init__(*args, **kwargs)
|
35
|
-
warnings.warn(
|
36
|
-
f"The `{type(self).__name__}` class is deprecated, please use "
|
37
|
-
f"`GoogleAnalyticsAdminHook` instead.",
|
38
|
-
AirflowProviderDeprecationWarning,
|
39
|
-
stacklevel=2,
|
40
|
-
)
|
41
|
-
|
42
39
|
self.api_version = api_version
|
43
40
|
self._conn = None
|
44
41
|
|
@@ -58,7 +55,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
|
|
58
55
|
return result
|
59
56
|
|
60
57
|
def get_conn(self) -> Resource:
|
61
|
-
"""
|
58
|
+
"""Retrieve connection to Google Analytics 360."""
|
62
59
|
if not self._conn:
|
63
60
|
http_authorized = self._authorize()
|
64
61
|
self._conn = build(
|
@@ -70,7 +67,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
|
|
70
67
|
return self._conn
|
71
68
|
|
72
69
|
def list_accounts(self) -> list[dict[str, Any]]:
|
73
|
-
"""
|
70
|
+
"""List accounts list from Google Analytics 360."""
|
74
71
|
self.log.info("Retrieving accounts list...")
|
75
72
|
conn = self.get_conn()
|
76
73
|
accounts = conn.management().accounts()
|
@@ -81,7 +78,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
|
|
81
78
|
self, account_id: str, web_property_id: str, web_property_ad_words_link_id: str
|
82
79
|
) -> dict[str, Any]:
|
83
80
|
"""
|
84
|
-
|
81
|
+
Return a web property-Google Ads link to which the user has access.
|
85
82
|
|
86
83
|
:param account_id: ID of the account which the given web property belongs to.
|
87
84
|
:param web_property_id: Web property-Google Ads link UA-string.
|
@@ -105,7 +102,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
|
|
105
102
|
|
106
103
|
def list_ad_words_links(self, account_id: str, web_property_id: str) -> list[dict[str, Any]]:
|
107
104
|
"""
|
108
|
-
|
105
|
+
List webProperty-Google Ads links for a given web property.
|
109
106
|
|
110
107
|
:param account_id: ID of the account which the given web property belongs to.
|
111
108
|
:param web_property_id: Web property UA-string to retrieve the Google Ads links for.
|
@@ -128,7 +125,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
|
|
128
125
|
resumable_upload: bool = False,
|
129
126
|
) -> None:
|
130
127
|
"""
|
131
|
-
|
128
|
+
Upload file to GA via the Data Import API.
|
132
129
|
|
133
130
|
:param file_location: The path and name of the file to upload.
|
134
131
|
:param account_id: The GA account Id to which the data upload belongs.
|
@@ -165,7 +162,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
|
|
165
162
|
delete_request_body: dict[str, Any],
|
166
163
|
) -> None:
|
167
164
|
"""
|
168
|
-
|
165
|
+
Delete the uploaded data for a given account/property/dataset.
|
169
166
|
|
170
167
|
:param account_id: The GA account Id to which the data upload belongs.
|
171
168
|
:param web_property_id: UA-string associated with the upload.
|