apache-airflow-providers-google 10.14.0rc1__py3-none-any.whl → 10.15.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/ads/hooks/ads.py +1 -2
- airflow/providers/google/cloud/hooks/automl.py +13 -13
- airflow/providers/google/cloud/hooks/bigquery.py +208 -256
- airflow/providers/google/cloud/hooks/bigquery_dts.py +6 -6
- airflow/providers/google/cloud/hooks/bigtable.py +8 -8
- airflow/providers/google/cloud/hooks/cloud_batch.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_build.py +19 -20
- airflow/providers/google/cloud/hooks/cloud_composer.py +4 -4
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +10 -10
- airflow/providers/google/cloud/hooks/cloud_run.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_sql.py +18 -19
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +3 -3
- airflow/providers/google/cloud/hooks/compute.py +16 -16
- airflow/providers/google/cloud/hooks/compute_ssh.py +1 -1
- airflow/providers/google/cloud/hooks/datacatalog.py +22 -22
- airflow/providers/google/cloud/hooks/dataflow.py +48 -49
- airflow/providers/google/cloud/hooks/dataform.py +16 -16
- airflow/providers/google/cloud/hooks/datafusion.py +15 -15
- airflow/providers/google/cloud/hooks/datapipeline.py +3 -3
- airflow/providers/google/cloud/hooks/dataplex.py +19 -19
- airflow/providers/google/cloud/hooks/dataprep.py +10 -10
- airflow/providers/google/cloud/hooks/dataproc.py +132 -14
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +13 -13
- airflow/providers/google/cloud/hooks/datastore.py +3 -3
- airflow/providers/google/cloud/hooks/dlp.py +25 -25
- airflow/providers/google/cloud/hooks/gcs.py +39 -27
- airflow/providers/google/cloud/hooks/gdm.py +3 -3
- airflow/providers/google/cloud/hooks/kms.py +3 -3
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +63 -48
- airflow/providers/google/cloud/hooks/life_sciences.py +13 -12
- airflow/providers/google/cloud/hooks/looker.py +8 -9
- airflow/providers/google/cloud/hooks/mlengine.py +12 -12
- airflow/providers/google/cloud/hooks/natural_language.py +2 -2
- airflow/providers/google/cloud/hooks/os_login.py +1 -1
- airflow/providers/google/cloud/hooks/pubsub.py +9 -9
- airflow/providers/google/cloud/hooks/secret_manager.py +1 -1
- airflow/providers/google/cloud/hooks/spanner.py +11 -11
- airflow/providers/google/cloud/hooks/speech_to_text.py +1 -1
- airflow/providers/google/cloud/hooks/stackdriver.py +7 -7
- airflow/providers/google/cloud/hooks/tasks.py +11 -11
- airflow/providers/google/cloud/hooks/text_to_speech.py +1 -1
- airflow/providers/google/cloud/hooks/translate.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +13 -13
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -6
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +45 -50
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +13 -13
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +9 -9
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +128 -11
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +10 -10
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +8 -8
- airflow/providers/google/cloud/hooks/video_intelligence.py +2 -2
- airflow/providers/google/cloud/hooks/vision.py +1 -1
- airflow/providers/google/cloud/hooks/workflows.py +10 -10
- airflow/providers/google/cloud/links/datafusion.py +12 -5
- airflow/providers/google/cloud/operators/bigquery.py +11 -11
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +3 -1
- airflow/providers/google/cloud/operators/dataflow.py +16 -16
- airflow/providers/google/cloud/operators/datafusion.py +9 -1
- airflow/providers/google/cloud/operators/dataproc.py +444 -69
- airflow/providers/google/cloud/operators/kubernetes_engine.py +6 -6
- airflow/providers/google/cloud/operators/life_sciences.py +10 -9
- airflow/providers/google/cloud/operators/mlengine.py +96 -96
- airflow/providers/google/cloud/operators/pubsub.py +2 -0
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +33 -3
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +59 -2
- airflow/providers/google/cloud/secrets/secret_manager.py +8 -7
- airflow/providers/google/cloud/sensors/bigquery.py +20 -16
- airflow/providers/google/cloud/sensors/cloud_composer.py +11 -8
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +12 -2
- airflow/providers/google/cloud/sensors/gcs.py +8 -7
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +1 -0
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +4 -4
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +1 -0
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/mysql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/presto_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/trino_to_gcs.py +1 -1
- airflow/providers/google/cloud/triggers/bigquery.py +12 -12
- airflow/providers/google/cloud/triggers/bigquery_dts.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +3 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_run.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +6 -6
- airflow/providers/google/cloud/triggers/dataflow.py +3 -1
- airflow/providers/google/cloud/triggers/datafusion.py +2 -2
- airflow/providers/google/cloud/triggers/dataplex.py +2 -2
- airflow/providers/google/cloud/triggers/dataproc.py +34 -14
- airflow/providers/google/cloud/triggers/gcs.py +12 -8
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +2 -2
- airflow/providers/google/cloud/triggers/mlengine.py +2 -2
- airflow/providers/google/cloud/triggers/pubsub.py +1 -1
- airflow/providers/google/cloud/triggers/vertex_ai.py +99 -0
- airflow/providers/google/cloud/utils/bigquery.py +2 -2
- airflow/providers/google/cloud/utils/credentials_provider.py +2 -2
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/dataproc.py +25 -0
- airflow/providers/google/cloud/utils/field_validator.py +2 -2
- airflow/providers/google/cloud/utils/helpers.py +2 -2
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +1 -1
- airflow/providers/google/cloud/utils/mlengine_prediction_summary.py +1 -1
- airflow/providers/google/common/auth_backend/google_openid.py +2 -2
- airflow/providers/google/common/hooks/base_google.py +87 -23
- airflow/providers/google/common/hooks/discovery_api.py +2 -2
- airflow/providers/google/common/utils/id_token_credentials.py +5 -5
- airflow/providers/google/firebase/hooks/firestore.py +3 -3
- airflow/providers/google/get_provider_info.py +7 -2
- airflow/providers/google/leveldb/hooks/leveldb.py +4 -4
- airflow/providers/google/marketing_platform/hooks/analytics.py +11 -14
- airflow/providers/google/marketing_platform/hooks/campaign_manager.py +11 -11
- airflow/providers/google/marketing_platform/hooks/display_video.py +13 -13
- airflow/providers/google/marketing_platform/hooks/search_ads.py +4 -4
- airflow/providers/google/marketing_platform/operators/analytics.py +37 -32
- airflow/providers/google/suite/hooks/calendar.py +2 -2
- airflow/providers/google/suite/hooks/drive.py +7 -7
- airflow/providers/google/suite/hooks/sheets.py +8 -8
- {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/METADATA +11 -11
- {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/RECORD +126 -124
- {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/entry_points.txt +0 -0
@@ -18,12 +18,12 @@
|
|
18
18
|
"""This module contains a Google Cloud API base hook."""
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
|
+
import datetime
|
21
22
|
import functools
|
22
23
|
import json
|
23
24
|
import logging
|
24
25
|
import os
|
25
26
|
import tempfile
|
26
|
-
import warnings
|
27
27
|
from contextlib import ExitStack, contextmanager
|
28
28
|
from subprocess import check_output
|
29
29
|
from typing import TYPE_CHECKING, Any, Callable, Generator, Sequence, TypeVar, cast
|
@@ -35,6 +35,8 @@ import google_auth_httplib2
|
|
35
35
|
import requests
|
36
36
|
import tenacity
|
37
37
|
from asgiref.sync import sync_to_async
|
38
|
+
from deprecated import deprecated
|
39
|
+
from gcloud.aio.auth.token import Token
|
38
40
|
from google.api_core.exceptions import Forbidden, ResourceExhausted, TooManyRequests
|
39
41
|
from google.auth import _cloud_sdk, compute_engine # type: ignore[attr-defined]
|
40
42
|
from google.auth.environment_vars import CLOUD_SDK_CONFIG_DIR, CREDENTIALS
|
@@ -43,6 +45,7 @@ from google.auth.transport import _http_client
|
|
43
45
|
from googleapiclient import discovery
|
44
46
|
from googleapiclient.errors import HttpError
|
45
47
|
from googleapiclient.http import MediaIoBaseDownload, build_http, set_user_agent
|
48
|
+
from requests import Session
|
46
49
|
|
47
50
|
from airflow import version
|
48
51
|
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
@@ -56,7 +59,9 @@ from airflow.providers.google.common.consts import CLIENT_INFO
|
|
56
59
|
from airflow.utils.process_utils import patch_environ
|
57
60
|
|
58
61
|
if TYPE_CHECKING:
|
62
|
+
from aiohttp import ClientSession
|
59
63
|
from google.api_core.gapic_v1.client_info import ClientInfo
|
64
|
+
from google.auth.credentials import Credentials
|
60
65
|
|
61
66
|
log = logging.getLogger(__name__)
|
62
67
|
|
@@ -74,6 +79,8 @@ INVALID_REASONS = [
|
|
74
79
|
|
75
80
|
def is_soft_quota_exception(exception: Exception):
|
76
81
|
"""
|
82
|
+
Check for quota violation errors.
|
83
|
+
|
77
84
|
API for Google services does not have a standardized way to report quota violation errors.
|
78
85
|
|
79
86
|
The function has been adapted by trial and error to the following services:
|
@@ -95,6 +102,8 @@ def is_soft_quota_exception(exception: Exception):
|
|
95
102
|
|
96
103
|
def is_operation_in_progress_exception(exception: Exception) -> bool:
|
97
104
|
"""
|
105
|
+
Handle operation in-progress exceptions.
|
106
|
+
|
98
107
|
Some calls return 429 (too many requests!) or 409 errors (Conflict) in case of operation in progress.
|
99
108
|
|
100
109
|
* Google Cloud SQL
|
@@ -190,7 +199,7 @@ class GoogleBaseHook(BaseHook):
|
|
190
199
|
|
191
200
|
@classmethod
|
192
201
|
def get_connection_form_widgets(cls) -> dict[str, Any]:
|
193
|
-
"""
|
202
|
+
"""Return connection widgets to add to connection form."""
|
194
203
|
from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget
|
195
204
|
from flask_babel import lazy_gettext
|
196
205
|
from wtforms import IntegerField, PasswordField, StringField
|
@@ -223,7 +232,7 @@ class GoogleBaseHook(BaseHook):
|
|
223
232
|
|
224
233
|
@classmethod
|
225
234
|
def get_ui_field_behaviour(cls) -> dict[str, Any]:
|
226
|
-
"""
|
235
|
+
"""Return custom field behaviour."""
|
227
236
|
return {
|
228
237
|
"hidden_fields": ["host", "schema", "login", "password", "port", "extra"],
|
229
238
|
"relabeling": {},
|
@@ -234,9 +243,8 @@ class GoogleBaseHook(BaseHook):
|
|
234
243
|
gcp_conn_id: str = "google_cloud_default",
|
235
244
|
delegate_to: str | None = None,
|
236
245
|
impersonation_chain: str | Sequence[str] | None = None,
|
237
|
-
**kwargs,
|
238
246
|
) -> None:
|
239
|
-
super().__init__(
|
247
|
+
super().__init__()
|
240
248
|
self.gcp_conn_id = gcp_conn_id
|
241
249
|
self.delegate_to = delegate_to
|
242
250
|
self.impersonation_chain = impersonation_chain
|
@@ -245,7 +253,7 @@ class GoogleBaseHook(BaseHook):
|
|
245
253
|
self._cached_project_id: str | None = None
|
246
254
|
|
247
255
|
def get_credentials_and_project_id(self) -> tuple[google.auth.credentials.Credentials, str | None]:
|
248
|
-
"""
|
256
|
+
"""Return the Credentials object for Google API and the associated project_id."""
|
249
257
|
if self._cached_credentials is not None:
|
250
258
|
return self._cached_credentials, self._cached_project_id
|
251
259
|
|
@@ -295,12 +303,12 @@ class GoogleBaseHook(BaseHook):
|
|
295
303
|
return credentials, project_id
|
296
304
|
|
297
305
|
def get_credentials(self) -> google.auth.credentials.Credentials:
|
298
|
-
"""
|
306
|
+
"""Return the Credentials object for Google API."""
|
299
307
|
credentials, _ = self.get_credentials_and_project_id()
|
300
308
|
return credentials
|
301
309
|
|
302
310
|
def _get_access_token(self) -> str:
|
303
|
-
"""
|
311
|
+
"""Return a valid access token from Google API Credentials."""
|
304
312
|
credentials = self.get_credentials()
|
305
313
|
auth_req = google.auth.transport.requests.Request()
|
306
314
|
# credentials.token is None
|
@@ -311,7 +319,7 @@ class GoogleBaseHook(BaseHook):
|
|
311
319
|
@functools.lru_cache(maxsize=None)
|
312
320
|
def _get_credentials_email(self) -> str:
|
313
321
|
"""
|
314
|
-
|
322
|
+
Return the email address associated with the currently logged in account.
|
315
323
|
|
316
324
|
If a service account is used, it returns the service account.
|
317
325
|
If user authentication (e.g. gcloud auth) is used, it returns the e-mail account of that user.
|
@@ -337,7 +345,7 @@ class GoogleBaseHook(BaseHook):
|
|
337
345
|
return oauth2_client.tokeninfo().execute()["email"]
|
338
346
|
|
339
347
|
def _authorize(self) -> google_auth_httplib2.AuthorizedHttp:
|
340
|
-
"""
|
348
|
+
"""Return an authorized HTTP object to be used to build a Google cloud service hook connection."""
|
341
349
|
credentials = self.get_credentials()
|
342
350
|
http = build_http()
|
343
351
|
http = set_user_agent(http, "airflow/" + version.version)
|
@@ -346,7 +354,7 @@ class GoogleBaseHook(BaseHook):
|
|
346
354
|
|
347
355
|
def _get_field(self, f: str, default: Any = None) -> Any:
|
348
356
|
"""
|
349
|
-
|
357
|
+
Fetch a field from extras, and returns it.
|
350
358
|
|
351
359
|
This is some Airflow magic. The google_cloud_platform hook type adds
|
352
360
|
custom UI elements to the hook page, which allow admins to specify
|
@@ -386,6 +394,10 @@ class GoogleBaseHook(BaseHook):
|
|
386
394
|
)
|
387
395
|
|
388
396
|
@property
|
397
|
+
@deprecated(
|
398
|
+
reason="Please use `airflow.providers.google.common.consts.CLIENT_INFO`.",
|
399
|
+
category=AirflowProviderDeprecationWarning,
|
400
|
+
)
|
389
401
|
def client_info(self) -> ClientInfo:
|
390
402
|
"""
|
391
403
|
Return client information used to generate a user-agent for API calls.
|
@@ -396,11 +408,6 @@ class GoogleBaseHook(BaseHook):
|
|
396
408
|
the Google Cloud. It is not supported by The Google APIs Python Client that use Discovery
|
397
409
|
based APIs.
|
398
410
|
"""
|
399
|
-
warnings.warn(
|
400
|
-
"This method is deprecated, please use `airflow.providers.google.common.consts.CLIENT_INFO`.",
|
401
|
-
AirflowProviderDeprecationWarning,
|
402
|
-
stacklevel=2,
|
403
|
-
)
|
404
411
|
return CLIENT_INFO
|
405
412
|
|
406
413
|
@property
|
@@ -416,7 +423,7 @@ class GoogleBaseHook(BaseHook):
|
|
416
423
|
|
417
424
|
@staticmethod
|
418
425
|
def quota_retry(*args, **kwargs) -> Callable:
|
419
|
-
"""
|
426
|
+
"""Provide a mechanism to repeat requests in response to exceeding a temporary quota limit."""
|
420
427
|
|
421
428
|
def decorator(fun: Callable):
|
422
429
|
default_kwargs = {
|
@@ -432,7 +439,7 @@ class GoogleBaseHook(BaseHook):
|
|
432
439
|
|
433
440
|
@staticmethod
|
434
441
|
def operation_in_progress_retry(*args, **kwargs) -> Callable[[T], T]:
|
435
|
-
"""
|
442
|
+
"""Provide a mechanism to repeat requests in response to operation in progress (HTTP 409) limit."""
|
436
443
|
|
437
444
|
def decorator(fun: T):
|
438
445
|
default_kwargs = {
|
@@ -449,7 +456,7 @@ class GoogleBaseHook(BaseHook):
|
|
449
456
|
@staticmethod
|
450
457
|
def fallback_to_default_project_id(func: Callable[..., RT]) -> Callable[..., RT]:
|
451
458
|
"""
|
452
|
-
|
459
|
+
Provide fallback for Google Cloud project id. To be used as a decorator.
|
453
460
|
|
454
461
|
If the project is None it will be replaced with the project_id from the
|
455
462
|
service account the Hook is authenticated with. Project id can be specified
|
@@ -482,7 +489,7 @@ class GoogleBaseHook(BaseHook):
|
|
482
489
|
@staticmethod
|
483
490
|
def provide_gcp_credential_file(func: T) -> T:
|
484
491
|
"""
|
485
|
-
|
492
|
+
Provide a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
|
486
493
|
|
487
494
|
It is recommended to use ``provide_gcp_credential_file_as_context`` context
|
488
495
|
manager to limit the scope when authorization data is available. Using context
|
@@ -499,7 +506,7 @@ class GoogleBaseHook(BaseHook):
|
|
499
506
|
@contextmanager
|
500
507
|
def provide_gcp_credential_file_as_context(self) -> Generator[str | None, None, None]:
|
501
508
|
"""
|
502
|
-
|
509
|
+
Provide a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
|
503
510
|
|
504
511
|
See:
|
505
512
|
`Application Default Credentials (ADC)
|
@@ -535,7 +542,7 @@ class GoogleBaseHook(BaseHook):
|
|
535
542
|
@contextmanager
|
536
543
|
def provide_authorized_gcloud(self) -> Generator[None, None, None]:
|
537
544
|
"""
|
538
|
-
|
545
|
+
Provide a separate gcloud configuration with current credentials.
|
539
546
|
|
540
547
|
The gcloud tool allows you to login to Google Cloud only - ``gcloud auth login`` and
|
541
548
|
for the needs of Application Default Credentials ``gcloud auth application-default login``.
|
@@ -623,13 +630,60 @@ class GoogleBaseHook(BaseHook):
|
|
623
630
|
return status, message
|
624
631
|
|
625
632
|
|
633
|
+
class _CredentialsToken(Token):
|
634
|
+
"""A token implementation which makes Google credentials objects accessible to [gcloud-aio](https://talkiq.github.io/gcloud-aio/) clients.
|
635
|
+
|
636
|
+
This class allows us to create token instances from credentials objects and thus supports a variety of use cases for Google
|
637
|
+
credentials in Airflow (i.e. impersonation chain). By relying on a existing credentials object we leverage functionality provided by the GoogleBaseHook
|
638
|
+
for generating credentials objects.
|
639
|
+
"""
|
640
|
+
|
641
|
+
def __init__(
|
642
|
+
self,
|
643
|
+
credentials: Credentials,
|
644
|
+
*,
|
645
|
+
project: str | None = None,
|
646
|
+
session: ClientSession | None = None,
|
647
|
+
scopes: Sequence[str] | None = None,
|
648
|
+
) -> None:
|
649
|
+
_scopes: list[str] | None = list(scopes) if scopes else None
|
650
|
+
super().__init__(session=cast(Session, session), scopes=_scopes)
|
651
|
+
self.credentials = credentials
|
652
|
+
self.project = project
|
653
|
+
|
654
|
+
@classmethod
|
655
|
+
async def from_hook(
|
656
|
+
cls,
|
657
|
+
hook: GoogleBaseHook,
|
658
|
+
*,
|
659
|
+
session: ClientSession | None = None,
|
660
|
+
) -> _CredentialsToken:
|
661
|
+
credentials, project = hook.get_credentials_and_project_id()
|
662
|
+
return cls(
|
663
|
+
credentials=credentials,
|
664
|
+
project=project,
|
665
|
+
session=session,
|
666
|
+
scopes=hook.scopes,
|
667
|
+
)
|
668
|
+
|
669
|
+
async def get_project(self) -> str | None:
|
670
|
+
return self.project
|
671
|
+
|
672
|
+
async def acquire_access_token(self, timeout: int = 10) -> None:
|
673
|
+
await sync_to_async(self.credentials.refresh)(google.auth.transport.requests.Request())
|
674
|
+
|
675
|
+
self.access_token = cast(str, self.credentials.token)
|
676
|
+
self.access_token_duration = 3600
|
677
|
+
self.access_token_acquired_at = datetime.datetime.utcnow()
|
678
|
+
self.acquiring = None
|
679
|
+
|
680
|
+
|
626
681
|
class GoogleBaseAsyncHook(BaseHook):
|
627
682
|
"""GoogleBaseAsyncHook inherits from BaseHook class, run on the trigger worker."""
|
628
683
|
|
629
684
|
sync_hook_class: Any = None
|
630
685
|
|
631
686
|
def __init__(self, **kwargs: Any):
|
632
|
-
super().__init__(logger_name=kwargs.pop("logger_name", None))
|
633
687
|
self._hook_kwargs = kwargs
|
634
688
|
self._sync_hook = None
|
635
689
|
|
@@ -639,6 +693,16 @@ class GoogleBaseAsyncHook(BaseHook):
|
|
639
693
|
self._sync_hook = await sync_to_async(self.sync_hook_class)(**self._hook_kwargs)
|
640
694
|
return self._sync_hook
|
641
695
|
|
696
|
+
async def get_token(self, *, session: ClientSession | None = None) -> _CredentialsToken:
|
697
|
+
"""Return a Token instance for use in [gcloud-aio](https://talkiq.github.io/gcloud-aio/) clients."""
|
698
|
+
sync_hook = await self.get_sync_hook()
|
699
|
+
return await _CredentialsToken.from_hook(sync_hook, session=session)
|
700
|
+
|
642
701
|
async def service_file_as_context(self) -> Any:
|
702
|
+
"""
|
703
|
+
Provide a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
|
704
|
+
|
705
|
+
This is the async equivalent of the non-async GoogleBaseHook's `provide_gcp_credential_file_as_context` method.
|
706
|
+
"""
|
643
707
|
sync_hook = await self.get_sync_hook()
|
644
708
|
return await sync_to_async(sync_hook.provide_gcp_credential_file_as_context)()
|
@@ -66,7 +66,7 @@ class GoogleDiscoveryApiHook(GoogleBaseHook):
|
|
66
66
|
|
67
67
|
def get_conn(self) -> Resource:
|
68
68
|
"""
|
69
|
-
|
69
|
+
Create an authenticated api client for the given api service name and credentials.
|
70
70
|
|
71
71
|
:return: the authenticated api service.
|
72
72
|
"""
|
@@ -84,7 +84,7 @@ class GoogleDiscoveryApiHook(GoogleBaseHook):
|
|
84
84
|
|
85
85
|
def query(self, endpoint: str, data: dict, paginate: bool = False, num_retries: int = 0) -> dict:
|
86
86
|
"""
|
87
|
-
|
87
|
+
Create a dynamic API call to any Google API registered in Google's API Client Library and queries it.
|
88
88
|
|
89
89
|
:param endpoint: The client libraries path to the api call's executing method.
|
90
90
|
For example: 'analyticsreporting.reports.batchGet'
|
@@ -81,7 +81,7 @@ def _load_credentials_from_file(
|
|
81
81
|
filename: str, target_audience: str | None
|
82
82
|
) -> google_auth_credentials.Credentials | None:
|
83
83
|
"""
|
84
|
-
|
84
|
+
Load credentials from a file.
|
85
85
|
|
86
86
|
The credentials file must be a service account key or a stored authorized user credential.
|
87
87
|
|
@@ -129,7 +129,7 @@ def _load_credentials_from_file(
|
|
129
129
|
def _get_explicit_environ_credentials(
|
130
130
|
target_audience: str | None,
|
131
131
|
) -> google_auth_credentials.Credentials | None:
|
132
|
-
"""
|
132
|
+
"""Get credentials from the GOOGLE_APPLICATION_CREDENTIALS environment variable."""
|
133
133
|
explicit_file = os.environ.get(environment_vars.CREDENTIALS)
|
134
134
|
|
135
135
|
if explicit_file is None:
|
@@ -145,7 +145,7 @@ def _get_explicit_environ_credentials(
|
|
145
145
|
def _get_gcloud_sdk_credentials(
|
146
146
|
target_audience: str | None,
|
147
147
|
) -> google_auth_credentials.Credentials | None:
|
148
|
-
"""
|
148
|
+
"""Get the credentials and project ID from the Cloud SDK."""
|
149
149
|
from google.auth import _cloud_sdk # type: ignore[attr-defined]
|
150
150
|
|
151
151
|
# Check if application default credentials exist.
|
@@ -162,7 +162,7 @@ def _get_gcloud_sdk_credentials(
|
|
162
162
|
def _get_gce_credentials(
|
163
163
|
target_audience: str | None, request: google.auth.transport.Request | None = None
|
164
164
|
) -> google_auth_credentials.Credentials | None:
|
165
|
-
"""
|
165
|
+
"""Get credentials and project ID from the GCE Metadata Service."""
|
166
166
|
# Ping requires a transport, but we want application default credentials
|
167
167
|
# to require no arguments. So, we'll use the _http_client transport which
|
168
168
|
# uses http.client. This is only acceptable because the metadata server
|
@@ -191,7 +191,7 @@ def _get_gce_credentials(
|
|
191
191
|
def get_default_id_token_credentials(
|
192
192
|
target_audience: str | None, request: google.auth.transport.Request = None
|
193
193
|
) -> google_auth_credentials.Credentials:
|
194
|
-
"""
|
194
|
+
"""Get the default ID Token credentials for the current environment.
|
195
195
|
|
196
196
|
`Application Default Credentials`_ provides an easy way to obtain credentials to call Google APIs for
|
197
197
|
server-to-server or local applications.
|
@@ -65,7 +65,7 @@ class CloudFirestoreHook(GoogleBaseHook):
|
|
65
65
|
|
66
66
|
def get_conn(self):
|
67
67
|
"""
|
68
|
-
|
68
|
+
Retrieve the connection to Cloud Firestore.
|
69
69
|
|
70
70
|
:return: Google Cloud Firestore services object.
|
71
71
|
"""
|
@@ -86,7 +86,7 @@ class CloudFirestoreHook(GoogleBaseHook):
|
|
86
86
|
self, body: dict, database_id: str = "(default)", project_id: str | None = None
|
87
87
|
) -> None:
|
88
88
|
"""
|
89
|
-
|
89
|
+
Start a export with the specified configuration.
|
90
90
|
|
91
91
|
:param database_id: The Database ID.
|
92
92
|
:param body: The request body.
|
@@ -110,7 +110,7 @@ class CloudFirestoreHook(GoogleBaseHook):
|
|
110
110
|
|
111
111
|
def _wait_for_operation_to_complete(self, operation_name: str) -> None:
|
112
112
|
"""
|
113
|
-
|
113
|
+
Wait for the named operation to complete - checks status of the asynchronous call.
|
114
114
|
|
115
115
|
:param operation_name: The name of the operation.
|
116
116
|
:return: The response returned by the operation.
|
@@ -28,8 +28,9 @@ def get_provider_info():
|
|
28
28
|
"name": "Google",
|
29
29
|
"description": "Google services including:\n\n - `Google Ads <https://ads.google.com/>`__\n - `Google Cloud (GCP) <https://cloud.google.com/>`__\n - `Google Firebase <https://firebase.google.com/>`__\n - `Google LevelDB <https://github.com/google/leveldb/>`__\n - `Google Marketing Platform <https://marketingplatform.google.com/>`__\n - `Google Workspace <https://workspace.google.com/>`__ (formerly Google Suite)\n",
|
30
30
|
"state": "ready",
|
31
|
-
"source-date-epoch":
|
31
|
+
"source-date-epoch": 1707636385,
|
32
32
|
"versions": [
|
33
|
+
"10.15.0",
|
33
34
|
"10.14.0",
|
34
35
|
"10.13.1",
|
35
36
|
"10.13.0",
|
@@ -92,7 +93,7 @@ def get_provider_info():
|
|
92
93
|
"gcsfs>=2023.10.0",
|
93
94
|
"google-ads>=22.1.0",
|
94
95
|
"google-analytics-admin",
|
95
|
-
"google-api-core>=2.11.0",
|
96
|
+
"google-api-core>=2.11.0,!=2.16.0",
|
96
97
|
"google-api-python-client>=1.6.0",
|
97
98
|
"google-auth>=1.0.0",
|
98
99
|
"google-auth-httplib2>=0.0.1",
|
@@ -1215,6 +1216,10 @@ def get_provider_info():
|
|
1215
1216
|
"integration-name": "Google Cloud",
|
1216
1217
|
"python-modules": ["airflow.providers.google.cloud.triggers.cloud_batch"],
|
1217
1218
|
},
|
1219
|
+
{
|
1220
|
+
"integration-name": "Google Vertex AI",
|
1221
|
+
"python-modules": ["airflow.providers.google.cloud.triggers.vertex_ai"],
|
1222
|
+
},
|
1218
1223
|
],
|
1219
1224
|
"transfers": [
|
1220
1225
|
{
|
@@ -45,15 +45,15 @@ class LevelDBHook(BaseHook):
|
|
45
45
|
conn_type = "leveldb"
|
46
46
|
hook_name = "LevelDB"
|
47
47
|
|
48
|
-
def __init__(self, leveldb_conn_id: str = default_conn_name
|
49
|
-
super().__init__(
|
48
|
+
def __init__(self, leveldb_conn_id: str = default_conn_name):
|
49
|
+
super().__init__()
|
50
50
|
self.leveldb_conn_id = leveldb_conn_id
|
51
51
|
self.connection = self.get_connection(leveldb_conn_id)
|
52
52
|
self.db: plyvel.DB | None = None
|
53
53
|
|
54
54
|
def get_conn(self, name: str = "/tmp/testdb/", create_if_missing: bool = False, **kwargs) -> DB:
|
55
55
|
"""
|
56
|
-
|
56
|
+
Create `Plyvel DB <https://plyvel.readthedocs.io/en/latest/api.html#DB>`__.
|
57
57
|
|
58
58
|
:param name: path to create database e.g. `/tmp/testdb/`)
|
59
59
|
:param create_if_missing: whether a new database should be created if needed
|
@@ -66,7 +66,7 @@ class LevelDBHook(BaseHook):
|
|
66
66
|
return self.db
|
67
67
|
|
68
68
|
def close_conn(self) -> None:
|
69
|
-
"""
|
69
|
+
"""Close connection."""
|
70
70
|
db = self.db
|
71
71
|
if db is not None:
|
72
72
|
db.close()
|
@@ -17,9 +17,9 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
import warnings
|
21
20
|
from typing import Any
|
22
21
|
|
22
|
+
from deprecated import deprecated
|
23
23
|
from googleapiclient.discovery import Resource, build
|
24
24
|
from googleapiclient.http import MediaFileUpload
|
25
25
|
|
@@ -27,18 +27,15 @@ from airflow.exceptions import AirflowProviderDeprecationWarning
|
|
27
27
|
from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
|
28
28
|
|
29
29
|
|
30
|
+
@deprecated(
|
31
|
+
reason="The `GoogleAnalyticsHook` class is deprecated, please use `GoogleAnalyticsAdminHook` instead.",
|
32
|
+
category=AirflowProviderDeprecationWarning,
|
33
|
+
)
|
30
34
|
class GoogleAnalyticsHook(GoogleBaseHook):
|
31
35
|
"""Hook for Google Analytics 360."""
|
32
36
|
|
33
37
|
def __init__(self, api_version: str = "v3", *args, **kwargs):
|
34
38
|
super().__init__(*args, **kwargs)
|
35
|
-
warnings.warn(
|
36
|
-
f"The `{type(self).__name__}` class is deprecated, please use "
|
37
|
-
f"`GoogleAnalyticsAdminHook` instead.",
|
38
|
-
AirflowProviderDeprecationWarning,
|
39
|
-
stacklevel=2,
|
40
|
-
)
|
41
|
-
|
42
39
|
self.api_version = api_version
|
43
40
|
self._conn = None
|
44
41
|
|
@@ -58,7 +55,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
|
|
58
55
|
return result
|
59
56
|
|
60
57
|
def get_conn(self) -> Resource:
|
61
|
-
"""
|
58
|
+
"""Retrieve connection to Google Analytics 360."""
|
62
59
|
if not self._conn:
|
63
60
|
http_authorized = self._authorize()
|
64
61
|
self._conn = build(
|
@@ -70,7 +67,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
|
|
70
67
|
return self._conn
|
71
68
|
|
72
69
|
def list_accounts(self) -> list[dict[str, Any]]:
|
73
|
-
"""
|
70
|
+
"""List accounts list from Google Analytics 360."""
|
74
71
|
self.log.info("Retrieving accounts list...")
|
75
72
|
conn = self.get_conn()
|
76
73
|
accounts = conn.management().accounts()
|
@@ -81,7 +78,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
|
|
81
78
|
self, account_id: str, web_property_id: str, web_property_ad_words_link_id: str
|
82
79
|
) -> dict[str, Any]:
|
83
80
|
"""
|
84
|
-
|
81
|
+
Return a web property-Google Ads link to which the user has access.
|
85
82
|
|
86
83
|
:param account_id: ID of the account which the given web property belongs to.
|
87
84
|
:param web_property_id: Web property-Google Ads link UA-string.
|
@@ -105,7 +102,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
|
|
105
102
|
|
106
103
|
def list_ad_words_links(self, account_id: str, web_property_id: str) -> list[dict[str, Any]]:
|
107
104
|
"""
|
108
|
-
|
105
|
+
List webProperty-Google Ads links for a given web property.
|
109
106
|
|
110
107
|
:param account_id: ID of the account which the given web property belongs to.
|
111
108
|
:param web_property_id: Web property UA-string to retrieve the Google Ads links for.
|
@@ -128,7 +125,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
|
|
128
125
|
resumable_upload: bool = False,
|
129
126
|
) -> None:
|
130
127
|
"""
|
131
|
-
|
128
|
+
Upload file to GA via the Data Import API.
|
132
129
|
|
133
130
|
:param file_location: The path and name of the file to upload.
|
134
131
|
:param account_id: The GA account Id to which the data upload belongs.
|
@@ -165,7 +162,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
|
|
165
162
|
delete_request_body: dict[str, Any],
|
166
163
|
) -> None:
|
167
164
|
"""
|
168
|
-
|
165
|
+
Delete the uploaded data for a given account/property/dataset.
|
169
166
|
|
170
167
|
:param account_id: The GA account Id to which the data upload belongs.
|
171
168
|
:param web_property_id: UA-string associated with the upload.
|
@@ -49,7 +49,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
|
|
49
49
|
self.api_version = api_version
|
50
50
|
|
51
51
|
def get_conn(self) -> Resource:
|
52
|
-
"""
|
52
|
+
"""Retrieve connection to Campaign Manager."""
|
53
53
|
if not self._conn:
|
54
54
|
http_authorized = self._authorize()
|
55
55
|
self._conn = build(
|
@@ -62,7 +62,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
|
|
62
62
|
|
63
63
|
def delete_report(self, profile_id: str, report_id: str) -> Any:
|
64
64
|
"""
|
65
|
-
|
65
|
+
Delete a report by its ID.
|
66
66
|
|
67
67
|
:param profile_id: The DFA user profile ID.
|
68
68
|
:param report_id: The ID of the report.
|
@@ -77,7 +77,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
|
|
77
77
|
|
78
78
|
def insert_report(self, profile_id: str, report: dict[str, Any]) -> Any:
|
79
79
|
"""
|
80
|
-
|
80
|
+
Create a report.
|
81
81
|
|
82
82
|
:param profile_id: The DFA user profile ID.
|
83
83
|
:param report: The report resource to be inserted.
|
@@ -99,7 +99,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
|
|
99
99
|
sort_order: str | None = None,
|
100
100
|
) -> list[dict]:
|
101
101
|
"""
|
102
|
-
|
102
|
+
Retrieve list of reports.
|
103
103
|
|
104
104
|
:param profile_id: The DFA user profile ID.
|
105
105
|
:param max_results: Maximum number of results to return.
|
@@ -125,7 +125,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
|
|
125
125
|
|
126
126
|
def patch_report(self, profile_id: str, report_id: str, update_mask: dict) -> Any:
|
127
127
|
"""
|
128
|
-
|
128
|
+
Update a report. This method supports patch semantics.
|
129
129
|
|
130
130
|
:param profile_id: The DFA user profile ID.
|
131
131
|
:param report_id: The ID of the report.
|
@@ -142,7 +142,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
|
|
142
142
|
|
143
143
|
def run_report(self, profile_id: str, report_id: str, synchronous: bool | None = None) -> Any:
|
144
144
|
"""
|
145
|
-
|
145
|
+
Run a report.
|
146
146
|
|
147
147
|
:param profile_id: The DFA profile ID.
|
148
148
|
:param report_id: The ID of the report.
|
@@ -158,7 +158,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
|
|
158
158
|
|
159
159
|
def update_report(self, profile_id: str, report_id: str) -> Any:
|
160
160
|
"""
|
161
|
-
|
161
|
+
Update a report.
|
162
162
|
|
163
163
|
:param profile_id: The DFA user profile ID.
|
164
164
|
:param report_id: The ID of the report.
|
@@ -173,7 +173,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
|
|
173
173
|
|
174
174
|
def get_report(self, file_id: str, profile_id: str, report_id: str) -> Any:
|
175
175
|
"""
|
176
|
-
|
176
|
+
Retrieve a report file.
|
177
177
|
|
178
178
|
:param profile_id: The DFA user profile ID.
|
179
179
|
:param report_id: The ID of the report.
|
@@ -190,7 +190,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
|
|
190
190
|
|
191
191
|
def get_report_file(self, file_id: str, profile_id: str, report_id: str) -> http.HttpRequest:
|
192
192
|
"""
|
193
|
-
|
193
|
+
Retrieve a media part of report file.
|
194
194
|
|
195
195
|
:param profile_id: The DFA user profile ID.
|
196
196
|
:param report_id: The ID of the report.
|
@@ -234,7 +234,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
|
|
234
234
|
max_failed_inserts: int = 0,
|
235
235
|
) -> Any:
|
236
236
|
"""
|
237
|
-
|
237
|
+
Insert conversions.
|
238
238
|
|
239
239
|
:param profile_id: User profile ID associated with this request.
|
240
240
|
:param conversions: Conversations to insert, should by type of Conversation:
|
@@ -278,7 +278,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
|
|
278
278
|
max_failed_updates: int = 0,
|
279
279
|
) -> Any:
|
280
280
|
"""
|
281
|
-
|
281
|
+
Update existing conversions.
|
282
282
|
|
283
283
|
:param profile_id: User profile ID associated with this request.
|
284
284
|
:param conversions: Conversations to update, should by type of Conversation:
|