apache-airflow-providers-google 10.14.0rc1__py3-none-any.whl → 10.15.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. airflow/providers/google/__init__.py +1 -1
  2. airflow/providers/google/ads/hooks/ads.py +1 -2
  3. airflow/providers/google/cloud/hooks/automl.py +13 -13
  4. airflow/providers/google/cloud/hooks/bigquery.py +208 -256
  5. airflow/providers/google/cloud/hooks/bigquery_dts.py +6 -6
  6. airflow/providers/google/cloud/hooks/bigtable.py +8 -8
  7. airflow/providers/google/cloud/hooks/cloud_batch.py +1 -1
  8. airflow/providers/google/cloud/hooks/cloud_build.py +19 -20
  9. airflow/providers/google/cloud/hooks/cloud_composer.py +4 -4
  10. airflow/providers/google/cloud/hooks/cloud_memorystore.py +10 -10
  11. airflow/providers/google/cloud/hooks/cloud_run.py +1 -1
  12. airflow/providers/google/cloud/hooks/cloud_sql.py +18 -19
  13. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +3 -3
  14. airflow/providers/google/cloud/hooks/compute.py +16 -16
  15. airflow/providers/google/cloud/hooks/compute_ssh.py +1 -1
  16. airflow/providers/google/cloud/hooks/datacatalog.py +22 -22
  17. airflow/providers/google/cloud/hooks/dataflow.py +48 -49
  18. airflow/providers/google/cloud/hooks/dataform.py +16 -16
  19. airflow/providers/google/cloud/hooks/datafusion.py +15 -15
  20. airflow/providers/google/cloud/hooks/datapipeline.py +3 -3
  21. airflow/providers/google/cloud/hooks/dataplex.py +19 -19
  22. airflow/providers/google/cloud/hooks/dataprep.py +10 -10
  23. airflow/providers/google/cloud/hooks/dataproc.py +132 -14
  24. airflow/providers/google/cloud/hooks/dataproc_metastore.py +13 -13
  25. airflow/providers/google/cloud/hooks/datastore.py +3 -3
  26. airflow/providers/google/cloud/hooks/dlp.py +25 -25
  27. airflow/providers/google/cloud/hooks/gcs.py +39 -27
  28. airflow/providers/google/cloud/hooks/gdm.py +3 -3
  29. airflow/providers/google/cloud/hooks/kms.py +3 -3
  30. airflow/providers/google/cloud/hooks/kubernetes_engine.py +63 -48
  31. airflow/providers/google/cloud/hooks/life_sciences.py +13 -12
  32. airflow/providers/google/cloud/hooks/looker.py +8 -9
  33. airflow/providers/google/cloud/hooks/mlengine.py +12 -12
  34. airflow/providers/google/cloud/hooks/natural_language.py +2 -2
  35. airflow/providers/google/cloud/hooks/os_login.py +1 -1
  36. airflow/providers/google/cloud/hooks/pubsub.py +9 -9
  37. airflow/providers/google/cloud/hooks/secret_manager.py +1 -1
  38. airflow/providers/google/cloud/hooks/spanner.py +11 -11
  39. airflow/providers/google/cloud/hooks/speech_to_text.py +1 -1
  40. airflow/providers/google/cloud/hooks/stackdriver.py +7 -7
  41. airflow/providers/google/cloud/hooks/tasks.py +11 -11
  42. airflow/providers/google/cloud/hooks/text_to_speech.py +1 -1
  43. airflow/providers/google/cloud/hooks/translate.py +1 -1
  44. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +13 -13
  45. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -6
  46. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +45 -50
  47. airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +13 -13
  48. airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +9 -9
  49. airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +128 -11
  50. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +10 -10
  51. airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +8 -8
  52. airflow/providers/google/cloud/hooks/video_intelligence.py +2 -2
  53. airflow/providers/google/cloud/hooks/vision.py +1 -1
  54. airflow/providers/google/cloud/hooks/workflows.py +10 -10
  55. airflow/providers/google/cloud/links/datafusion.py +12 -5
  56. airflow/providers/google/cloud/operators/bigquery.py +11 -11
  57. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +3 -1
  58. airflow/providers/google/cloud/operators/dataflow.py +16 -16
  59. airflow/providers/google/cloud/operators/datafusion.py +9 -1
  60. airflow/providers/google/cloud/operators/dataproc.py +444 -69
  61. airflow/providers/google/cloud/operators/kubernetes_engine.py +6 -6
  62. airflow/providers/google/cloud/operators/life_sciences.py +10 -9
  63. airflow/providers/google/cloud/operators/mlengine.py +96 -96
  64. airflow/providers/google/cloud/operators/pubsub.py +2 -0
  65. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +33 -3
  66. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +59 -2
  67. airflow/providers/google/cloud/secrets/secret_manager.py +8 -7
  68. airflow/providers/google/cloud/sensors/bigquery.py +20 -16
  69. airflow/providers/google/cloud/sensors/cloud_composer.py +11 -8
  70. airflow/providers/google/cloud/sensors/dataproc_metastore.py +12 -2
  71. airflow/providers/google/cloud/sensors/gcs.py +8 -7
  72. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +1 -0
  73. airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +4 -4
  74. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +1 -0
  75. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
  76. airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
  77. airflow/providers/google/cloud/transfers/mysql_to_gcs.py +1 -1
  78. airflow/providers/google/cloud/transfers/oracle_to_gcs.py +1 -1
  79. airflow/providers/google/cloud/transfers/postgres_to_gcs.py +1 -1
  80. airflow/providers/google/cloud/transfers/presto_to_gcs.py +1 -1
  81. airflow/providers/google/cloud/transfers/s3_to_gcs.py +3 -3
  82. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
  83. airflow/providers/google/cloud/transfers/sql_to_gcs.py +3 -3
  84. airflow/providers/google/cloud/transfers/trino_to_gcs.py +1 -1
  85. airflow/providers/google/cloud/triggers/bigquery.py +12 -12
  86. airflow/providers/google/cloud/triggers/bigquery_dts.py +1 -1
  87. airflow/providers/google/cloud/triggers/cloud_batch.py +3 -1
  88. airflow/providers/google/cloud/triggers/cloud_build.py +2 -2
  89. airflow/providers/google/cloud/triggers/cloud_run.py +1 -1
  90. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +6 -6
  91. airflow/providers/google/cloud/triggers/dataflow.py +3 -1
  92. airflow/providers/google/cloud/triggers/datafusion.py +2 -2
  93. airflow/providers/google/cloud/triggers/dataplex.py +2 -2
  94. airflow/providers/google/cloud/triggers/dataproc.py +34 -14
  95. airflow/providers/google/cloud/triggers/gcs.py +12 -8
  96. airflow/providers/google/cloud/triggers/kubernetes_engine.py +2 -2
  97. airflow/providers/google/cloud/triggers/mlengine.py +2 -2
  98. airflow/providers/google/cloud/triggers/pubsub.py +1 -1
  99. airflow/providers/google/cloud/triggers/vertex_ai.py +99 -0
  100. airflow/providers/google/cloud/utils/bigquery.py +2 -2
  101. airflow/providers/google/cloud/utils/credentials_provider.py +2 -2
  102. airflow/providers/google/cloud/utils/dataform.py +1 -1
  103. airflow/providers/google/cloud/utils/dataproc.py +25 -0
  104. airflow/providers/google/cloud/utils/field_validator.py +2 -2
  105. airflow/providers/google/cloud/utils/helpers.py +2 -2
  106. airflow/providers/google/cloud/utils/mlengine_operator_utils.py +1 -1
  107. airflow/providers/google/cloud/utils/mlengine_prediction_summary.py +1 -1
  108. airflow/providers/google/common/auth_backend/google_openid.py +2 -2
  109. airflow/providers/google/common/hooks/base_google.py +87 -23
  110. airflow/providers/google/common/hooks/discovery_api.py +2 -2
  111. airflow/providers/google/common/utils/id_token_credentials.py +5 -5
  112. airflow/providers/google/firebase/hooks/firestore.py +3 -3
  113. airflow/providers/google/get_provider_info.py +7 -2
  114. airflow/providers/google/leveldb/hooks/leveldb.py +4 -4
  115. airflow/providers/google/marketing_platform/hooks/analytics.py +11 -14
  116. airflow/providers/google/marketing_platform/hooks/campaign_manager.py +11 -11
  117. airflow/providers/google/marketing_platform/hooks/display_video.py +13 -13
  118. airflow/providers/google/marketing_platform/hooks/search_ads.py +4 -4
  119. airflow/providers/google/marketing_platform/operators/analytics.py +37 -32
  120. airflow/providers/google/suite/hooks/calendar.py +2 -2
  121. airflow/providers/google/suite/hooks/drive.py +7 -7
  122. airflow/providers/google/suite/hooks/sheets.py +8 -8
  123. {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/METADATA +11 -11
  124. {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/RECORD +126 -124
  125. {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/WHEEL +0 -0
  126. {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/entry_points.txt +0 -0
@@ -18,12 +18,12 @@
18
18
  """This module contains a Google Cloud API base hook."""
19
19
  from __future__ import annotations
20
20
 
21
+ import datetime
21
22
  import functools
22
23
  import json
23
24
  import logging
24
25
  import os
25
26
  import tempfile
26
- import warnings
27
27
  from contextlib import ExitStack, contextmanager
28
28
  from subprocess import check_output
29
29
  from typing import TYPE_CHECKING, Any, Callable, Generator, Sequence, TypeVar, cast
@@ -35,6 +35,8 @@ import google_auth_httplib2
35
35
  import requests
36
36
  import tenacity
37
37
  from asgiref.sync import sync_to_async
38
+ from deprecated import deprecated
39
+ from gcloud.aio.auth.token import Token
38
40
  from google.api_core.exceptions import Forbidden, ResourceExhausted, TooManyRequests
39
41
  from google.auth import _cloud_sdk, compute_engine # type: ignore[attr-defined]
40
42
  from google.auth.environment_vars import CLOUD_SDK_CONFIG_DIR, CREDENTIALS
@@ -43,6 +45,7 @@ from google.auth.transport import _http_client
43
45
  from googleapiclient import discovery
44
46
  from googleapiclient.errors import HttpError
45
47
  from googleapiclient.http import MediaIoBaseDownload, build_http, set_user_agent
48
+ from requests import Session
46
49
 
47
50
  from airflow import version
48
51
  from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
@@ -56,7 +59,9 @@ from airflow.providers.google.common.consts import CLIENT_INFO
56
59
  from airflow.utils.process_utils import patch_environ
57
60
 
58
61
  if TYPE_CHECKING:
62
+ from aiohttp import ClientSession
59
63
  from google.api_core.gapic_v1.client_info import ClientInfo
64
+ from google.auth.credentials import Credentials
60
65
 
61
66
  log = logging.getLogger(__name__)
62
67
 
@@ -74,6 +79,8 @@ INVALID_REASONS = [
74
79
 
75
80
  def is_soft_quota_exception(exception: Exception):
76
81
  """
82
+ Check for quota violation errors.
83
+
77
84
  API for Google services does not have a standardized way to report quota violation errors.
78
85
 
79
86
  The function has been adapted by trial and error to the following services:
@@ -95,6 +102,8 @@ def is_soft_quota_exception(exception: Exception):
95
102
 
96
103
  def is_operation_in_progress_exception(exception: Exception) -> bool:
97
104
  """
105
+ Handle operation in-progress exceptions.
106
+
98
107
  Some calls return 429 (too many requests!) or 409 errors (Conflict) in case of operation in progress.
99
108
 
100
109
  * Google Cloud SQL
@@ -190,7 +199,7 @@ class GoogleBaseHook(BaseHook):
190
199
 
191
200
  @classmethod
192
201
  def get_connection_form_widgets(cls) -> dict[str, Any]:
193
- """Returns connection widgets to add to connection form."""
202
+ """Return connection widgets to add to connection form."""
194
203
  from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget
195
204
  from flask_babel import lazy_gettext
196
205
  from wtforms import IntegerField, PasswordField, StringField
@@ -223,7 +232,7 @@ class GoogleBaseHook(BaseHook):
223
232
 
224
233
  @classmethod
225
234
  def get_ui_field_behaviour(cls) -> dict[str, Any]:
226
- """Returns custom field behaviour."""
235
+ """Return custom field behaviour."""
227
236
  return {
228
237
  "hidden_fields": ["host", "schema", "login", "password", "port", "extra"],
229
238
  "relabeling": {},
@@ -234,9 +243,8 @@ class GoogleBaseHook(BaseHook):
234
243
  gcp_conn_id: str = "google_cloud_default",
235
244
  delegate_to: str | None = None,
236
245
  impersonation_chain: str | Sequence[str] | None = None,
237
- **kwargs,
238
246
  ) -> None:
239
- super().__init__(**kwargs)
247
+ super().__init__()
240
248
  self.gcp_conn_id = gcp_conn_id
241
249
  self.delegate_to = delegate_to
242
250
  self.impersonation_chain = impersonation_chain
@@ -245,7 +253,7 @@ class GoogleBaseHook(BaseHook):
245
253
  self._cached_project_id: str | None = None
246
254
 
247
255
  def get_credentials_and_project_id(self) -> tuple[google.auth.credentials.Credentials, str | None]:
248
- """Returns the Credentials object for Google API and the associated project_id."""
256
+ """Return the Credentials object for Google API and the associated project_id."""
249
257
  if self._cached_credentials is not None:
250
258
  return self._cached_credentials, self._cached_project_id
251
259
 
@@ -295,12 +303,12 @@ class GoogleBaseHook(BaseHook):
295
303
  return credentials, project_id
296
304
 
297
305
  def get_credentials(self) -> google.auth.credentials.Credentials:
298
- """Returns the Credentials object for Google API."""
306
+ """Return the Credentials object for Google API."""
299
307
  credentials, _ = self.get_credentials_and_project_id()
300
308
  return credentials
301
309
 
302
310
  def _get_access_token(self) -> str:
303
- """Returns a valid access token from Google API Credentials."""
311
+ """Return a valid access token from Google API Credentials."""
304
312
  credentials = self.get_credentials()
305
313
  auth_req = google.auth.transport.requests.Request()
306
314
  # credentials.token is None
@@ -311,7 +319,7 @@ class GoogleBaseHook(BaseHook):
311
319
  @functools.lru_cache(maxsize=None)
312
320
  def _get_credentials_email(self) -> str:
313
321
  """
314
- Returns the email address associated with the currently logged in account.
322
+ Return the email address associated with the currently logged in account.
315
323
 
316
324
  If a service account is used, it returns the service account.
317
325
  If user authentication (e.g. gcloud auth) is used, it returns the e-mail account of that user.
@@ -337,7 +345,7 @@ class GoogleBaseHook(BaseHook):
337
345
  return oauth2_client.tokeninfo().execute()["email"]
338
346
 
339
347
  def _authorize(self) -> google_auth_httplib2.AuthorizedHttp:
340
- """Returns an authorized HTTP object to be used to build a Google cloud service hook connection."""
348
+ """Return an authorized HTTP object to be used to build a Google cloud service hook connection."""
341
349
  credentials = self.get_credentials()
342
350
  http = build_http()
343
351
  http = set_user_agent(http, "airflow/" + version.version)
@@ -346,7 +354,7 @@ class GoogleBaseHook(BaseHook):
346
354
 
347
355
  def _get_field(self, f: str, default: Any = None) -> Any:
348
356
  """
349
- Fetches a field from extras, and returns it.
357
+ Fetch a field from extras, and returns it.
350
358
 
351
359
  This is some Airflow magic. The google_cloud_platform hook type adds
352
360
  custom UI elements to the hook page, which allow admins to specify
@@ -386,6 +394,10 @@ class GoogleBaseHook(BaseHook):
386
394
  )
387
395
 
388
396
  @property
397
+ @deprecated(
398
+ reason="Please use `airflow.providers.google.common.consts.CLIENT_INFO`.",
399
+ category=AirflowProviderDeprecationWarning,
400
+ )
389
401
  def client_info(self) -> ClientInfo:
390
402
  """
391
403
  Return client information used to generate a user-agent for API calls.
@@ -396,11 +408,6 @@ class GoogleBaseHook(BaseHook):
396
408
  the Google Cloud. It is not supported by The Google APIs Python Client that use Discovery
397
409
  based APIs.
398
410
  """
399
- warnings.warn(
400
- "This method is deprecated, please use `airflow.providers.google.common.consts.CLIENT_INFO`.",
401
- AirflowProviderDeprecationWarning,
402
- stacklevel=2,
403
- )
404
411
  return CLIENT_INFO
405
412
 
406
413
  @property
@@ -416,7 +423,7 @@ class GoogleBaseHook(BaseHook):
416
423
 
417
424
  @staticmethod
418
425
  def quota_retry(*args, **kwargs) -> Callable:
419
- """Provides a mechanism to repeat requests in response to exceeding a temporary quota limit."""
426
+ """Provide a mechanism to repeat requests in response to exceeding a temporary quota limit."""
420
427
 
421
428
  def decorator(fun: Callable):
422
429
  default_kwargs = {
@@ -432,7 +439,7 @@ class GoogleBaseHook(BaseHook):
432
439
 
433
440
  @staticmethod
434
441
  def operation_in_progress_retry(*args, **kwargs) -> Callable[[T], T]:
435
- """Provides a mechanism to repeat requests in response to operation in progress (HTTP 409) limit."""
442
+ """Provide a mechanism to repeat requests in response to operation in progress (HTTP 409) limit."""
436
443
 
437
444
  def decorator(fun: T):
438
445
  default_kwargs = {
@@ -449,7 +456,7 @@ class GoogleBaseHook(BaseHook):
449
456
  @staticmethod
450
457
  def fallback_to_default_project_id(func: Callable[..., RT]) -> Callable[..., RT]:
451
458
  """
452
- Decorator that provides fallback for Google Cloud project id.
459
+ Provide fallback for Google Cloud project id. To be used as a decorator.
453
460
 
454
461
  If the project is None it will be replaced with the project_id from the
455
462
  service account the Hook is authenticated with. Project id can be specified
@@ -482,7 +489,7 @@ class GoogleBaseHook(BaseHook):
482
489
  @staticmethod
483
490
  def provide_gcp_credential_file(func: T) -> T:
484
491
  """
485
- Provides a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
492
+ Provide a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
486
493
 
487
494
  It is recommended to use ``provide_gcp_credential_file_as_context`` context
488
495
  manager to limit the scope when authorization data is available. Using context
@@ -499,7 +506,7 @@ class GoogleBaseHook(BaseHook):
499
506
  @contextmanager
500
507
  def provide_gcp_credential_file_as_context(self) -> Generator[str | None, None, None]:
501
508
  """
502
- Provides a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
509
+ Provide a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
503
510
 
504
511
  See:
505
512
  `Application Default Credentials (ADC)
@@ -535,7 +542,7 @@ class GoogleBaseHook(BaseHook):
535
542
  @contextmanager
536
543
  def provide_authorized_gcloud(self) -> Generator[None, None, None]:
537
544
  """
538
- Provides a separate gcloud configuration with current credentials.
545
+ Provide a separate gcloud configuration with current credentials.
539
546
 
540
547
  The gcloud tool allows you to login to Google Cloud only - ``gcloud auth login`` and
541
548
  for the needs of Application Default Credentials ``gcloud auth application-default login``.
@@ -623,13 +630,60 @@ class GoogleBaseHook(BaseHook):
623
630
  return status, message
624
631
 
625
632
 
633
+ class _CredentialsToken(Token):
634
+ """A token implementation which makes Google credentials objects accessible to [gcloud-aio](https://talkiq.github.io/gcloud-aio/) clients.
635
+
636
+ This class allows us to create token instances from credentials objects and thus supports a variety of use cases for Google
637
+ credentials in Airflow (i.e. impersonation chain). By relying on a existing credentials object we leverage functionality provided by the GoogleBaseHook
638
+ for generating credentials objects.
639
+ """
640
+
641
+ def __init__(
642
+ self,
643
+ credentials: Credentials,
644
+ *,
645
+ project: str | None = None,
646
+ session: ClientSession | None = None,
647
+ scopes: Sequence[str] | None = None,
648
+ ) -> None:
649
+ _scopes: list[str] | None = list(scopes) if scopes else None
650
+ super().__init__(session=cast(Session, session), scopes=_scopes)
651
+ self.credentials = credentials
652
+ self.project = project
653
+
654
+ @classmethod
655
+ async def from_hook(
656
+ cls,
657
+ hook: GoogleBaseHook,
658
+ *,
659
+ session: ClientSession | None = None,
660
+ ) -> _CredentialsToken:
661
+ credentials, project = hook.get_credentials_and_project_id()
662
+ return cls(
663
+ credentials=credentials,
664
+ project=project,
665
+ session=session,
666
+ scopes=hook.scopes,
667
+ )
668
+
669
+ async def get_project(self) -> str | None:
670
+ return self.project
671
+
672
+ async def acquire_access_token(self, timeout: int = 10) -> None:
673
+ await sync_to_async(self.credentials.refresh)(google.auth.transport.requests.Request())
674
+
675
+ self.access_token = cast(str, self.credentials.token)
676
+ self.access_token_duration = 3600
677
+ self.access_token_acquired_at = datetime.datetime.utcnow()
678
+ self.acquiring = None
679
+
680
+
626
681
  class GoogleBaseAsyncHook(BaseHook):
627
682
  """GoogleBaseAsyncHook inherits from BaseHook class, run on the trigger worker."""
628
683
 
629
684
  sync_hook_class: Any = None
630
685
 
631
686
  def __init__(self, **kwargs: Any):
632
- super().__init__(logger_name=kwargs.pop("logger_name", None))
633
687
  self._hook_kwargs = kwargs
634
688
  self._sync_hook = None
635
689
 
@@ -639,6 +693,16 @@ class GoogleBaseAsyncHook(BaseHook):
639
693
  self._sync_hook = await sync_to_async(self.sync_hook_class)(**self._hook_kwargs)
640
694
  return self._sync_hook
641
695
 
696
+ async def get_token(self, *, session: ClientSession | None = None) -> _CredentialsToken:
697
+ """Return a Token instance for use in [gcloud-aio](https://talkiq.github.io/gcloud-aio/) clients."""
698
+ sync_hook = await self.get_sync_hook()
699
+ return await _CredentialsToken.from_hook(sync_hook, session=session)
700
+
642
701
  async def service_file_as_context(self) -> Any:
702
+ """
703
+ Provide a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
704
+
705
+ This is the async equivalent of the non-async GoogleBaseHook's `provide_gcp_credential_file_as_context` method.
706
+ """
643
707
  sync_hook = await self.get_sync_hook()
644
708
  return await sync_to_async(sync_hook.provide_gcp_credential_file_as_context)()
@@ -66,7 +66,7 @@ class GoogleDiscoveryApiHook(GoogleBaseHook):
66
66
 
67
67
  def get_conn(self) -> Resource:
68
68
  """
69
- Creates an authenticated api client for the given api service name and credentials.
69
+ Create an authenticated api client for the given api service name and credentials.
70
70
 
71
71
  :return: the authenticated api service.
72
72
  """
@@ -84,7 +84,7 @@ class GoogleDiscoveryApiHook(GoogleBaseHook):
84
84
 
85
85
  def query(self, endpoint: str, data: dict, paginate: bool = False, num_retries: int = 0) -> dict:
86
86
  """
87
- Creates a dynamic API call to any Google API registered in Google's API Client Library and queries it.
87
+ Create a dynamic API call to any Google API registered in Google's API Client Library and queries it.
88
88
 
89
89
  :param endpoint: The client libraries path to the api call's executing method.
90
90
  For example: 'analyticsreporting.reports.batchGet'
@@ -81,7 +81,7 @@ def _load_credentials_from_file(
81
81
  filename: str, target_audience: str | None
82
82
  ) -> google_auth_credentials.Credentials | None:
83
83
  """
84
- Loads credentials from a file.
84
+ Load credentials from a file.
85
85
 
86
86
  The credentials file must be a service account key or a stored authorized user credential.
87
87
 
@@ -129,7 +129,7 @@ def _load_credentials_from_file(
129
129
  def _get_explicit_environ_credentials(
130
130
  target_audience: str | None,
131
131
  ) -> google_auth_credentials.Credentials | None:
132
- """Gets credentials from the GOOGLE_APPLICATION_CREDENTIALS environment variable."""
132
+ """Get credentials from the GOOGLE_APPLICATION_CREDENTIALS environment variable."""
133
133
  explicit_file = os.environ.get(environment_vars.CREDENTIALS)
134
134
 
135
135
  if explicit_file is None:
@@ -145,7 +145,7 @@ def _get_explicit_environ_credentials(
145
145
  def _get_gcloud_sdk_credentials(
146
146
  target_audience: str | None,
147
147
  ) -> google_auth_credentials.Credentials | None:
148
- """Gets the credentials and project ID from the Cloud SDK."""
148
+ """Get the credentials and project ID from the Cloud SDK."""
149
149
  from google.auth import _cloud_sdk # type: ignore[attr-defined]
150
150
 
151
151
  # Check if application default credentials exist.
@@ -162,7 +162,7 @@ def _get_gcloud_sdk_credentials(
162
162
  def _get_gce_credentials(
163
163
  target_audience: str | None, request: google.auth.transport.Request | None = None
164
164
  ) -> google_auth_credentials.Credentials | None:
165
- """Gets credentials and project ID from the GCE Metadata Service."""
165
+ """Get credentials and project ID from the GCE Metadata Service."""
166
166
  # Ping requires a transport, but we want application default credentials
167
167
  # to require no arguments. So, we'll use the _http_client transport which
168
168
  # uses http.client. This is only acceptable because the metadata server
@@ -191,7 +191,7 @@ def _get_gce_credentials(
191
191
  def get_default_id_token_credentials(
192
192
  target_audience: str | None, request: google.auth.transport.Request = None
193
193
  ) -> google_auth_credentials.Credentials:
194
- """Gets the default ID Token credentials for the current environment.
194
+ """Get the default ID Token credentials for the current environment.
195
195
 
196
196
  `Application Default Credentials`_ provides an easy way to obtain credentials to call Google APIs for
197
197
  server-to-server or local applications.
@@ -65,7 +65,7 @@ class CloudFirestoreHook(GoogleBaseHook):
65
65
 
66
66
  def get_conn(self):
67
67
  """
68
- Retrieves the connection to Cloud Firestore.
68
+ Retrieve the connection to Cloud Firestore.
69
69
 
70
70
  :return: Google Cloud Firestore services object.
71
71
  """
@@ -86,7 +86,7 @@ class CloudFirestoreHook(GoogleBaseHook):
86
86
  self, body: dict, database_id: str = "(default)", project_id: str | None = None
87
87
  ) -> None:
88
88
  """
89
- Starts a export with the specified configuration.
89
+ Start a export with the specified configuration.
90
90
 
91
91
  :param database_id: The Database ID.
92
92
  :param body: The request body.
@@ -110,7 +110,7 @@ class CloudFirestoreHook(GoogleBaseHook):
110
110
 
111
111
  def _wait_for_operation_to_complete(self, operation_name: str) -> None:
112
112
  """
113
- Waits for the named operation to complete - checks status of the asynchronous call.
113
+ Wait for the named operation to complete - checks status of the asynchronous call.
114
114
 
115
115
  :param operation_name: The name of the operation.
116
116
  :return: The response returned by the operation.
@@ -28,8 +28,9 @@ def get_provider_info():
28
28
  "name": "Google",
29
29
  "description": "Google services including:\n\n - `Google Ads <https://ads.google.com/>`__\n - `Google Cloud (GCP) <https://cloud.google.com/>`__\n - `Google Firebase <https://firebase.google.com/>`__\n - `Google LevelDB <https://github.com/google/leveldb/>`__\n - `Google Marketing Platform <https://marketingplatform.google.com/>`__\n - `Google Workspace <https://workspace.google.com/>`__ (formerly Google Suite)\n",
30
30
  "state": "ready",
31
- "source-date-epoch": 1705912108,
31
+ "source-date-epoch": 1707636385,
32
32
  "versions": [
33
+ "10.15.0",
33
34
  "10.14.0",
34
35
  "10.13.1",
35
36
  "10.13.0",
@@ -92,7 +93,7 @@ def get_provider_info():
92
93
  "gcsfs>=2023.10.0",
93
94
  "google-ads>=22.1.0",
94
95
  "google-analytics-admin",
95
- "google-api-core>=2.11.0",
96
+ "google-api-core>=2.11.0,!=2.16.0",
96
97
  "google-api-python-client>=1.6.0",
97
98
  "google-auth>=1.0.0",
98
99
  "google-auth-httplib2>=0.0.1",
@@ -1215,6 +1216,10 @@ def get_provider_info():
1215
1216
  "integration-name": "Google Cloud",
1216
1217
  "python-modules": ["airflow.providers.google.cloud.triggers.cloud_batch"],
1217
1218
  },
1219
+ {
1220
+ "integration-name": "Google Vertex AI",
1221
+ "python-modules": ["airflow.providers.google.cloud.triggers.vertex_ai"],
1222
+ },
1218
1223
  ],
1219
1224
  "transfers": [
1220
1225
  {
@@ -45,15 +45,15 @@ class LevelDBHook(BaseHook):
45
45
  conn_type = "leveldb"
46
46
  hook_name = "LevelDB"
47
47
 
48
- def __init__(self, leveldb_conn_id: str = default_conn_name, **kwargs):
49
- super().__init__(**kwargs)
48
+ def __init__(self, leveldb_conn_id: str = default_conn_name):
49
+ super().__init__()
50
50
  self.leveldb_conn_id = leveldb_conn_id
51
51
  self.connection = self.get_connection(leveldb_conn_id)
52
52
  self.db: plyvel.DB | None = None
53
53
 
54
54
  def get_conn(self, name: str = "/tmp/testdb/", create_if_missing: bool = False, **kwargs) -> DB:
55
55
  """
56
- Creates `Plyvel DB <https://plyvel.readthedocs.io/en/latest/api.html#DB>`__.
56
+ Create `Plyvel DB <https://plyvel.readthedocs.io/en/latest/api.html#DB>`__.
57
57
 
58
58
  :param name: path to create database e.g. `/tmp/testdb/`)
59
59
  :param create_if_missing: whether a new database should be created if needed
@@ -66,7 +66,7 @@ class LevelDBHook(BaseHook):
66
66
  return self.db
67
67
 
68
68
  def close_conn(self) -> None:
69
- """Closes connection."""
69
+ """Close connection."""
70
70
  db = self.db
71
71
  if db is not None:
72
72
  db.close()
@@ -17,9 +17,9 @@
17
17
  # under the License.
18
18
  from __future__ import annotations
19
19
 
20
- import warnings
21
20
  from typing import Any
22
21
 
22
+ from deprecated import deprecated
23
23
  from googleapiclient.discovery import Resource, build
24
24
  from googleapiclient.http import MediaFileUpload
25
25
 
@@ -27,18 +27,15 @@ from airflow.exceptions import AirflowProviderDeprecationWarning
27
27
  from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
28
28
 
29
29
 
30
+ @deprecated(
31
+ reason="The `GoogleAnalyticsHook` class is deprecated, please use `GoogleAnalyticsAdminHook` instead.",
32
+ category=AirflowProviderDeprecationWarning,
33
+ )
30
34
  class GoogleAnalyticsHook(GoogleBaseHook):
31
35
  """Hook for Google Analytics 360."""
32
36
 
33
37
  def __init__(self, api_version: str = "v3", *args, **kwargs):
34
38
  super().__init__(*args, **kwargs)
35
- warnings.warn(
36
- f"The `{type(self).__name__}` class is deprecated, please use "
37
- f"`GoogleAnalyticsAdminHook` instead.",
38
- AirflowProviderDeprecationWarning,
39
- stacklevel=2,
40
- )
41
-
42
39
  self.api_version = api_version
43
40
  self._conn = None
44
41
 
@@ -58,7 +55,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
58
55
  return result
59
56
 
60
57
  def get_conn(self) -> Resource:
61
- """Retrieves connection to Google Analytics 360."""
58
+ """Retrieve connection to Google Analytics 360."""
62
59
  if not self._conn:
63
60
  http_authorized = self._authorize()
64
61
  self._conn = build(
@@ -70,7 +67,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
70
67
  return self._conn
71
68
 
72
69
  def list_accounts(self) -> list[dict[str, Any]]:
73
- """Lists accounts list from Google Analytics 360."""
70
+ """List accounts list from Google Analytics 360."""
74
71
  self.log.info("Retrieving accounts list...")
75
72
  conn = self.get_conn()
76
73
  accounts = conn.management().accounts()
@@ -81,7 +78,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
81
78
  self, account_id: str, web_property_id: str, web_property_ad_words_link_id: str
82
79
  ) -> dict[str, Any]:
83
80
  """
84
- Returns a web property-Google Ads link to which the user has access.
81
+ Return a web property-Google Ads link to which the user has access.
85
82
 
86
83
  :param account_id: ID of the account which the given web property belongs to.
87
84
  :param web_property_id: Web property-Google Ads link UA-string.
@@ -105,7 +102,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
105
102
 
106
103
  def list_ad_words_links(self, account_id: str, web_property_id: str) -> list[dict[str, Any]]:
107
104
  """
108
- Lists webProperty-Google Ads links for a given web property.
105
+ List webProperty-Google Ads links for a given web property.
109
106
 
110
107
  :param account_id: ID of the account which the given web property belongs to.
111
108
  :param web_property_id: Web property UA-string to retrieve the Google Ads links for.
@@ -128,7 +125,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
128
125
  resumable_upload: bool = False,
129
126
  ) -> None:
130
127
  """
131
- Uploads file to GA via the Data Import API.
128
+ Upload file to GA via the Data Import API.
132
129
 
133
130
  :param file_location: The path and name of the file to upload.
134
131
  :param account_id: The GA account Id to which the data upload belongs.
@@ -165,7 +162,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
165
162
  delete_request_body: dict[str, Any],
166
163
  ) -> None:
167
164
  """
168
- Deletes the uploaded data for a given account/property/dataset.
165
+ Delete the uploaded data for a given account/property/dataset.
169
166
 
170
167
  :param account_id: The GA account Id to which the data upload belongs.
171
168
  :param web_property_id: UA-string associated with the upload.
@@ -49,7 +49,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
49
49
  self.api_version = api_version
50
50
 
51
51
  def get_conn(self) -> Resource:
52
- """Retrieves connection to Campaign Manager."""
52
+ """Retrieve connection to Campaign Manager."""
53
53
  if not self._conn:
54
54
  http_authorized = self._authorize()
55
55
  self._conn = build(
@@ -62,7 +62,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
62
62
 
63
63
  def delete_report(self, profile_id: str, report_id: str) -> Any:
64
64
  """
65
- Deletes a report by its ID.
65
+ Delete a report by its ID.
66
66
 
67
67
  :param profile_id: The DFA user profile ID.
68
68
  :param report_id: The ID of the report.
@@ -77,7 +77,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
77
77
 
78
78
  def insert_report(self, profile_id: str, report: dict[str, Any]) -> Any:
79
79
  """
80
- Creates a report.
80
+ Create a report.
81
81
 
82
82
  :param profile_id: The DFA user profile ID.
83
83
  :param report: The report resource to be inserted.
@@ -99,7 +99,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
99
99
  sort_order: str | None = None,
100
100
  ) -> list[dict]:
101
101
  """
102
- Retrieves list of reports.
102
+ Retrieve list of reports.
103
103
 
104
104
  :param profile_id: The DFA user profile ID.
105
105
  :param max_results: Maximum number of results to return.
@@ -125,7 +125,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
125
125
 
126
126
  def patch_report(self, profile_id: str, report_id: str, update_mask: dict) -> Any:
127
127
  """
128
- Updates a report. This method supports patch semantics.
128
+ Update a report. This method supports patch semantics.
129
129
 
130
130
  :param profile_id: The DFA user profile ID.
131
131
  :param report_id: The ID of the report.
@@ -142,7 +142,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
142
142
 
143
143
  def run_report(self, profile_id: str, report_id: str, synchronous: bool | None = None) -> Any:
144
144
  """
145
- Runs a report.
145
+ Run a report.
146
146
 
147
147
  :param profile_id: The DFA profile ID.
148
148
  :param report_id: The ID of the report.
@@ -158,7 +158,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
158
158
 
159
159
  def update_report(self, profile_id: str, report_id: str) -> Any:
160
160
  """
161
- Updates a report.
161
+ Update a report.
162
162
 
163
163
  :param profile_id: The DFA user profile ID.
164
164
  :param report_id: The ID of the report.
@@ -173,7 +173,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
173
173
 
174
174
  def get_report(self, file_id: str, profile_id: str, report_id: str) -> Any:
175
175
  """
176
- Retrieves a report file.
176
+ Retrieve a report file.
177
177
 
178
178
  :param profile_id: The DFA user profile ID.
179
179
  :param report_id: The ID of the report.
@@ -190,7 +190,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
190
190
 
191
191
  def get_report_file(self, file_id: str, profile_id: str, report_id: str) -> http.HttpRequest:
192
192
  """
193
- Retrieves a media part of report file.
193
+ Retrieve a media part of report file.
194
194
 
195
195
  :param profile_id: The DFA user profile ID.
196
196
  :param report_id: The ID of the report.
@@ -234,7 +234,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
234
234
  max_failed_inserts: int = 0,
235
235
  ) -> Any:
236
236
  """
237
- Inserts conversions.
237
+ Insert conversions.
238
238
 
239
239
  :param profile_id: User profile ID associated with this request.
240
240
  :param conversions: Conversations to insert, should by type of Conversation:
@@ -278,7 +278,7 @@ class GoogleCampaignManagerHook(GoogleBaseHook):
278
278
  max_failed_updates: int = 0,
279
279
  ) -> Any:
280
280
  """
281
- Updates existing conversions.
281
+ Update existing conversions.
282
282
 
283
283
  :param profile_id: User profile ID associated with this request.
284
284
  :param conversions: Conversations to update, should by type of Conversation: