apache-airflow-providers-google 10.14.0rc2__py3-none-any.whl → 10.15.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. airflow/providers/google/__init__.py +1 -1
  2. airflow/providers/google/cloud/hooks/automl.py +13 -13
  3. airflow/providers/google/cloud/hooks/bigquery.py +193 -246
  4. airflow/providers/google/cloud/hooks/bigquery_dts.py +6 -6
  5. airflow/providers/google/cloud/hooks/bigtable.py +8 -8
  6. airflow/providers/google/cloud/hooks/cloud_batch.py +1 -1
  7. airflow/providers/google/cloud/hooks/cloud_build.py +19 -20
  8. airflow/providers/google/cloud/hooks/cloud_composer.py +4 -4
  9. airflow/providers/google/cloud/hooks/cloud_memorystore.py +10 -10
  10. airflow/providers/google/cloud/hooks/cloud_run.py +1 -1
  11. airflow/providers/google/cloud/hooks/cloud_sql.py +17 -17
  12. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +3 -3
  13. airflow/providers/google/cloud/hooks/compute.py +16 -16
  14. airflow/providers/google/cloud/hooks/compute_ssh.py +1 -1
  15. airflow/providers/google/cloud/hooks/datacatalog.py +22 -22
  16. airflow/providers/google/cloud/hooks/dataflow.py +48 -49
  17. airflow/providers/google/cloud/hooks/dataform.py +16 -16
  18. airflow/providers/google/cloud/hooks/datafusion.py +15 -15
  19. airflow/providers/google/cloud/hooks/datapipeline.py +3 -3
  20. airflow/providers/google/cloud/hooks/dataplex.py +19 -19
  21. airflow/providers/google/cloud/hooks/dataprep.py +8 -8
  22. airflow/providers/google/cloud/hooks/dataproc.py +88 -0
  23. airflow/providers/google/cloud/hooks/dataproc_metastore.py +13 -13
  24. airflow/providers/google/cloud/hooks/datastore.py +3 -3
  25. airflow/providers/google/cloud/hooks/dlp.py +25 -25
  26. airflow/providers/google/cloud/hooks/gcs.py +25 -23
  27. airflow/providers/google/cloud/hooks/gdm.py +3 -3
  28. airflow/providers/google/cloud/hooks/kms.py +3 -3
  29. airflow/providers/google/cloud/hooks/kubernetes_engine.py +63 -48
  30. airflow/providers/google/cloud/hooks/life_sciences.py +13 -12
  31. airflow/providers/google/cloud/hooks/looker.py +7 -7
  32. airflow/providers/google/cloud/hooks/mlengine.py +12 -12
  33. airflow/providers/google/cloud/hooks/natural_language.py +2 -2
  34. airflow/providers/google/cloud/hooks/os_login.py +1 -1
  35. airflow/providers/google/cloud/hooks/pubsub.py +9 -9
  36. airflow/providers/google/cloud/hooks/secret_manager.py +1 -1
  37. airflow/providers/google/cloud/hooks/spanner.py +11 -11
  38. airflow/providers/google/cloud/hooks/speech_to_text.py +1 -1
  39. airflow/providers/google/cloud/hooks/stackdriver.py +7 -7
  40. airflow/providers/google/cloud/hooks/tasks.py +11 -11
  41. airflow/providers/google/cloud/hooks/text_to_speech.py +1 -1
  42. airflow/providers/google/cloud/hooks/translate.py +1 -1
  43. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +13 -13
  44. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -6
  45. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +45 -50
  46. airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +13 -13
  47. airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +9 -9
  48. airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +128 -11
  49. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +10 -10
  50. airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +8 -8
  51. airflow/providers/google/cloud/hooks/video_intelligence.py +2 -2
  52. airflow/providers/google/cloud/hooks/vision.py +1 -1
  53. airflow/providers/google/cloud/hooks/workflows.py +10 -10
  54. airflow/providers/google/cloud/links/datafusion.py +12 -5
  55. airflow/providers/google/cloud/operators/bigquery.py +9 -11
  56. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +3 -1
  57. airflow/providers/google/cloud/operators/dataflow.py +16 -16
  58. airflow/providers/google/cloud/operators/datafusion.py +9 -1
  59. airflow/providers/google/cloud/operators/dataproc.py +298 -65
  60. airflow/providers/google/cloud/operators/kubernetes_engine.py +6 -6
  61. airflow/providers/google/cloud/operators/life_sciences.py +10 -9
  62. airflow/providers/google/cloud/operators/mlengine.py +96 -96
  63. airflow/providers/google/cloud/operators/pubsub.py +2 -0
  64. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +33 -3
  65. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +59 -2
  66. airflow/providers/google/cloud/secrets/secret_manager.py +8 -7
  67. airflow/providers/google/cloud/sensors/bigquery.py +20 -16
  68. airflow/providers/google/cloud/sensors/cloud_composer.py +11 -8
  69. airflow/providers/google/cloud/sensors/gcs.py +8 -7
  70. airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +4 -4
  71. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
  72. airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
  73. airflow/providers/google/cloud/transfers/mysql_to_gcs.py +1 -1
  74. airflow/providers/google/cloud/transfers/oracle_to_gcs.py +1 -1
  75. airflow/providers/google/cloud/transfers/postgres_to_gcs.py +1 -1
  76. airflow/providers/google/cloud/transfers/presto_to_gcs.py +1 -1
  77. airflow/providers/google/cloud/transfers/s3_to_gcs.py +3 -3
  78. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
  79. airflow/providers/google/cloud/transfers/sql_to_gcs.py +3 -3
  80. airflow/providers/google/cloud/transfers/trino_to_gcs.py +1 -1
  81. airflow/providers/google/cloud/triggers/bigquery.py +12 -12
  82. airflow/providers/google/cloud/triggers/bigquery_dts.py +1 -1
  83. airflow/providers/google/cloud/triggers/cloud_batch.py +3 -1
  84. airflow/providers/google/cloud/triggers/cloud_build.py +2 -2
  85. airflow/providers/google/cloud/triggers/cloud_run.py +1 -1
  86. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +6 -6
  87. airflow/providers/google/cloud/triggers/dataflow.py +3 -1
  88. airflow/providers/google/cloud/triggers/datafusion.py +2 -2
  89. airflow/providers/google/cloud/triggers/dataplex.py +2 -2
  90. airflow/providers/google/cloud/triggers/dataproc.py +2 -2
  91. airflow/providers/google/cloud/triggers/gcs.py +12 -8
  92. airflow/providers/google/cloud/triggers/kubernetes_engine.py +2 -2
  93. airflow/providers/google/cloud/triggers/mlengine.py +2 -2
  94. airflow/providers/google/cloud/triggers/pubsub.py +1 -1
  95. airflow/providers/google/cloud/triggers/vertex_ai.py +99 -0
  96. airflow/providers/google/cloud/utils/bigquery.py +2 -2
  97. airflow/providers/google/cloud/utils/credentials_provider.py +2 -2
  98. airflow/providers/google/cloud/utils/dataform.py +1 -1
  99. airflow/providers/google/cloud/utils/field_validator.py +2 -2
  100. airflow/providers/google/cloud/utils/helpers.py +2 -2
  101. airflow/providers/google/cloud/utils/mlengine_operator_utils.py +1 -1
  102. airflow/providers/google/cloud/utils/mlengine_prediction_summary.py +1 -1
  103. airflow/providers/google/common/auth_backend/google_openid.py +2 -2
  104. airflow/providers/google/common/hooks/base_google.py +29 -22
  105. airflow/providers/google/common/hooks/discovery_api.py +2 -2
  106. airflow/providers/google/common/utils/id_token_credentials.py +5 -5
  107. airflow/providers/google/firebase/hooks/firestore.py +3 -3
  108. airflow/providers/google/get_provider_info.py +7 -2
  109. airflow/providers/google/leveldb/hooks/leveldb.py +2 -2
  110. airflow/providers/google/marketing_platform/hooks/analytics.py +11 -14
  111. airflow/providers/google/marketing_platform/hooks/campaign_manager.py +11 -11
  112. airflow/providers/google/marketing_platform/hooks/display_video.py +13 -13
  113. airflow/providers/google/marketing_platform/hooks/search_ads.py +4 -4
  114. airflow/providers/google/marketing_platform/operators/analytics.py +37 -32
  115. airflow/providers/google/suite/hooks/calendar.py +2 -2
  116. airflow/providers/google/suite/hooks/drive.py +7 -7
  117. airflow/providers/google/suite/hooks/sheets.py +8 -8
  118. {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/METADATA +11 -11
  119. {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/RECORD +121 -120
  120. {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/WHEEL +0 -0
  121. {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/entry_points.txt +0 -0
@@ -79,7 +79,7 @@ class PubsubPullTrigger(BaseTrigger):
79
79
  self.hook = PubSubAsyncHook()
80
80
 
81
81
  def serialize(self) -> tuple[str, dict[str, Any]]:
82
- """Serializes PubsubPullTrigger arguments and classpath."""
82
+ """Serialize PubsubPullTrigger arguments and classpath."""
83
83
  return (
84
84
  "airflow.providers.google.cloud.triggers.pubsub.PubsubPullTrigger",
85
85
  {
@@ -0,0 +1,99 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+ from __future__ import annotations
18
+
19
+ from typing import Any, AsyncIterator, Sequence
20
+
21
+ from google.cloud.aiplatform_v1 import HyperparameterTuningJob, JobState
22
+
23
+ from airflow.exceptions import AirflowException
24
+ from airflow.providers.google.cloud.hooks.vertex_ai.hyperparameter_tuning_job import (
25
+ HyperparameterTuningJobAsyncHook,
26
+ )
27
+ from airflow.triggers.base import BaseTrigger, TriggerEvent
28
+
29
+
30
+ class CreateHyperparameterTuningJobTrigger(BaseTrigger):
31
+ """CreateHyperparameterTuningJobTrigger run on the trigger worker to perform create operation."""
32
+
33
+ statuses_success = {
34
+ JobState.JOB_STATE_PAUSED,
35
+ JobState.JOB_STATE_SUCCEEDED,
36
+ }
37
+
38
+ def __init__(
39
+ self,
40
+ conn_id: str,
41
+ project_id: str,
42
+ location: str,
43
+ job_id: str,
44
+ poll_interval: int,
45
+ impersonation_chain: str | Sequence[str] | None = None,
46
+ ):
47
+ super().__init__()
48
+ self.conn_id = conn_id
49
+ self.project_id = project_id
50
+ self.location = location
51
+ self.job_id = job_id
52
+ self.poll_interval = poll_interval
53
+ self.impersonation_chain = impersonation_chain
54
+
55
+ def serialize(self) -> tuple[str, dict[str, Any]]:
56
+ return (
57
+ "airflow.providers.google.cloud.triggers.vertex_ai.CreateHyperparameterTuningJobTrigger",
58
+ {
59
+ "conn_id": self.conn_id,
60
+ "project_id": self.project_id,
61
+ "location": self.location,
62
+ "job_id": self.job_id,
63
+ "poll_interval": self.poll_interval,
64
+ "impersonation_chain": self.impersonation_chain,
65
+ },
66
+ )
67
+
68
+ async def run(self) -> AsyncIterator[TriggerEvent]:
69
+ hook = self._get_async_hook()
70
+ try:
71
+ job = await hook.wait_hyperparameter_tuning_job(
72
+ project_id=self.project_id,
73
+ location=self.location,
74
+ job_id=self.job_id,
75
+ poll_interval=self.poll_interval,
76
+ )
77
+ except AirflowException as ex:
78
+ yield TriggerEvent(
79
+ {
80
+ "status": "error",
81
+ "message": str(ex),
82
+ }
83
+ )
84
+ return
85
+
86
+ status = "success" if job.state in self.statuses_success else "error"
87
+ message = f"Hyperparameter tuning job {job.name} completed with status {job.state.name}"
88
+ yield TriggerEvent(
89
+ {
90
+ "status": status,
91
+ "message": message,
92
+ "job": HyperparameterTuningJob.to_dict(job),
93
+ }
94
+ )
95
+
96
+ def _get_async_hook(self) -> HyperparameterTuningJobAsyncHook:
97
+ return HyperparameterTuningJobAsyncHook(
98
+ gcp_conn_id=self.conn_id, impersonation_chain=self.impersonation_chain
99
+ )
@@ -21,7 +21,7 @@ from typing import Any
21
21
 
22
22
  def bq_cast(string_field: str, bq_type: str) -> None | int | float | bool | str:
23
23
  """
24
- Helper method that casts a BigQuery row to the appropriate data types.
24
+ Cast a BigQuery row to the appropriate data types.
25
25
 
26
26
  This is useful because BigQuery returns all fields as strings.
27
27
  """
@@ -41,7 +41,7 @@ def bq_cast(string_field: str, bq_type: str) -> None | int | float | bool | str:
41
41
 
42
42
  def convert_job_id(job_id: str | list[str], project_id: str, location: str | None) -> Any:
43
43
  """
44
- Helper method that converts to path: project_id:location:job_id.
44
+ Convert job_id to path: project_id:location:job_id.
45
45
 
46
46
  :param project_id: Required. The ID of the Google Cloud project where workspace located.
47
47
  :param location: Optional. The ID of the Google Cloud region where workspace located.
@@ -358,7 +358,7 @@ class _CredentialProvider(LoggingMixin):
358
358
 
359
359
 
360
360
  def get_credentials_and_project_id(*args, **kwargs) -> tuple[google.auth.credentials.Credentials, str]:
361
- """Returns the Credentials object for Google API and the associated project_id."""
361
+ """Return the Credentials object for Google API and the associated project_id."""
362
362
  return _CredentialProvider(*args, **kwargs).get_credentials_and_project()
363
363
 
364
364
 
@@ -398,7 +398,7 @@ def _get_target_principal_and_delegates(
398
398
 
399
399
  def _get_project_id_from_service_account_email(service_account_email: str) -> str:
400
400
  """
401
- Extracts project_id from service account's email address.
401
+ Extract project_id from service account's email address.
402
402
 
403
403
  :param service_account_email: email of the service account.
404
404
 
@@ -45,7 +45,7 @@ def make_initialization_workspace_flow(
45
45
  without_installation: bool = False,
46
46
  ) -> tuple:
47
47
  """
48
- Creates flow which simulates the initialization of the default project.
48
+ Create flow which simulates the initialization of the default project.
49
49
 
50
50
  :param project_id: Required. The ID of the Google Cloud project where workspace located.
51
51
  :param region: Required. The ID of the Google Cloud region where workspace located.
@@ -309,7 +309,7 @@ class GcpBodyFieldValidator(LoggingMixin):
309
309
 
310
310
  def _validate_field(self, validation_spec, dictionary_to_validate, parent=None, force_optional=False):
311
311
  """
312
- Validates if field is OK.
312
+ Validate if field is OK.
313
313
 
314
314
  :param validation_spec: specification of the field
315
315
  :param dictionary_to_validate: dictionary where the field should be present
@@ -413,7 +413,7 @@ class GcpBodyFieldValidator(LoggingMixin):
413
413
 
414
414
  def validate(self, body_to_validate: dict) -> None:
415
415
  """
416
- Validates if the body (dictionary) follows specification that the validator was instantiated with.
416
+ Validate if the body (dictionary) follows specification that the validator was instantiated with.
417
417
 
418
418
  Raises ValidationSpecificationException or ValidationFieldException in case of problems
419
419
  with specification or the body not conforming to the specification respectively.
@@ -19,12 +19,12 @@ from __future__ import annotations
19
19
 
20
20
 
21
21
  def normalize_directory_path(source_object: str | None) -> str | None:
22
- """Makes sure dir path ends with a slash."""
22
+ """Make sure dir path ends with a slash."""
23
23
  return source_object + "/" if source_object and not source_object.endswith("/") else source_object
24
24
 
25
25
 
26
26
  def resource_path_to_dict(resource_name: str) -> dict[str, str]:
27
- """Converts a path-like GCP resource name into a dictionary.
27
+ """Convert a path-like GCP resource name into a dictionary.
28
28
 
29
29
  For example, the path `projects/my-project/locations/my-location/instances/my-instance` will be converted
30
30
  to a dict:
@@ -57,7 +57,7 @@ def create_evaluate_ops(
57
57
  py_interpreter="python3",
58
58
  ) -> tuple[MLEngineStartBatchPredictionJobOperator, BeamRunPythonPipelineOperator, PythonOperator]:
59
59
  r"""
60
- Creates Operators needed for model evaluation and returns.
60
+ Create Operators needed for model evaluation and returns.
61
61
 
62
62
  This function is deprecated. All the functionality of legacy MLEngine and new features are available
63
63
  on the Vertex AI platform.
@@ -151,7 +151,7 @@ def MakeSummary(pcoll, metric_fn, metric_keys):
151
151
 
152
152
 
153
153
  def run(argv=None):
154
- """Helper for obtaining prediction summary."""
154
+ """Obtain prediction summary."""
155
155
  parser = argparse.ArgumentParser()
156
156
  parser.add_argument(
157
157
  "--prediction_path",
@@ -52,7 +52,7 @@ def create_client_session():
52
52
 
53
53
 
54
54
  def init_app(_):
55
- """Initializes authentication."""
55
+ """Initialize authentication."""
56
56
 
57
57
 
58
58
  def _get_id_token_from_request(request) -> str | None:
@@ -110,7 +110,7 @@ T = TypeVar("T", bound=Callable)
110
110
 
111
111
 
112
112
  def requires_authentication(function: T):
113
- """Decorator for functions that require authentication."""
113
+ """Decorator for function that require authentication."""
114
114
 
115
115
  @wraps(function)
116
116
  def decorated(*args, **kwargs):
@@ -24,7 +24,6 @@ import json
24
24
  import logging
25
25
  import os
26
26
  import tempfile
27
- import warnings
28
27
  from contextlib import ExitStack, contextmanager
29
28
  from subprocess import check_output
30
29
  from typing import TYPE_CHECKING, Any, Callable, Generator, Sequence, TypeVar, cast
@@ -36,6 +35,7 @@ import google_auth_httplib2
36
35
  import requests
37
36
  import tenacity
38
37
  from asgiref.sync import sync_to_async
38
+ from deprecated import deprecated
39
39
  from gcloud.aio.auth.token import Token
40
40
  from google.api_core.exceptions import Forbidden, ResourceExhausted, TooManyRequests
41
41
  from google.auth import _cloud_sdk, compute_engine # type: ignore[attr-defined]
@@ -79,6 +79,8 @@ INVALID_REASONS = [
79
79
 
80
80
  def is_soft_quota_exception(exception: Exception):
81
81
  """
82
+ Check for quota violation errors.
83
+
82
84
  API for Google services does not have a standardized way to report quota violation errors.
83
85
 
84
86
  The function has been adapted by trial and error to the following services:
@@ -100,6 +102,8 @@ def is_soft_quota_exception(exception: Exception):
100
102
 
101
103
  def is_operation_in_progress_exception(exception: Exception) -> bool:
102
104
  """
105
+ Handle operation in-progress exceptions.
106
+
103
107
  Some calls return 429 (too many requests!) or 409 errors (Conflict) in case of operation in progress.
104
108
 
105
109
  * Google Cloud SQL
@@ -195,7 +199,7 @@ class GoogleBaseHook(BaseHook):
195
199
 
196
200
  @classmethod
197
201
  def get_connection_form_widgets(cls) -> dict[str, Any]:
198
- """Returns connection widgets to add to connection form."""
202
+ """Return connection widgets to add to connection form."""
199
203
  from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget
200
204
  from flask_babel import lazy_gettext
201
205
  from wtforms import IntegerField, PasswordField, StringField
@@ -228,7 +232,7 @@ class GoogleBaseHook(BaseHook):
228
232
 
229
233
  @classmethod
230
234
  def get_ui_field_behaviour(cls) -> dict[str, Any]:
231
- """Returns custom field behaviour."""
235
+ """Return custom field behaviour."""
232
236
  return {
233
237
  "hidden_fields": ["host", "schema", "login", "password", "port", "extra"],
234
238
  "relabeling": {},
@@ -249,7 +253,7 @@ class GoogleBaseHook(BaseHook):
249
253
  self._cached_project_id: str | None = None
250
254
 
251
255
  def get_credentials_and_project_id(self) -> tuple[google.auth.credentials.Credentials, str | None]:
252
- """Returns the Credentials object for Google API and the associated project_id."""
256
+ """Return the Credentials object for Google API and the associated project_id."""
253
257
  if self._cached_credentials is not None:
254
258
  return self._cached_credentials, self._cached_project_id
255
259
 
@@ -299,12 +303,12 @@ class GoogleBaseHook(BaseHook):
299
303
  return credentials, project_id
300
304
 
301
305
  def get_credentials(self) -> google.auth.credentials.Credentials:
302
- """Returns the Credentials object for Google API."""
306
+ """Return the Credentials object for Google API."""
303
307
  credentials, _ = self.get_credentials_and_project_id()
304
308
  return credentials
305
309
 
306
310
  def _get_access_token(self) -> str:
307
- """Returns a valid access token from Google API Credentials."""
311
+ """Return a valid access token from Google API Credentials."""
308
312
  credentials = self.get_credentials()
309
313
  auth_req = google.auth.transport.requests.Request()
310
314
  # credentials.token is None
@@ -315,7 +319,7 @@ class GoogleBaseHook(BaseHook):
315
319
  @functools.lru_cache(maxsize=None)
316
320
  def _get_credentials_email(self) -> str:
317
321
  """
318
- Returns the email address associated with the currently logged in account.
322
+ Return the email address associated with the currently logged in account.
319
323
 
320
324
  If a service account is used, it returns the service account.
321
325
  If user authentication (e.g. gcloud auth) is used, it returns the e-mail account of that user.
@@ -341,7 +345,7 @@ class GoogleBaseHook(BaseHook):
341
345
  return oauth2_client.tokeninfo().execute()["email"]
342
346
 
343
347
  def _authorize(self) -> google_auth_httplib2.AuthorizedHttp:
344
- """Returns an authorized HTTP object to be used to build a Google cloud service hook connection."""
348
+ """Return an authorized HTTP object to be used to build a Google cloud service hook connection."""
345
349
  credentials = self.get_credentials()
346
350
  http = build_http()
347
351
  http = set_user_agent(http, "airflow/" + version.version)
@@ -350,7 +354,7 @@ class GoogleBaseHook(BaseHook):
350
354
 
351
355
  def _get_field(self, f: str, default: Any = None) -> Any:
352
356
  """
353
- Fetches a field from extras, and returns it.
357
+ Fetch a field from extras, and returns it.
354
358
 
355
359
  This is some Airflow magic. The google_cloud_platform hook type adds
356
360
  custom UI elements to the hook page, which allow admins to specify
@@ -390,6 +394,10 @@ class GoogleBaseHook(BaseHook):
390
394
  )
391
395
 
392
396
  @property
397
+ @deprecated(
398
+ reason="Please use `airflow.providers.google.common.consts.CLIENT_INFO`.",
399
+ category=AirflowProviderDeprecationWarning,
400
+ )
393
401
  def client_info(self) -> ClientInfo:
394
402
  """
395
403
  Return client information used to generate a user-agent for API calls.
@@ -400,11 +408,6 @@ class GoogleBaseHook(BaseHook):
400
408
  the Google Cloud. It is not supported by The Google APIs Python Client that use Discovery
401
409
  based APIs.
402
410
  """
403
- warnings.warn(
404
- "This method is deprecated, please use `airflow.providers.google.common.consts.CLIENT_INFO`.",
405
- AirflowProviderDeprecationWarning,
406
- stacklevel=2,
407
- )
408
411
  return CLIENT_INFO
409
412
 
410
413
  @property
@@ -420,7 +423,7 @@ class GoogleBaseHook(BaseHook):
420
423
 
421
424
  @staticmethod
422
425
  def quota_retry(*args, **kwargs) -> Callable:
423
- """Provides a mechanism to repeat requests in response to exceeding a temporary quota limit."""
426
+ """Provide a mechanism to repeat requests in response to exceeding a temporary quota limit."""
424
427
 
425
428
  def decorator(fun: Callable):
426
429
  default_kwargs = {
@@ -436,7 +439,7 @@ class GoogleBaseHook(BaseHook):
436
439
 
437
440
  @staticmethod
438
441
  def operation_in_progress_retry(*args, **kwargs) -> Callable[[T], T]:
439
- """Provides a mechanism to repeat requests in response to operation in progress (HTTP 409) limit."""
442
+ """Provide a mechanism to repeat requests in response to operation in progress (HTTP 409) limit."""
440
443
 
441
444
  def decorator(fun: T):
442
445
  default_kwargs = {
@@ -453,7 +456,7 @@ class GoogleBaseHook(BaseHook):
453
456
  @staticmethod
454
457
  def fallback_to_default_project_id(func: Callable[..., RT]) -> Callable[..., RT]:
455
458
  """
456
- Decorator that provides fallback for Google Cloud project id.
459
+ Provide fallback for Google Cloud project id. To be used as a decorator.
457
460
 
458
461
  If the project is None it will be replaced with the project_id from the
459
462
  service account the Hook is authenticated with. Project id can be specified
@@ -486,7 +489,7 @@ class GoogleBaseHook(BaseHook):
486
489
  @staticmethod
487
490
  def provide_gcp_credential_file(func: T) -> T:
488
491
  """
489
- Provides a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
492
+ Provide a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
490
493
 
491
494
  It is recommended to use ``provide_gcp_credential_file_as_context`` context
492
495
  manager to limit the scope when authorization data is available. Using context
@@ -503,7 +506,7 @@ class GoogleBaseHook(BaseHook):
503
506
  @contextmanager
504
507
  def provide_gcp_credential_file_as_context(self) -> Generator[str | None, None, None]:
505
508
  """
506
- Provides a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
509
+ Provide a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
507
510
 
508
511
  See:
509
512
  `Application Default Credentials (ADC)
@@ -539,7 +542,7 @@ class GoogleBaseHook(BaseHook):
539
542
  @contextmanager
540
543
  def provide_authorized_gcloud(self) -> Generator[None, None, None]:
541
544
  """
542
- Provides a separate gcloud configuration with current credentials.
545
+ Provide a separate gcloud configuration with current credentials.
543
546
 
544
547
  The gcloud tool allows you to login to Google Cloud only - ``gcloud auth login`` and
545
548
  for the needs of Application Default Credentials ``gcloud auth application-default login``.
@@ -691,11 +694,15 @@ class GoogleBaseAsyncHook(BaseHook):
691
694
  return self._sync_hook
692
695
 
693
696
  async def get_token(self, *, session: ClientSession | None = None) -> _CredentialsToken:
694
- """Returns a Token instance for use in [gcloud-aio](https://talkiq.github.io/gcloud-aio/) clients."""
697
+ """Return a Token instance for use in [gcloud-aio](https://talkiq.github.io/gcloud-aio/) clients."""
695
698
  sync_hook = await self.get_sync_hook()
696
699
  return await _CredentialsToken.from_hook(sync_hook, session=session)
697
700
 
698
701
  async def service_file_as_context(self) -> Any:
699
- """This is the async equivalent of the non-async GoogleBaseHook's `provide_gcp_credential_file_as_context` method."""
702
+ """
703
+ Provide a Google Cloud credentials for Application Default Credentials (ADC) strategy support.
704
+
705
+ This is the async equivalent of the non-async GoogleBaseHook's `provide_gcp_credential_file_as_context` method.
706
+ """
700
707
  sync_hook = await self.get_sync_hook()
701
708
  return await sync_to_async(sync_hook.provide_gcp_credential_file_as_context)()
@@ -66,7 +66,7 @@ class GoogleDiscoveryApiHook(GoogleBaseHook):
66
66
 
67
67
  def get_conn(self) -> Resource:
68
68
  """
69
- Creates an authenticated api client for the given api service name and credentials.
69
+ Create an authenticated api client for the given api service name and credentials.
70
70
 
71
71
  :return: the authenticated api service.
72
72
  """
@@ -84,7 +84,7 @@ class GoogleDiscoveryApiHook(GoogleBaseHook):
84
84
 
85
85
  def query(self, endpoint: str, data: dict, paginate: bool = False, num_retries: int = 0) -> dict:
86
86
  """
87
- Creates a dynamic API call to any Google API registered in Google's API Client Library and queries it.
87
+ Create a dynamic API call to any Google API registered in Google's API Client Library and queries it.
88
88
 
89
89
  :param endpoint: The client libraries path to the api call's executing method.
90
90
  For example: 'analyticsreporting.reports.batchGet'
@@ -81,7 +81,7 @@ def _load_credentials_from_file(
81
81
  filename: str, target_audience: str | None
82
82
  ) -> google_auth_credentials.Credentials | None:
83
83
  """
84
- Loads credentials from a file.
84
+ Load credentials from a file.
85
85
 
86
86
  The credentials file must be a service account key or a stored authorized user credential.
87
87
 
@@ -129,7 +129,7 @@ def _load_credentials_from_file(
129
129
  def _get_explicit_environ_credentials(
130
130
  target_audience: str | None,
131
131
  ) -> google_auth_credentials.Credentials | None:
132
- """Gets credentials from the GOOGLE_APPLICATION_CREDENTIALS environment variable."""
132
+ """Get credentials from the GOOGLE_APPLICATION_CREDENTIALS environment variable."""
133
133
  explicit_file = os.environ.get(environment_vars.CREDENTIALS)
134
134
 
135
135
  if explicit_file is None:
@@ -145,7 +145,7 @@ def _get_explicit_environ_credentials(
145
145
  def _get_gcloud_sdk_credentials(
146
146
  target_audience: str | None,
147
147
  ) -> google_auth_credentials.Credentials | None:
148
- """Gets the credentials and project ID from the Cloud SDK."""
148
+ """Get the credentials and project ID from the Cloud SDK."""
149
149
  from google.auth import _cloud_sdk # type: ignore[attr-defined]
150
150
 
151
151
  # Check if application default credentials exist.
@@ -162,7 +162,7 @@ def _get_gcloud_sdk_credentials(
162
162
  def _get_gce_credentials(
163
163
  target_audience: str | None, request: google.auth.transport.Request | None = None
164
164
  ) -> google_auth_credentials.Credentials | None:
165
- """Gets credentials and project ID from the GCE Metadata Service."""
165
+ """Get credentials and project ID from the GCE Metadata Service."""
166
166
  # Ping requires a transport, but we want application default credentials
167
167
  # to require no arguments. So, we'll use the _http_client transport which
168
168
  # uses http.client. This is only acceptable because the metadata server
@@ -191,7 +191,7 @@ def _get_gce_credentials(
191
191
  def get_default_id_token_credentials(
192
192
  target_audience: str | None, request: google.auth.transport.Request = None
193
193
  ) -> google_auth_credentials.Credentials:
194
- """Gets the default ID Token credentials for the current environment.
194
+ """Get the default ID Token credentials for the current environment.
195
195
 
196
196
  `Application Default Credentials`_ provides an easy way to obtain credentials to call Google APIs for
197
197
  server-to-server or local applications.
@@ -65,7 +65,7 @@ class CloudFirestoreHook(GoogleBaseHook):
65
65
 
66
66
  def get_conn(self):
67
67
  """
68
- Retrieves the connection to Cloud Firestore.
68
+ Retrieve the connection to Cloud Firestore.
69
69
 
70
70
  :return: Google Cloud Firestore services object.
71
71
  """
@@ -86,7 +86,7 @@ class CloudFirestoreHook(GoogleBaseHook):
86
86
  self, body: dict, database_id: str = "(default)", project_id: str | None = None
87
87
  ) -> None:
88
88
  """
89
- Starts a export with the specified configuration.
89
+ Start a export with the specified configuration.
90
90
 
91
91
  :param database_id: The Database ID.
92
92
  :param body: The request body.
@@ -110,7 +110,7 @@ class CloudFirestoreHook(GoogleBaseHook):
110
110
 
111
111
  def _wait_for_operation_to_complete(self, operation_name: str) -> None:
112
112
  """
113
- Waits for the named operation to complete - checks status of the asynchronous call.
113
+ Wait for the named operation to complete - checks status of the asynchronous call.
114
114
 
115
115
  :param operation_name: The name of the operation.
116
116
  :return: The response returned by the operation.
@@ -28,8 +28,9 @@ def get_provider_info():
28
28
  "name": "Google",
29
29
  "description": "Google services including:\n\n - `Google Ads <https://ads.google.com/>`__\n - `Google Cloud (GCP) <https://cloud.google.com/>`__\n - `Google Firebase <https://firebase.google.com/>`__\n - `Google LevelDB <https://github.com/google/leveldb/>`__\n - `Google Marketing Platform <https://marketingplatform.google.com/>`__\n - `Google Workspace <https://workspace.google.com/>`__ (formerly Google Suite)\n",
30
30
  "state": "ready",
31
- "source-date-epoch": 1705912108,
31
+ "source-date-epoch": 1707636385,
32
32
  "versions": [
33
+ "10.15.0",
33
34
  "10.14.0",
34
35
  "10.13.1",
35
36
  "10.13.0",
@@ -92,7 +93,7 @@ def get_provider_info():
92
93
  "gcsfs>=2023.10.0",
93
94
  "google-ads>=22.1.0",
94
95
  "google-analytics-admin",
95
- "google-api-core>=2.11.0",
96
+ "google-api-core>=2.11.0,!=2.16.0",
96
97
  "google-api-python-client>=1.6.0",
97
98
  "google-auth>=1.0.0",
98
99
  "google-auth-httplib2>=0.0.1",
@@ -1215,6 +1216,10 @@ def get_provider_info():
1215
1216
  "integration-name": "Google Cloud",
1216
1217
  "python-modules": ["airflow.providers.google.cloud.triggers.cloud_batch"],
1217
1218
  },
1219
+ {
1220
+ "integration-name": "Google Vertex AI",
1221
+ "python-modules": ["airflow.providers.google.cloud.triggers.vertex_ai"],
1222
+ },
1218
1223
  ],
1219
1224
  "transfers": [
1220
1225
  {
@@ -53,7 +53,7 @@ class LevelDBHook(BaseHook):
53
53
 
54
54
  def get_conn(self, name: str = "/tmp/testdb/", create_if_missing: bool = False, **kwargs) -> DB:
55
55
  """
56
- Creates `Plyvel DB <https://plyvel.readthedocs.io/en/latest/api.html#DB>`__.
56
+ Create `Plyvel DB <https://plyvel.readthedocs.io/en/latest/api.html#DB>`__.
57
57
 
58
58
  :param name: path to create database e.g. `/tmp/testdb/`)
59
59
  :param create_if_missing: whether a new database should be created if needed
@@ -66,7 +66,7 @@ class LevelDBHook(BaseHook):
66
66
  return self.db
67
67
 
68
68
  def close_conn(self) -> None:
69
- """Closes connection."""
69
+ """Close connection."""
70
70
  db = self.db
71
71
  if db is not None:
72
72
  db.close()
@@ -17,9 +17,9 @@
17
17
  # under the License.
18
18
  from __future__ import annotations
19
19
 
20
- import warnings
21
20
  from typing import Any
22
21
 
22
+ from deprecated import deprecated
23
23
  from googleapiclient.discovery import Resource, build
24
24
  from googleapiclient.http import MediaFileUpload
25
25
 
@@ -27,18 +27,15 @@ from airflow.exceptions import AirflowProviderDeprecationWarning
27
27
  from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
28
28
 
29
29
 
30
+ @deprecated(
31
+ reason="The `GoogleAnalyticsHook` class is deprecated, please use `GoogleAnalyticsAdminHook` instead.",
32
+ category=AirflowProviderDeprecationWarning,
33
+ )
30
34
  class GoogleAnalyticsHook(GoogleBaseHook):
31
35
  """Hook for Google Analytics 360."""
32
36
 
33
37
  def __init__(self, api_version: str = "v3", *args, **kwargs):
34
38
  super().__init__(*args, **kwargs)
35
- warnings.warn(
36
- f"The `{type(self).__name__}` class is deprecated, please use "
37
- f"`GoogleAnalyticsAdminHook` instead.",
38
- AirflowProviderDeprecationWarning,
39
- stacklevel=2,
40
- )
41
-
42
39
  self.api_version = api_version
43
40
  self._conn = None
44
41
 
@@ -58,7 +55,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
58
55
  return result
59
56
 
60
57
  def get_conn(self) -> Resource:
61
- """Retrieves connection to Google Analytics 360."""
58
+ """Retrieve connection to Google Analytics 360."""
62
59
  if not self._conn:
63
60
  http_authorized = self._authorize()
64
61
  self._conn = build(
@@ -70,7 +67,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
70
67
  return self._conn
71
68
 
72
69
  def list_accounts(self) -> list[dict[str, Any]]:
73
- """Lists accounts list from Google Analytics 360."""
70
+ """List accounts list from Google Analytics 360."""
74
71
  self.log.info("Retrieving accounts list...")
75
72
  conn = self.get_conn()
76
73
  accounts = conn.management().accounts()
@@ -81,7 +78,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
81
78
  self, account_id: str, web_property_id: str, web_property_ad_words_link_id: str
82
79
  ) -> dict[str, Any]:
83
80
  """
84
- Returns a web property-Google Ads link to which the user has access.
81
+ Return a web property-Google Ads link to which the user has access.
85
82
 
86
83
  :param account_id: ID of the account which the given web property belongs to.
87
84
  :param web_property_id: Web property-Google Ads link UA-string.
@@ -105,7 +102,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
105
102
 
106
103
  def list_ad_words_links(self, account_id: str, web_property_id: str) -> list[dict[str, Any]]:
107
104
  """
108
- Lists webProperty-Google Ads links for a given web property.
105
+ List webProperty-Google Ads links for a given web property.
109
106
 
110
107
  :param account_id: ID of the account which the given web property belongs to.
111
108
  :param web_property_id: Web property UA-string to retrieve the Google Ads links for.
@@ -128,7 +125,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
128
125
  resumable_upload: bool = False,
129
126
  ) -> None:
130
127
  """
131
- Uploads file to GA via the Data Import API.
128
+ Upload file to GA via the Data Import API.
132
129
 
133
130
  :param file_location: The path and name of the file to upload.
134
131
  :param account_id: The GA account Id to which the data upload belongs.
@@ -165,7 +162,7 @@ class GoogleAnalyticsHook(GoogleBaseHook):
165
162
  delete_request_body: dict[str, Any],
166
163
  ) -> None:
167
164
  """
168
- Deletes the uploaded data for a given account/property/dataset.
165
+ Delete the uploaded data for a given account/property/dataset.
169
166
 
170
167
  :param account_id: The GA account Id to which the data upload belongs.
171
168
  :param web_property_id: UA-string associated with the upload.