apache-airflow-providers-google 10.2.0rc1__py3-none-any.whl → 10.3.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. airflow/providers/google/__init__.py +1 -1
  2. airflow/providers/google/ads/hooks/ads.py +38 -39
  3. airflow/providers/google/ads/transfers/ads_to_gcs.py +4 -4
  4. airflow/providers/google/cloud/_internal_client/secret_manager_client.py +6 -9
  5. airflow/providers/google/cloud/hooks/bigquery.py +328 -318
  6. airflow/providers/google/cloud/hooks/cloud_sql.py +66 -22
  7. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +46 -70
  8. airflow/providers/google/cloud/hooks/dataflow.py +11 -15
  9. airflow/providers/google/cloud/hooks/dataform.py +3 -3
  10. airflow/providers/google/cloud/hooks/dataproc.py +577 -573
  11. airflow/providers/google/cloud/hooks/functions.py +60 -76
  12. airflow/providers/google/cloud/hooks/gcs.py +108 -18
  13. airflow/providers/google/cloud/hooks/kubernetes_engine.py +69 -90
  14. airflow/providers/google/cloud/links/datafusion.py +4 -3
  15. airflow/providers/google/cloud/operators/bigquery.py +201 -191
  16. airflow/providers/google/cloud/operators/bigquery_dts.py +2 -1
  17. airflow/providers/google/cloud/operators/cloud_build.py +2 -1
  18. airflow/providers/google/cloud/operators/cloud_composer.py +4 -3
  19. airflow/providers/google/cloud/operators/cloud_sql.py +62 -28
  20. airflow/providers/google/cloud/operators/dataflow.py +6 -4
  21. airflow/providers/google/cloud/operators/dataform.py +3 -2
  22. airflow/providers/google/cloud/operators/dataproc.py +127 -123
  23. airflow/providers/google/cloud/operators/dataproc_metastore.py +18 -26
  24. airflow/providers/google/cloud/operators/gcs.py +35 -13
  25. airflow/providers/google/cloud/operators/kubernetes_engine.py +92 -42
  26. airflow/providers/google/cloud/operators/mlengine.py +2 -6
  27. airflow/providers/google/cloud/operators/vision.py +47 -56
  28. airflow/providers/google/cloud/sensors/bigquery.py +3 -2
  29. airflow/providers/google/cloud/sensors/gcs.py +5 -7
  30. airflow/providers/google/cloud/sensors/pubsub.py +2 -2
  31. airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +3 -2
  32. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +2 -1
  33. airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -4
  34. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +6 -5
  35. airflow/providers/google/cloud/transfers/gcs_to_gcs.py +46 -7
  36. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +5 -2
  37. airflow/providers/google/cloud/triggers/cloud_sql.py +102 -0
  38. airflow/providers/google/cloud/triggers/kubernetes_engine.py +28 -6
  39. airflow/providers/google/cloud/utils/bigquery.py +17 -0
  40. airflow/providers/google/get_provider_info.py +7 -2
  41. airflow/providers/google/suite/transfers/gcs_to_gdrive.py +4 -0
  42. airflow/providers/google/suite/transfers/local_to_drive.py +28 -26
  43. apache_airflow_providers_google-10.3.0rc1.dist-info/METADATA +289 -0
  44. {apache_airflow_providers_google-10.2.0rc1.dist-info → apache_airflow_providers_google-10.3.0rc1.dist-info}/RECORD +49 -48
  45. apache_airflow_providers_google-10.2.0rc1.dist-info/METADATA +0 -1824
  46. {apache_airflow_providers_google-10.2.0rc1.dist-info → apache_airflow_providers_google-10.3.0rc1.dist-info}/LICENSE +0 -0
  47. {apache_airflow_providers_google-10.2.0rc1.dist-info → apache_airflow_providers_google-10.3.0rc1.dist-info}/NOTICE +0 -0
  48. {apache_airflow_providers_google-10.2.0rc1.dist-info → apache_airflow_providers_google-10.3.0rc1.dist-info}/WHEEL +0 -0
  49. {apache_airflow_providers_google-10.2.0rc1.dist-info → apache_airflow_providers_google-10.3.0rc1.dist-info}/entry_points.txt +0 -0
  50. {apache_airflow_providers_google-10.2.0rc1.dist-info → apache_airflow_providers_google-10.3.0rc1.dist-info}/top_level.txt +0 -0
@@ -28,7 +28,7 @@ import packaging.version
28
28
 
29
29
  __all__ = ["__version__"]
30
30
 
31
- __version__ = "10.2.0"
31
+ __version__ = "10.3.0"
32
32
 
33
33
  try:
34
34
  from airflow import __version__ as airflow_version
@@ -24,9 +24,9 @@ from typing import IO, Any
24
24
 
25
25
  from google.ads.googleads.client import GoogleAdsClient
26
26
  from google.ads.googleads.errors import GoogleAdsException
27
- from google.ads.googleads.v13.services.services.customer_service import CustomerServiceClient
28
- from google.ads.googleads.v13.services.services.google_ads_service import GoogleAdsServiceClient
29
- from google.ads.googleads.v13.services.types.google_ads_service import GoogleAdsRow
27
+ from google.ads.googleads.v14.services.services.customer_service import CustomerServiceClient
28
+ from google.ads.googleads.v14.services.services.google_ads_service import GoogleAdsServiceClient
29
+ from google.ads.googleads.v14.services.types.google_ads_service import GoogleAdsRow
30
30
  from google.api_core.page_iterator import GRPCIterator
31
31
  from google.auth.exceptions import GoogleAuthError
32
32
 
@@ -36,14 +36,13 @@ from airflow.providers.google.common.hooks.base_google import get_field
36
36
 
37
37
 
38
38
  class GoogleAdsHook(BaseHook):
39
- """
40
- Hook for the Google Ads API.
39
+ """Interact with Google Ads API.
41
40
 
42
41
  This hook requires two connections:
43
42
 
44
- - gcp_conn_id - provides service account details (like any other GCP connection)
45
- - google_ads_conn_id - which contains information from Google Ads config.yaml file
46
- in the ``extras``. Example of the ``extras``:
43
+ - gcp_conn_id - provides service account details (like any other GCP connection)
44
+ - google_ads_conn_id - which contains information from Google Ads config.yaml file
45
+ in the ``extras``. Example of the ``extras``:
47
46
 
48
47
  .. code-block:: json
49
48
 
@@ -69,11 +68,9 @@ class GoogleAdsHook(BaseHook):
69
68
  :param gcp_conn_id: The connection ID with the service account details.
70
69
  :param google_ads_conn_id: The connection ID with the details of Google Ads config.yaml file.
71
70
  :param api_version: The Google Ads API version to use.
72
-
73
- :return: list of Google Ads Row object(s)
74
71
  """
75
72
 
76
- default_api_version = "v13"
73
+ default_api_version = "v14"
77
74
 
78
75
  def __init__(
79
76
  self,
@@ -90,10 +87,10 @@ class GoogleAdsHook(BaseHook):
90
87
  def search(
91
88
  self, client_ids: list[str], query: str, page_size: int = 10000, **kwargs
92
89
  ) -> list[GoogleAdsRow]:
93
- """
94
- Pulls data from the Google Ads API and returns it as native protobuf
95
- message instances (those seen in versions prior to 10.0.0 of the
96
- google-ads library).
90
+ """Pull data from the Google Ads API.
91
+
92
+ Native protobuf message instances are returned (those seen in versions
93
+ prior to 10.0.0 of the google-ads library).
97
94
 
98
95
  This method is for backwards compatibility with older versions of the
99
96
  google_ads_hook.
@@ -105,7 +102,7 @@ class GoogleAdsHook(BaseHook):
105
102
  :param client_ids: Google Ads client ID(s) to query the API for.
106
103
  :param query: Google Ads Query Language query.
107
104
  :param page_size: Number of results to return per page. Max 10000.
108
- :return: Google Ads API response, converted to Google Ads Row objects
105
+ :return: Google Ads API response, converted to Google Ads Row objects.
109
106
  """
110
107
  data_proto_plus = self._search(client_ids, query, page_size, **kwargs)
111
108
  data_native_pb = [row._pb for row in data_proto_plus]
@@ -115,9 +112,10 @@ class GoogleAdsHook(BaseHook):
115
112
  def search_proto_plus(
116
113
  self, client_ids: list[str], query: str, page_size: int = 10000, **kwargs
117
114
  ) -> list[GoogleAdsRow]:
118
- """
119
- Pulls data from the Google Ads API and returns it as proto-plus-python
120
- message instances that behave more like conventional python objects.
115
+ """Pull data from the Google Ads API.
116
+
117
+ Instances of proto-plus-python message are returned, which behave more
118
+ like conventional Python objects.
121
119
 
122
120
  :param client_ids: Google Ads client ID(s) to query the API for.
123
121
  :param query: Google Ads Query Language query.
@@ -127,12 +125,14 @@ class GoogleAdsHook(BaseHook):
127
125
  return self._search(client_ids, query, page_size, **kwargs)
128
126
 
129
127
  def list_accessible_customers(self) -> list[str]:
130
- """
131
- Returns resource names of customers directly accessible by the user authenticating the call.
132
- The resulting list of customers is based on your OAuth credentials. The request returns a list
133
- of all accounts that you are able to act upon directly given your current credentials. This will
134
- not necessarily include all accounts within the account hierarchy; rather, it will only include
135
- accounts where your authenticated user has been added with admin or other rights in the account.
128
+ """List resource names of customers.
129
+
130
+ The resulting list of customers is based on your OAuth credentials. The
131
+ request returns a list of all accounts that you are able to act upon
132
+ directly given your current credentials. This will not necessarily
133
+ include all accounts within the account hierarchy; rather, it will only
134
+ include accounts where your authenticated user has been added with admin
135
+ or other rights in the account.
136
136
 
137
137
  ..seealso::
138
138
  https://developers.google.com/google-ads/api/reference/rpc
@@ -152,7 +152,7 @@ class GoogleAdsHook(BaseHook):
152
152
 
153
153
  @cached_property
154
154
  def _get_service(self) -> GoogleAdsServiceClient:
155
- """Connects and authenticates with the Google Ads API using a service account."""
155
+ """Connect and authenticate with the Google Ads API using a service account."""
156
156
  client = self._get_client
157
157
  return client.get_service("GoogleAdsService", version=self.api_version)
158
158
 
@@ -170,7 +170,7 @@ class GoogleAdsHook(BaseHook):
170
170
 
171
171
  @cached_property
172
172
  def _get_customer_service(self) -> CustomerServiceClient:
173
- """Connects and authenticates with the Google Ads API using a service account."""
173
+ """Connect and authenticate with the Google Ads API using a service account."""
174
174
  with NamedTemporaryFile("w", suffix=".json") as secrets_temp:
175
175
  self._get_config()
176
176
  self._update_config_with_secret(secrets_temp)
@@ -182,9 +182,10 @@ class GoogleAdsHook(BaseHook):
182
182
  raise
183
183
 
184
184
  def _get_config(self) -> None:
185
- """
186
- Gets google ads connection from meta db and sets google_ads_config attribute with returned config
187
- file.
185
+ """Set up Google Ads config from Connection.
186
+
187
+ This pulls the connections from db, and uses it to set up
188
+ ``google_ads_config``.
188
189
  """
189
190
  conn = self.get_connection(self.google_ads_conn_id)
190
191
  if "google_ads_client" not in conn.extra_dejson:
@@ -193,10 +194,11 @@ class GoogleAdsHook(BaseHook):
193
194
  self.google_ads_config = conn.extra_dejson["google_ads_client"]
194
195
 
195
196
  def _update_config_with_secret(self, secrets_temp: IO[str]) -> None:
196
- """
197
- Gets Google Cloud secret from connection and saves the contents to the temp file
198
- Updates google ads config with file path of the temp file containing the secret
199
- Note, the secret must be passed as a file path for Google Ads API.
197
+ """Set up Google Cloud config secret from Connection.
198
+
199
+ This pulls the connection, saves the contents to a temp file, and point
200
+ the config to the path containing the secret. Note that the secret must
201
+ be passed as a file path for Google Ads API.
200
202
  """
201
203
  extras = self.get_connection(self.gcp_conn_id).extra_dejson
202
204
  secret = get_field(extras, "keyfile_dict")
@@ -210,8 +212,7 @@ class GoogleAdsHook(BaseHook):
210
212
  def _search(
211
213
  self, client_ids: list[str], query: str, page_size: int = 10000, **kwargs
212
214
  ) -> list[GoogleAdsRow]:
213
- """
214
- Pulls data from the Google Ads API.
215
+ """Pull data from the Google Ads API.
215
216
 
216
217
  :param client_ids: Google Ads client ID(s) to query the API for.
217
218
  :param query: Google Ads Query Language query.
@@ -231,11 +232,9 @@ class GoogleAdsHook(BaseHook):
231
232
  return self._extract_rows(iterators)
232
233
 
233
234
  def _extract_rows(self, iterators: list[GRPCIterator]) -> list[GoogleAdsRow]:
234
- """
235
- Convert Google Page Iterator (GRPCIterator) objects to Google Ads Rows.
235
+ """Convert Google Page Iterator (GRPCIterator) objects to Google Ads Rows.
236
236
 
237
237
  :param iterators: List of Google Page Iterator (GRPCIterator) objects
238
-
239
238
  :return: API response for all clients in the form of Google Ads Row object(s)
240
239
  """
241
240
  try:
@@ -30,10 +30,10 @@ if TYPE_CHECKING:
30
30
 
31
31
 
32
32
  class GoogleAdsToGcsOperator(BaseOperator):
33
- """
34
- Fetches the daily results from the Google Ads API for 1-n clients
35
- Converts and saves the data as a temporary CSV file
36
- Uploads the CSV to Google Cloud Storage.
33
+ """Fetch daily results from the Google Ads API for 1-n clients.
34
+
35
+ Converts and saves the data as a temporary CSV file Uploads the CSV to
36
+ Google Cloud Storage.
37
37
 
38
38
  .. seealso::
39
39
  For more information on the Google Ads API, take a look at the API docs:
@@ -30,11 +30,11 @@ SECRET_ID_PATTERN = r"^[a-zA-Z0-9-_]*$"
30
30
 
31
31
 
32
32
  class _SecretManagerClient(LoggingMixin):
33
- """
34
- Retrieves Secrets object from Google Cloud Secrets Manager. This is a common class reused between
35
- SecretsManager and Secrets Hook that provides the shared authentication and verification mechanisms.
36
- This class should not be used directly, use SecretsManager or SecretsHook instead.
33
+ """Retrieve Secrets object from Google Cloud Secrets Manager.
37
34
 
35
+ This is a common class reused between SecretsManager and Secrets Hook that
36
+ provides the shared authentication and verification mechanisms. This class
37
+ should not be used directly; use SecretsManager or SecretsHook instead.
38
38
 
39
39
  :param credentials: Credentials used to authenticate to GCP
40
40
  """
@@ -48,11 +48,9 @@ class _SecretManagerClient(LoggingMixin):
48
48
 
49
49
  @staticmethod
50
50
  def is_valid_secret_name(secret_name: str) -> bool:
51
- """
52
- Returns true if the secret name is valid.
51
+ """Whether the secret name is valid.
53
52
 
54
53
  :param secret_name: name of the secret
55
- :return:
56
54
  """
57
55
  return bool(re.match(SECRET_ID_PATTERN, secret_name))
58
56
 
@@ -63,8 +61,7 @@ class _SecretManagerClient(LoggingMixin):
63
61
  return _client
64
62
 
65
63
  def get_secret(self, secret_id: str, project_id: str, secret_version: str = "latest") -> str | None:
66
- """
67
- Get secret value from the Secret Manager.
64
+ """Get secret value from the Secret Manager.
68
65
 
69
66
  :param secret_id: Secret Key
70
67
  :param project_id: Project id to use