apache-airflow-providers-google 10.2.0rc1__py3-none-any.whl → 10.3.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/ads/hooks/ads.py +38 -39
- airflow/providers/google/ads/transfers/ads_to_gcs.py +4 -4
- airflow/providers/google/cloud/_internal_client/secret_manager_client.py +6 -9
- airflow/providers/google/cloud/hooks/bigquery.py +328 -318
- airflow/providers/google/cloud/hooks/cloud_sql.py +66 -22
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +46 -70
- airflow/providers/google/cloud/hooks/dataflow.py +11 -15
- airflow/providers/google/cloud/hooks/dataform.py +3 -3
- airflow/providers/google/cloud/hooks/dataproc.py +577 -573
- airflow/providers/google/cloud/hooks/functions.py +60 -76
- airflow/providers/google/cloud/hooks/gcs.py +108 -18
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +69 -90
- airflow/providers/google/cloud/links/datafusion.py +4 -3
- airflow/providers/google/cloud/operators/bigquery.py +201 -191
- airflow/providers/google/cloud/operators/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/operators/cloud_build.py +2 -1
- airflow/providers/google/cloud/operators/cloud_composer.py +4 -3
- airflow/providers/google/cloud/operators/cloud_sql.py +62 -28
- airflow/providers/google/cloud/operators/dataflow.py +6 -4
- airflow/providers/google/cloud/operators/dataform.py +3 -2
- airflow/providers/google/cloud/operators/dataproc.py +127 -123
- airflow/providers/google/cloud/operators/dataproc_metastore.py +18 -26
- airflow/providers/google/cloud/operators/gcs.py +35 -13
- airflow/providers/google/cloud/operators/kubernetes_engine.py +92 -42
- airflow/providers/google/cloud/operators/mlengine.py +2 -6
- airflow/providers/google/cloud/operators/vision.py +47 -56
- airflow/providers/google/cloud/sensors/bigquery.py +3 -2
- airflow/providers/google/cloud/sensors/gcs.py +5 -7
- airflow/providers/google/cloud/sensors/pubsub.py +2 -2
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +3 -2
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -4
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +6 -5
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +46 -7
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +5 -2
- airflow/providers/google/cloud/triggers/cloud_sql.py +102 -0
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +28 -6
- airflow/providers/google/cloud/utils/bigquery.py +17 -0
- airflow/providers/google/get_provider_info.py +7 -2
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +4 -0
- airflow/providers/google/suite/transfers/local_to_drive.py +28 -26
- apache_airflow_providers_google-10.3.0rc1.dist-info/METADATA +289 -0
- {apache_airflow_providers_google-10.2.0rc1.dist-info → apache_airflow_providers_google-10.3.0rc1.dist-info}/RECORD +49 -48
- apache_airflow_providers_google-10.2.0rc1.dist-info/METADATA +0 -1824
- {apache_airflow_providers_google-10.2.0rc1.dist-info → apache_airflow_providers_google-10.3.0rc1.dist-info}/LICENSE +0 -0
- {apache_airflow_providers_google-10.2.0rc1.dist-info → apache_airflow_providers_google-10.3.0rc1.dist-info}/NOTICE +0 -0
- {apache_airflow_providers_google-10.2.0rc1.dist-info → apache_airflow_providers_google-10.3.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-10.2.0rc1.dist-info → apache_airflow_providers_google-10.3.0rc1.dist-info}/entry_points.txt +0 -0
- {apache_airflow_providers_google-10.2.0rc1.dist-info → apache_airflow_providers_google-10.3.0rc1.dist-info}/top_level.txt +0 -0
@@ -24,9 +24,9 @@ from typing import IO, Any
|
|
24
24
|
|
25
25
|
from google.ads.googleads.client import GoogleAdsClient
|
26
26
|
from google.ads.googleads.errors import GoogleAdsException
|
27
|
-
from google.ads.googleads.
|
28
|
-
from google.ads.googleads.
|
29
|
-
from google.ads.googleads.
|
27
|
+
from google.ads.googleads.v14.services.services.customer_service import CustomerServiceClient
|
28
|
+
from google.ads.googleads.v14.services.services.google_ads_service import GoogleAdsServiceClient
|
29
|
+
from google.ads.googleads.v14.services.types.google_ads_service import GoogleAdsRow
|
30
30
|
from google.api_core.page_iterator import GRPCIterator
|
31
31
|
from google.auth.exceptions import GoogleAuthError
|
32
32
|
|
@@ -36,14 +36,13 @@ from airflow.providers.google.common.hooks.base_google import get_field
|
|
36
36
|
|
37
37
|
|
38
38
|
class GoogleAdsHook(BaseHook):
|
39
|
-
"""
|
40
|
-
Hook for the Google Ads API.
|
39
|
+
"""Interact with Google Ads API.
|
41
40
|
|
42
41
|
This hook requires two connections:
|
43
42
|
|
44
|
-
|
45
|
-
|
46
|
-
|
43
|
+
- gcp_conn_id - provides service account details (like any other GCP connection)
|
44
|
+
- google_ads_conn_id - which contains information from Google Ads config.yaml file
|
45
|
+
in the ``extras``. Example of the ``extras``:
|
47
46
|
|
48
47
|
.. code-block:: json
|
49
48
|
|
@@ -69,11 +68,9 @@ class GoogleAdsHook(BaseHook):
|
|
69
68
|
:param gcp_conn_id: The connection ID with the service account details.
|
70
69
|
:param google_ads_conn_id: The connection ID with the details of Google Ads config.yaml file.
|
71
70
|
:param api_version: The Google Ads API version to use.
|
72
|
-
|
73
|
-
:return: list of Google Ads Row object(s)
|
74
71
|
"""
|
75
72
|
|
76
|
-
default_api_version = "
|
73
|
+
default_api_version = "v14"
|
77
74
|
|
78
75
|
def __init__(
|
79
76
|
self,
|
@@ -90,10 +87,10 @@ class GoogleAdsHook(BaseHook):
|
|
90
87
|
def search(
|
91
88
|
self, client_ids: list[str], query: str, page_size: int = 10000, **kwargs
|
92
89
|
) -> list[GoogleAdsRow]:
|
93
|
-
"""
|
94
|
-
|
95
|
-
message instances (those seen in versions
|
96
|
-
google-ads library).
|
90
|
+
"""Pull data from the Google Ads API.
|
91
|
+
|
92
|
+
Native protobuf message instances are returned (those seen in versions
|
93
|
+
prior to 10.0.0 of the google-ads library).
|
97
94
|
|
98
95
|
This method is for backwards compatibility with older versions of the
|
99
96
|
google_ads_hook.
|
@@ -105,7 +102,7 @@ class GoogleAdsHook(BaseHook):
|
|
105
102
|
:param client_ids: Google Ads client ID(s) to query the API for.
|
106
103
|
:param query: Google Ads Query Language query.
|
107
104
|
:param page_size: Number of results to return per page. Max 10000.
|
108
|
-
:return: Google Ads API response, converted to Google Ads Row objects
|
105
|
+
:return: Google Ads API response, converted to Google Ads Row objects.
|
109
106
|
"""
|
110
107
|
data_proto_plus = self._search(client_ids, query, page_size, **kwargs)
|
111
108
|
data_native_pb = [row._pb for row in data_proto_plus]
|
@@ -115,9 +112,10 @@ class GoogleAdsHook(BaseHook):
|
|
115
112
|
def search_proto_plus(
|
116
113
|
self, client_ids: list[str], query: str, page_size: int = 10000, **kwargs
|
117
114
|
) -> list[GoogleAdsRow]:
|
118
|
-
"""
|
119
|
-
|
120
|
-
|
115
|
+
"""Pull data from the Google Ads API.
|
116
|
+
|
117
|
+
Instances of proto-plus-python message are returned, which behave more
|
118
|
+
like conventional Python objects.
|
121
119
|
|
122
120
|
:param client_ids: Google Ads client ID(s) to query the API for.
|
123
121
|
:param query: Google Ads Query Language query.
|
@@ -127,12 +125,14 @@ class GoogleAdsHook(BaseHook):
|
|
127
125
|
return self._search(client_ids, query, page_size, **kwargs)
|
128
126
|
|
129
127
|
def list_accessible_customers(self) -> list[str]:
|
130
|
-
"""
|
131
|
-
|
132
|
-
The resulting list of customers is based on your OAuth credentials. The
|
133
|
-
of all accounts that you are able to act upon
|
134
|
-
|
135
|
-
|
128
|
+
"""List resource names of customers.
|
129
|
+
|
130
|
+
The resulting list of customers is based on your OAuth credentials. The
|
131
|
+
request returns a list of all accounts that you are able to act upon
|
132
|
+
directly given your current credentials. This will not necessarily
|
133
|
+
include all accounts within the account hierarchy; rather, it will only
|
134
|
+
include accounts where your authenticated user has been added with admin
|
135
|
+
or other rights in the account.
|
136
136
|
|
137
137
|
..seealso::
|
138
138
|
https://developers.google.com/google-ads/api/reference/rpc
|
@@ -152,7 +152,7 @@ class GoogleAdsHook(BaseHook):
|
|
152
152
|
|
153
153
|
@cached_property
|
154
154
|
def _get_service(self) -> GoogleAdsServiceClient:
|
155
|
-
"""
|
155
|
+
"""Connect and authenticate with the Google Ads API using a service account."""
|
156
156
|
client = self._get_client
|
157
157
|
return client.get_service("GoogleAdsService", version=self.api_version)
|
158
158
|
|
@@ -170,7 +170,7 @@ class GoogleAdsHook(BaseHook):
|
|
170
170
|
|
171
171
|
@cached_property
|
172
172
|
def _get_customer_service(self) -> CustomerServiceClient:
|
173
|
-
"""
|
173
|
+
"""Connect and authenticate with the Google Ads API using a service account."""
|
174
174
|
with NamedTemporaryFile("w", suffix=".json") as secrets_temp:
|
175
175
|
self._get_config()
|
176
176
|
self._update_config_with_secret(secrets_temp)
|
@@ -182,9 +182,10 @@ class GoogleAdsHook(BaseHook):
|
|
182
182
|
raise
|
183
183
|
|
184
184
|
def _get_config(self) -> None:
|
185
|
-
"""
|
186
|
-
|
187
|
-
|
185
|
+
"""Set up Google Ads config from Connection.
|
186
|
+
|
187
|
+
This pulls the connections from db, and uses it to set up
|
188
|
+
``google_ads_config``.
|
188
189
|
"""
|
189
190
|
conn = self.get_connection(self.google_ads_conn_id)
|
190
191
|
if "google_ads_client" not in conn.extra_dejson:
|
@@ -193,10 +194,11 @@ class GoogleAdsHook(BaseHook):
|
|
193
194
|
self.google_ads_config = conn.extra_dejson["google_ads_client"]
|
194
195
|
|
195
196
|
def _update_config_with_secret(self, secrets_temp: IO[str]) -> None:
|
196
|
-
"""
|
197
|
-
|
198
|
-
|
199
|
-
|
197
|
+
"""Set up Google Cloud config secret from Connection.
|
198
|
+
|
199
|
+
This pulls the connection, saves the contents to a temp file, and point
|
200
|
+
the config to the path containing the secret. Note that the secret must
|
201
|
+
be passed as a file path for Google Ads API.
|
200
202
|
"""
|
201
203
|
extras = self.get_connection(self.gcp_conn_id).extra_dejson
|
202
204
|
secret = get_field(extras, "keyfile_dict")
|
@@ -210,8 +212,7 @@ class GoogleAdsHook(BaseHook):
|
|
210
212
|
def _search(
|
211
213
|
self, client_ids: list[str], query: str, page_size: int = 10000, **kwargs
|
212
214
|
) -> list[GoogleAdsRow]:
|
213
|
-
"""
|
214
|
-
Pulls data from the Google Ads API.
|
215
|
+
"""Pull data from the Google Ads API.
|
215
216
|
|
216
217
|
:param client_ids: Google Ads client ID(s) to query the API for.
|
217
218
|
:param query: Google Ads Query Language query.
|
@@ -231,11 +232,9 @@ class GoogleAdsHook(BaseHook):
|
|
231
232
|
return self._extract_rows(iterators)
|
232
233
|
|
233
234
|
def _extract_rows(self, iterators: list[GRPCIterator]) -> list[GoogleAdsRow]:
|
234
|
-
"""
|
235
|
-
Convert Google Page Iterator (GRPCIterator) objects to Google Ads Rows.
|
235
|
+
"""Convert Google Page Iterator (GRPCIterator) objects to Google Ads Rows.
|
236
236
|
|
237
237
|
:param iterators: List of Google Page Iterator (GRPCIterator) objects
|
238
|
-
|
239
238
|
:return: API response for all clients in the form of Google Ads Row object(s)
|
240
239
|
"""
|
241
240
|
try:
|
@@ -30,10 +30,10 @@ if TYPE_CHECKING:
|
|
30
30
|
|
31
31
|
|
32
32
|
class GoogleAdsToGcsOperator(BaseOperator):
|
33
|
-
"""
|
34
|
-
|
35
|
-
Converts and saves the data as a temporary CSV file
|
36
|
-
|
33
|
+
"""Fetch daily results from the Google Ads API for 1-n clients.
|
34
|
+
|
35
|
+
Converts and saves the data as a temporary CSV file Uploads the CSV to
|
36
|
+
Google Cloud Storage.
|
37
37
|
|
38
38
|
.. seealso::
|
39
39
|
For more information on the Google Ads API, take a look at the API docs:
|
@@ -30,11 +30,11 @@ SECRET_ID_PATTERN = r"^[a-zA-Z0-9-_]*$"
|
|
30
30
|
|
31
31
|
|
32
32
|
class _SecretManagerClient(LoggingMixin):
|
33
|
-
"""
|
34
|
-
Retrieves Secrets object from Google Cloud Secrets Manager. This is a common class reused between
|
35
|
-
SecretsManager and Secrets Hook that provides the shared authentication and verification mechanisms.
|
36
|
-
This class should not be used directly, use SecretsManager or SecretsHook instead.
|
33
|
+
"""Retrieve Secrets object from Google Cloud Secrets Manager.
|
37
34
|
|
35
|
+
This is a common class reused between SecretsManager and Secrets Hook that
|
36
|
+
provides the shared authentication and verification mechanisms. This class
|
37
|
+
should not be used directly; use SecretsManager or SecretsHook instead.
|
38
38
|
|
39
39
|
:param credentials: Credentials used to authenticate to GCP
|
40
40
|
"""
|
@@ -48,11 +48,9 @@ class _SecretManagerClient(LoggingMixin):
|
|
48
48
|
|
49
49
|
@staticmethod
|
50
50
|
def is_valid_secret_name(secret_name: str) -> bool:
|
51
|
-
"""
|
52
|
-
Returns true if the secret name is valid.
|
51
|
+
"""Whether the secret name is valid.
|
53
52
|
|
54
53
|
:param secret_name: name of the secret
|
55
|
-
:return:
|
56
54
|
"""
|
57
55
|
return bool(re.match(SECRET_ID_PATTERN, secret_name))
|
58
56
|
|
@@ -63,8 +61,7 @@ class _SecretManagerClient(LoggingMixin):
|
|
63
61
|
return _client
|
64
62
|
|
65
63
|
def get_secret(self, secret_id: str, project_id: str, secret_version: str = "latest") -> str | None:
|
66
|
-
"""
|
67
|
-
Get secret value from the Secret Manager.
|
64
|
+
"""Get secret value from the Secret Manager.
|
68
65
|
|
69
66
|
:param secret_id: Secret Key
|
70
67
|
:param project_id: Project id to use
|