apache-airflow-providers-google 9.0.0rc1__py3-none-any.whl → 10.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/cloud/hooks/automl.py +5 -6
- airflow/providers/google/cloud/hooks/bigquery.py +5 -6
- airflow/providers/google/cloud/hooks/bigquery_dts.py +11 -8
- airflow/providers/google/cloud/hooks/bigtable.py +5 -6
- airflow/providers/google/cloud/hooks/cloud_build.py +14 -11
- airflow/providers/google/cloud/hooks/cloud_composer.py +16 -0
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +5 -14
- airflow/providers/google/cloud/hooks/cloud_sql.py +5 -7
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +5 -5
- airflow/providers/google/cloud/hooks/compute.py +5 -6
- airflow/providers/google/cloud/hooks/compute_ssh.py +8 -12
- airflow/providers/google/cloud/hooks/datacatalog.py +5 -9
- airflow/providers/google/cloud/hooks/dataflow.py +14 -6
- airflow/providers/google/cloud/hooks/dataform.py +8 -0
- airflow/providers/google/cloud/hooks/datafusion.py +13 -6
- airflow/providers/google/cloud/hooks/dataplex.py +5 -8
- airflow/providers/google/cloud/hooks/dataproc.py +13 -8
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +8 -0
- airflow/providers/google/cloud/hooks/datastore.py +5 -6
- airflow/providers/google/cloud/hooks/dlp.py +5 -9
- airflow/providers/google/cloud/hooks/functions.py +5 -6
- airflow/providers/google/cloud/hooks/gcs.py +6 -6
- airflow/providers/google/cloud/hooks/gdm.py +5 -6
- airflow/providers/google/cloud/hooks/kms.py +5 -9
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +6 -4
- airflow/providers/google/cloud/hooks/life_sciences.py +5 -9
- airflow/providers/google/cloud/hooks/mlengine.py +8 -0
- airflow/providers/google/cloud/hooks/natural_language.py +5 -9
- airflow/providers/google/cloud/hooks/os_login.py +5 -6
- airflow/providers/google/cloud/hooks/pubsub.py +5 -5
- airflow/providers/google/cloud/hooks/secret_manager.py +5 -9
- airflow/providers/google/cloud/hooks/spanner.py +5 -6
- airflow/providers/google/cloud/hooks/speech_to_text.py +5 -9
- airflow/providers/google/cloud/hooks/stackdriver.py +5 -6
- airflow/providers/google/cloud/hooks/tasks.py +5 -9
- airflow/providers/google/cloud/hooks/text_to_speech.py +5 -9
- airflow/providers/google/cloud/hooks/translate.py +5 -6
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +5 -5
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +5 -6
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +5 -6
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +8 -0
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +8 -0
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +5 -6
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +8 -0
- airflow/providers/google/cloud/hooks/video_intelligence.py +5 -9
- airflow/providers/google/cloud/hooks/vision.py +5 -6
- airflow/providers/google/cloud/hooks/workflows.py +8 -0
- airflow/providers/google/cloud/operators/bigquery.py +1 -148
- airflow/providers/google/cloud/operators/cloud_build.py +0 -13
- airflow/providers/google/cloud/operators/cloud_composer.py +2 -68
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +0 -21
- airflow/providers/google/cloud/operators/dataflow.py +2 -72
- airflow/providers/google/cloud/operators/dataform.py +5 -141
- airflow/providers/google/cloud/operators/datafusion.py +0 -102
- airflow/providers/google/cloud/operators/dataplex.py +3 -61
- airflow/providers/google/cloud/operators/dataproc.py +5 -13
- airflow/providers/google/cloud/operators/datastore.py +2 -86
- airflow/providers/google/cloud/operators/gcs.py +1 -43
- airflow/providers/google/cloud/operators/mlengine.py +1 -123
- airflow/providers/google/cloud/operators/pubsub.py +0 -61
- airflow/providers/google/cloud/operators/stackdriver.py +0 -101
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +0 -26
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +0 -41
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +0 -39
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +0 -71
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +0 -71
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +0 -35
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +0 -41
- airflow/providers/google/cloud/sensors/bigquery.py +0 -24
- airflow/providers/google/cloud/sensors/cloud_composer.py +0 -10
- airflow/providers/google/cloud/sensors/dataflow.py +0 -42
- airflow/providers/google/cloud/sensors/dataform.py +0 -12
- airflow/providers/google/cloud/sensors/datafusion.py +0 -11
- airflow/providers/google/cloud/sensors/dataplex.py +0 -10
- airflow/providers/google/cloud/sensors/gcs.py +0 -44
- airflow/providers/google/cloud/sensors/pubsub.py +0 -11
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +0 -12
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +0 -12
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +0 -10
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +0 -11
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +0 -11
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +0 -11
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +0 -12
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +0 -11
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +0 -13
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +0 -11
- airflow/providers/google/cloud/transfers/gcs_to_local.py +0 -11
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +0 -11
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +0 -12
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +0 -11
- airflow/providers/google/cloud/transfers/local_to_gcs.py +0 -9
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +0 -11
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +0 -11
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +0 -12
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +0 -11
- airflow/providers/google/cloud/triggers/bigquery_dts.py +0 -12
- airflow/providers/google/cloud/triggers/cloud_build.py +0 -11
- airflow/providers/google/cloud/triggers/cloud_composer.py +0 -9
- airflow/providers/google/cloud/triggers/dataflow.py +0 -7
- airflow/providers/google/cloud/triggers/datafusion.py +0 -11
- airflow/providers/google/cloud/triggers/dataproc.py +2 -8
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +0 -4
- airflow/providers/google/cloud/triggers/mlengine.py +0 -3
- airflow/providers/google/common/hooks/discovery_api.py +0 -7
- airflow/providers/google/firebase/hooks/firestore.py +0 -10
- airflow/providers/google/get_provider_info.py +1 -0
- airflow/providers/google/marketing_platform/example_dags/example_display_video.py +0 -64
- airflow/providers/google/marketing_platform/hooks/campaign_manager.py +3 -3
- airflow/providers/google/marketing_platform/hooks/display_video.py +7 -38
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +22 -22
- airflow/providers/google/marketing_platform/operators/display_video.py +0 -284
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +3 -3
- airflow/providers/google/marketing_platform/sensors/display_video.py +0 -65
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +0 -4
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +0 -4
- airflow/providers/google_vendor/googleads/interceptors/interceptor.py +1 -1
- {apache_airflow_providers_google-9.0.0rc1.dist-info → apache_airflow_providers_google-10.0.0rc1.dist-info}/METADATA +34 -4
- {apache_airflow_providers_google-9.0.0rc1.dist-info → apache_airflow_providers_google-10.0.0rc1.dist-info}/RECORD +123 -123
- {apache_airflow_providers_google-9.0.0rc1.dist-info → apache_airflow_providers_google-10.0.0rc1.dist-info}/LICENSE +0 -0
- {apache_airflow_providers_google-9.0.0rc1.dist-info → apache_airflow_providers_google-10.0.0rc1.dist-info}/NOTICE +0 -0
- {apache_airflow_providers_google-9.0.0rc1.dist-info → apache_airflow_providers_google-10.0.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-9.0.0rc1.dist-info → apache_airflow_providers_google-10.0.0rc1.dist-info}/entry_points.txt +0 -0
- {apache_airflow_providers_google-9.0.0rc1.dist-info → apache_airflow_providers_google-10.0.0rc1.dist-info}/top_level.txt +0 -0
@@ -17,7 +17,6 @@
|
|
17
17
|
"""This module contains Google Dataplex sensors."""
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
import warnings
|
21
20
|
from typing import TYPE_CHECKING, Sequence
|
22
21
|
|
23
22
|
if TYPE_CHECKING:
|
@@ -54,8 +53,6 @@ class DataplexTaskStateSensor(BaseSensorOperator):
|
|
54
53
|
will not be retried.
|
55
54
|
:param metadata: Additional metadata that is provided to the method.
|
56
55
|
:param gcp_conn_id: The connection ID to use when fetching connection info.
|
57
|
-
:param delegate_to: The account to impersonate, if any. For this to work, the service accountmaking the
|
58
|
-
request must have domain-wide delegation enabled.
|
59
56
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
60
57
|
credentials, or chained list of accounts required to get the access_token
|
61
58
|
of the last account in the list, which will be impersonated in the request.
|
@@ -78,7 +75,6 @@ class DataplexTaskStateSensor(BaseSensorOperator):
|
|
78
75
|
retry: Retry | _MethodDefault = DEFAULT,
|
79
76
|
metadata: Sequence[tuple[str, str]] = (),
|
80
77
|
gcp_conn_id: str = "google_cloud_default",
|
81
|
-
delegate_to: str | None = None,
|
82
78
|
impersonation_chain: str | Sequence[str] | None = None,
|
83
79
|
*args,
|
84
80
|
**kwargs,
|
@@ -92,18 +88,12 @@ class DataplexTaskStateSensor(BaseSensorOperator):
|
|
92
88
|
self.retry = retry
|
93
89
|
self.metadata = metadata
|
94
90
|
self.gcp_conn_id = gcp_conn_id
|
95
|
-
if delegate_to:
|
96
|
-
warnings.warn(
|
97
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
98
|
-
)
|
99
|
-
self.delegate_to = delegate_to
|
100
91
|
self.impersonation_chain = impersonation_chain
|
101
92
|
|
102
93
|
def poke(self, context: Context) -> bool:
|
103
94
|
self.log.info("Waiting for task %s to be %s", self.dataplex_task_id, TaskState.ACTIVE)
|
104
95
|
hook = DataplexHook(
|
105
96
|
gcp_conn_id=self.gcp_conn_id,
|
106
|
-
delegate_to=self.delegate_to,
|
107
97
|
api_version=self.api_version,
|
108
98
|
impersonation_chain=self.impersonation_chain,
|
109
99
|
)
|
@@ -45,9 +45,6 @@ class GCSObjectExistenceSensor(BaseSensorOperator):
|
|
45
45
|
storage bucket.
|
46
46
|
:param google_cloud_conn_id: The connection ID to use when
|
47
47
|
connecting to Google Cloud Storage.
|
48
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
49
|
-
if any. For this to work, the service account making the request must have
|
50
|
-
domain-wide delegation enabled.
|
51
48
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
52
49
|
credentials, or chained list of accounts required to get the access_token
|
53
50
|
of the last account in the list, which will be impersonated in the request.
|
@@ -72,7 +69,6 @@ class GCSObjectExistenceSensor(BaseSensorOperator):
|
|
72
69
|
bucket: str,
|
73
70
|
object: str,
|
74
71
|
google_cloud_conn_id: str = "google_cloud_default",
|
75
|
-
delegate_to: str | None = None,
|
76
72
|
impersonation_chain: str | Sequence[str] | None = None,
|
77
73
|
retry: Retry = DEFAULT_RETRY,
|
78
74
|
deferrable: bool = False,
|
@@ -83,11 +79,6 @@ class GCSObjectExistenceSensor(BaseSensorOperator):
|
|
83
79
|
self.bucket = bucket
|
84
80
|
self.object = object
|
85
81
|
self.google_cloud_conn_id = google_cloud_conn_id
|
86
|
-
if delegate_to:
|
87
|
-
warnings.warn(
|
88
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
89
|
-
)
|
90
|
-
self.delegate_to = delegate_to
|
91
82
|
self.impersonation_chain = impersonation_chain
|
92
83
|
self.retry = retry
|
93
84
|
|
@@ -97,7 +88,6 @@ class GCSObjectExistenceSensor(BaseSensorOperator):
|
|
97
88
|
self.log.info("Sensor checks existence of : %s, %s", self.bucket, self.object)
|
98
89
|
hook = GCSHook(
|
99
90
|
gcp_conn_id=self.google_cloud_conn_id,
|
100
|
-
delegate_to=self.delegate_to,
|
101
91
|
impersonation_chain=self.impersonation_chain,
|
102
92
|
)
|
103
93
|
return hook.exists(self.bucket, self.object, self.retry)
|
@@ -115,7 +105,6 @@ class GCSObjectExistenceSensor(BaseSensorOperator):
|
|
115
105
|
poke_interval=self.poke_interval,
|
116
106
|
google_cloud_conn_id=self.google_cloud_conn_id,
|
117
107
|
hook_params={
|
118
|
-
"delegate_to": self.delegate_to,
|
119
108
|
"impersonation_chain": self.impersonation_chain,
|
120
109
|
},
|
121
110
|
),
|
@@ -143,9 +132,6 @@ class GCSObjectExistenceAsyncSensor(GCSObjectExistenceSensor):
|
|
143
132
|
:param bucket: The Google Cloud Storage bucket where the object is.
|
144
133
|
:param object: The name of the object to check in the Google cloud storage bucket.
|
145
134
|
:param google_cloud_conn_id: The connection ID to use when connecting to Google Cloud Storage.
|
146
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
147
|
-
if any. For this to work, the service account making the request must have
|
148
|
-
domain-wide delegation enabled.
|
149
135
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
150
136
|
credentials, or chained list of accounts required to get the access_token
|
151
137
|
of the last account in the list, which will be impersonated in the request.
|
@@ -190,9 +176,6 @@ class GCSObjectUpdateSensor(BaseSensorOperator):
|
|
190
176
|
as parameter.
|
191
177
|
:param google_cloud_conn_id: The connection ID to use when
|
192
178
|
connecting to Google Cloud Storage.
|
193
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
194
|
-
if any. For this to work, the service account making the request must have
|
195
|
-
domain-wide delegation enabled.
|
196
179
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
197
180
|
credentials, or chained list of accounts required to get the access_token
|
198
181
|
of the last account in the list, which will be impersonated in the request.
|
@@ -216,7 +199,6 @@ class GCSObjectUpdateSensor(BaseSensorOperator):
|
|
216
199
|
object: str,
|
217
200
|
ts_func: Callable = ts_function,
|
218
201
|
google_cloud_conn_id: str = "google_cloud_default",
|
219
|
-
delegate_to: str | None = None,
|
220
202
|
impersonation_chain: str | Sequence[str] | None = None,
|
221
203
|
**kwargs,
|
222
204
|
) -> None:
|
@@ -226,18 +208,12 @@ class GCSObjectUpdateSensor(BaseSensorOperator):
|
|
226
208
|
self.object = object
|
227
209
|
self.ts_func = ts_func
|
228
210
|
self.google_cloud_conn_id = google_cloud_conn_id
|
229
|
-
if delegate_to:
|
230
|
-
warnings.warn(
|
231
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
232
|
-
)
|
233
|
-
self.delegate_to = delegate_to
|
234
211
|
self.impersonation_chain = impersonation_chain
|
235
212
|
|
236
213
|
def poke(self, context: Context) -> bool:
|
237
214
|
self.log.info("Sensor checks existence of : %s, %s", self.bucket, self.object)
|
238
215
|
hook = GCSHook(
|
239
216
|
gcp_conn_id=self.google_cloud_conn_id,
|
240
|
-
delegate_to=self.delegate_to,
|
241
217
|
impersonation_chain=self.impersonation_chain,
|
242
218
|
)
|
243
219
|
return hook.is_updated_after(self.bucket, self.object, self.ts_func(context))
|
@@ -256,9 +232,6 @@ class GCSObjectsWithPrefixExistenceSensor(BaseSensorOperator):
|
|
256
232
|
storage bucket.
|
257
233
|
:param google_cloud_conn_id: The connection ID to use when
|
258
234
|
connecting to Google Cloud Storage.
|
259
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
260
|
-
if any. For this to work, the service account making the request must have
|
261
|
-
domain-wide delegation enabled.
|
262
235
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
263
236
|
credentials, or chained list of accounts required to get the access_token
|
264
237
|
of the last account in the list, which will be impersonated in the request.
|
@@ -281,7 +254,6 @@ class GCSObjectsWithPrefixExistenceSensor(BaseSensorOperator):
|
|
281
254
|
bucket: str,
|
282
255
|
prefix: str,
|
283
256
|
google_cloud_conn_id: str = "google_cloud_default",
|
284
|
-
delegate_to: str | None = None,
|
285
257
|
impersonation_chain: str | Sequence[str] | None = None,
|
286
258
|
**kwargs,
|
287
259
|
) -> None:
|
@@ -289,11 +261,6 @@ class GCSObjectsWithPrefixExistenceSensor(BaseSensorOperator):
|
|
289
261
|
self.bucket = bucket
|
290
262
|
self.prefix = prefix
|
291
263
|
self.google_cloud_conn_id = google_cloud_conn_id
|
292
|
-
if delegate_to:
|
293
|
-
warnings.warn(
|
294
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
295
|
-
)
|
296
|
-
self.delegate_to = delegate_to
|
297
264
|
self._matches: list[str] = []
|
298
265
|
self.impersonation_chain = impersonation_chain
|
299
266
|
|
@@ -301,7 +268,6 @@ class GCSObjectsWithPrefixExistenceSensor(BaseSensorOperator):
|
|
301
268
|
self.log.info("Sensor checks existence of objects: %s, %s", self.bucket, self.prefix)
|
302
269
|
hook = GCSHook(
|
303
270
|
gcp_conn_id=self.google_cloud_conn_id,
|
304
|
-
delegate_to=self.delegate_to,
|
305
271
|
impersonation_chain=self.impersonation_chain,
|
306
272
|
)
|
307
273
|
self._matches = hook.list(self.bucket, prefix=self.prefix)
|
@@ -346,9 +312,6 @@ class GCSUploadSessionCompleteSensor(BaseSensorOperator):
|
|
346
312
|
when this happens. If false an error will be raised.
|
347
313
|
:param google_cloud_conn_id: The connection ID to use when connecting
|
348
314
|
to Google Cloud Storage.
|
349
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
350
|
-
if any. For this to work, the service account making the request must have
|
351
|
-
domain-wide delegation enabled.
|
352
315
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
353
316
|
credentials, or chained list of accounts required to get the access_token
|
354
317
|
of the last account in the list, which will be impersonated in the request.
|
@@ -375,7 +338,6 @@ class GCSUploadSessionCompleteSensor(BaseSensorOperator):
|
|
375
338
|
previous_objects: set[str] | None = None,
|
376
339
|
allow_delete: bool = True,
|
377
340
|
google_cloud_conn_id: str = "google_cloud_default",
|
378
|
-
delegate_to: str | None = None,
|
379
341
|
impersonation_chain: str | Sequence[str] | None = None,
|
380
342
|
**kwargs,
|
381
343
|
) -> None:
|
@@ -392,11 +354,6 @@ class GCSUploadSessionCompleteSensor(BaseSensorOperator):
|
|
392
354
|
self.inactivity_seconds = 0
|
393
355
|
self.allow_delete = allow_delete
|
394
356
|
self.google_cloud_conn_id = google_cloud_conn_id
|
395
|
-
if delegate_to:
|
396
|
-
warnings.warn(
|
397
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
398
|
-
)
|
399
|
-
self.delegate_to = delegate_to
|
400
357
|
self.last_activity_time = None
|
401
358
|
self.impersonation_chain = impersonation_chain
|
402
359
|
self.hook: GCSHook | None = None
|
@@ -405,7 +362,6 @@ class GCSUploadSessionCompleteSensor(BaseSensorOperator):
|
|
405
362
|
if not self.hook:
|
406
363
|
self.hook = GCSHook(
|
407
364
|
gcp_conn_id=self.google_cloud_conn_id,
|
408
|
-
delegate_to=self.delegate_to,
|
409
365
|
impersonation_chain=self.impersonation_chain,
|
410
366
|
)
|
411
367
|
return self.hook
|
@@ -18,7 +18,6 @@
|
|
18
18
|
"""This module contains a Google PubSub sensor."""
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
|
-
import warnings
|
22
21
|
from typing import TYPE_CHECKING, Any, Callable, Sequence
|
23
22
|
|
24
23
|
from google.cloud.pubsub_v1.types import ReceivedMessage
|
@@ -67,9 +66,6 @@ class PubSubPullSensor(BaseSensorOperator):
|
|
67
66
|
immediately rather than by any downstream tasks
|
68
67
|
:param gcp_conn_id: The connection ID to use connecting to
|
69
68
|
Google Cloud.
|
70
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
71
|
-
if any. For this to work, the service account making the request must have
|
72
|
-
domain-wide delegation enabled.
|
73
69
|
:param messages_callback: (Optional) Callback to process received messages.
|
74
70
|
It's return value will be saved to XCom.
|
75
71
|
If you are pulling large messages, you probably want to provide a custom callback.
|
@@ -101,18 +97,12 @@ class PubSubPullSensor(BaseSensorOperator):
|
|
101
97
|
ack_messages: bool = False,
|
102
98
|
gcp_conn_id: str = "google_cloud_default",
|
103
99
|
messages_callback: Callable[[list[ReceivedMessage], Context], Any] | None = None,
|
104
|
-
delegate_to: str | None = None,
|
105
100
|
impersonation_chain: str | Sequence[str] | None = None,
|
106
101
|
**kwargs,
|
107
102
|
) -> None:
|
108
103
|
|
109
104
|
super().__init__(**kwargs)
|
110
105
|
self.gcp_conn_id = gcp_conn_id
|
111
|
-
if delegate_to:
|
112
|
-
warnings.warn(
|
113
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
114
|
-
)
|
115
|
-
self.delegate_to = delegate_to
|
116
106
|
self.project_id = project_id
|
117
107
|
self.subscription = subscription
|
118
108
|
self.max_messages = max_messages
|
@@ -130,7 +120,6 @@ class PubSubPullSensor(BaseSensorOperator):
|
|
130
120
|
def poke(self, context: Context) -> bool:
|
131
121
|
hook = PubSubHook(
|
132
122
|
gcp_conn_id=self.gcp_conn_id,
|
133
|
-
delegate_to=self.delegate_to,
|
134
123
|
impersonation_chain=self.impersonation_chain,
|
135
124
|
)
|
136
125
|
|
@@ -22,7 +22,6 @@ Google Cloud Storage operator.
|
|
22
22
|
from __future__ import annotations
|
23
23
|
|
24
24
|
import os
|
25
|
-
import warnings
|
26
25
|
from tempfile import NamedTemporaryFile
|
27
26
|
from typing import TYPE_CHECKING, Sequence
|
28
27
|
|
@@ -46,9 +45,6 @@ class ADLSToGCSOperator(ADLSListOperator):
|
|
46
45
|
:param azure_data_lake_conn_id: The connection ID to use when
|
47
46
|
connecting to Azure Data Lake Storage.
|
48
47
|
:param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud.
|
49
|
-
:param delegate_to: Google account to impersonate using domain-wide delegation of authority,
|
50
|
-
if any. For this to work, the service account making the request must have
|
51
|
-
domain-wide delegation enabled.
|
52
48
|
:param google_impersonation_chain: Optional Google service account to impersonate using
|
53
49
|
short-term credentials, or chained list of accounts required to get the access_token
|
54
50
|
of the last account in the list, which will be impersonated in the request.
|
@@ -111,7 +107,6 @@ class ADLSToGCSOperator(ADLSListOperator):
|
|
111
107
|
dest_gcs: str,
|
112
108
|
azure_data_lake_conn_id: str,
|
113
109
|
gcp_conn_id: str = "google_cloud_default",
|
114
|
-
delegate_to: str | None = None,
|
115
110
|
replace: bool = False,
|
116
111
|
gzip: bool = False,
|
117
112
|
google_impersonation_chain: str | Sequence[str] | None = None,
|
@@ -124,12 +119,6 @@ class ADLSToGCSOperator(ADLSListOperator):
|
|
124
119
|
self.dest_gcs = dest_gcs
|
125
120
|
self.replace = replace
|
126
121
|
self.gcp_conn_id = gcp_conn_id
|
127
|
-
if delegate_to:
|
128
|
-
warnings.warn(
|
129
|
-
"'delegate_to' parameter is deprecated, please use 'google_impersonation_chain'",
|
130
|
-
DeprecationWarning,
|
131
|
-
)
|
132
|
-
self.delegate_to = delegate_to
|
133
122
|
self.gzip = gzip
|
134
123
|
self.google_impersonation_chain = google_impersonation_chain
|
135
124
|
|
@@ -138,7 +127,6 @@ class ADLSToGCSOperator(ADLSListOperator):
|
|
138
127
|
files = super().execute(context)
|
139
128
|
g_hook = GCSHook(
|
140
129
|
gcp_conn_id=self.gcp_conn_id,
|
141
|
-
delegate_to=self.delegate_to,
|
142
130
|
impersonation_chain=self.google_impersonation_chain,
|
143
131
|
)
|
144
132
|
|
@@ -17,7 +17,6 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
import warnings
|
21
20
|
from tempfile import NamedTemporaryFile
|
22
21
|
from typing import TYPE_CHECKING, Sequence
|
23
22
|
|
@@ -44,9 +43,6 @@ class AzureFileShareToGCSOperator(BaseOperator):
|
|
44
43
|
:param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud.
|
45
44
|
:param dest_gcs: The destination Google Cloud Storage bucket and prefix
|
46
45
|
where you want to store the files. (templated)
|
47
|
-
:param delegate_to: Google account to impersonate using domain-wide delegation of authority,
|
48
|
-
if any. For this to work, the service account making the request must have
|
49
|
-
domain-wide delegation enabled.
|
50
46
|
:param replace: Whether you want to replace existing destination files
|
51
47
|
or not.
|
52
48
|
:param gzip: Option to compress file for upload
|
@@ -79,7 +75,6 @@ class AzureFileShareToGCSOperator(BaseOperator):
|
|
79
75
|
prefix: str = "",
|
80
76
|
azure_fileshare_conn_id: str = "azure_fileshare_default",
|
81
77
|
gcp_conn_id: str = "google_cloud_default",
|
82
|
-
delegate_to: str | None = None,
|
83
78
|
replace: bool = False,
|
84
79
|
gzip: bool = False,
|
85
80
|
google_impersonation_chain: str | Sequence[str] | None = None,
|
@@ -93,12 +88,6 @@ class AzureFileShareToGCSOperator(BaseOperator):
|
|
93
88
|
self.azure_fileshare_conn_id = azure_fileshare_conn_id
|
94
89
|
self.gcp_conn_id = gcp_conn_id
|
95
90
|
self.dest_gcs = dest_gcs
|
96
|
-
if delegate_to:
|
97
|
-
warnings.warn(
|
98
|
-
"'delegate_to' parameter is deprecated, please use 'google_impersonation_chain'",
|
99
|
-
DeprecationWarning,
|
100
|
-
)
|
101
|
-
self.delegate_to = delegate_to
|
102
91
|
self.replace = replace
|
103
92
|
self.gzip = gzip
|
104
93
|
self.google_impersonation_chain = google_impersonation_chain
|
@@ -123,7 +112,6 @@ class AzureFileShareToGCSOperator(BaseOperator):
|
|
123
112
|
|
124
113
|
gcs_hook = GCSHook(
|
125
114
|
gcp_conn_id=self.gcp_conn_id,
|
126
|
-
delegate_to=self.delegate_to,
|
127
115
|
impersonation_chain=self.google_impersonation_chain,
|
128
116
|
)
|
129
117
|
|
@@ -50,9 +50,6 @@ class BigQueryToBigQueryOperator(BaseOperator):
|
|
50
50
|
:param write_disposition: The write disposition if the table already exists.
|
51
51
|
:param create_disposition: The create disposition if the table doesn't exist.
|
52
52
|
:param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud.
|
53
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
54
|
-
if any. For this to work, the service account making the request must have
|
55
|
-
domain-wide delegation enabled.
|
56
53
|
:param labels: a dictionary containing labels for the job/query,
|
57
54
|
passed to BigQuery
|
58
55
|
:param encryption_configuration: [Optional] Custom encryption configuration (e.g., Cloud KMS keys).
|
@@ -94,7 +91,6 @@ class BigQueryToBigQueryOperator(BaseOperator):
|
|
94
91
|
write_disposition: str = "WRITE_EMPTY",
|
95
92
|
create_disposition: str = "CREATE_IF_NEEDED",
|
96
93
|
gcp_conn_id: str = "google_cloud_default",
|
97
|
-
delegate_to: str | None = None,
|
98
94
|
labels: dict | None = None,
|
99
95
|
encryption_configuration: dict | None = None,
|
100
96
|
location: str | None = None,
|
@@ -108,11 +104,6 @@ class BigQueryToBigQueryOperator(BaseOperator):
|
|
108
104
|
self.write_disposition = write_disposition
|
109
105
|
self.create_disposition = create_disposition
|
110
106
|
self.gcp_conn_id = gcp_conn_id
|
111
|
-
if delegate_to:
|
112
|
-
warnings.warn(
|
113
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
114
|
-
)
|
115
|
-
self.delegate_to = delegate_to
|
116
107
|
self.labels = labels
|
117
108
|
self.encryption_configuration = encryption_configuration
|
118
109
|
self.location = location
|
@@ -126,7 +117,6 @@ class BigQueryToBigQueryOperator(BaseOperator):
|
|
126
117
|
)
|
127
118
|
hook = BigQueryHook(
|
128
119
|
gcp_conn_id=self.gcp_conn_id,
|
129
|
-
delegate_to=self.delegate_to,
|
130
120
|
location=self.location,
|
131
121
|
impersonation_chain=self.impersonation_chain,
|
132
122
|
)
|
@@ -18,7 +18,6 @@
|
|
18
18
|
"""This module contains Google BigQuery to Google Cloud Storage operator."""
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
|
-
import warnings
|
22
21
|
from typing import TYPE_CHECKING, Any, Sequence
|
23
22
|
|
24
23
|
from google.api_core.exceptions import Conflict
|
@@ -60,9 +59,6 @@ class BigQueryToGCSOperator(BaseOperator):
|
|
60
59
|
:param field_delimiter: The delimiter to use when extracting to a CSV.
|
61
60
|
:param print_header: Whether to print a header for a CSV file extract.
|
62
61
|
:param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud.
|
63
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
64
|
-
if any. For this to work, the service account making the request must have
|
65
|
-
domain-wide delegation enabled.
|
66
62
|
:param labels: a dictionary containing labels for the job/query,
|
67
63
|
passed to BigQuery
|
68
64
|
:param location: The location used for the operation.
|
@@ -110,7 +106,6 @@ class BigQueryToGCSOperator(BaseOperator):
|
|
110
106
|
field_delimiter: str = ",",
|
111
107
|
print_header: bool = True,
|
112
108
|
gcp_conn_id: str = "google_cloud_default",
|
113
|
-
delegate_to: str | None = None,
|
114
109
|
labels: dict | None = None,
|
115
110
|
location: str | None = None,
|
116
111
|
impersonation_chain: str | Sequence[str] | None = None,
|
@@ -131,11 +126,6 @@ class BigQueryToGCSOperator(BaseOperator):
|
|
131
126
|
self.field_delimiter = field_delimiter
|
132
127
|
self.print_header = print_header
|
133
128
|
self.gcp_conn_id = gcp_conn_id
|
134
|
-
if delegate_to:
|
135
|
-
warnings.warn(
|
136
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
137
|
-
)
|
138
|
-
self.delegate_to = delegate_to
|
139
129
|
self.labels = labels
|
140
130
|
self.location = location
|
141
131
|
self.impersonation_chain = impersonation_chain
|
@@ -209,7 +199,6 @@ class BigQueryToGCSOperator(BaseOperator):
|
|
209
199
|
)
|
210
200
|
hook = BigQueryHook(
|
211
201
|
gcp_conn_id=self.gcp_conn_id,
|
212
|
-
delegate_to=self.delegate_to,
|
213
202
|
location=self.location,
|
214
203
|
impersonation_chain=self.impersonation_chain,
|
215
204
|
)
|
@@ -18,7 +18,6 @@
|
|
18
18
|
"""This module contains Google BigQuery to MSSQL operator."""
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
|
-
import warnings
|
22
21
|
from typing import TYPE_CHECKING, Sequence
|
23
22
|
|
24
23
|
from airflow.models import BaseOperator
|
@@ -63,9 +62,6 @@ class BigQueryToMsSqlOperator(BaseOperator):
|
|
63
62
|
:param selected_fields: List of fields to return (comma-separated). If
|
64
63
|
unspecified, all fields are returned.
|
65
64
|
:param gcp_conn_id: reference to a specific Google Cloud hook.
|
66
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
67
|
-
if any. For this to work, the service account making the request must have
|
68
|
-
domain-wide delegation enabled.
|
69
65
|
:param mssql_conn_id: reference to a specific mssql hook
|
70
66
|
:param database: name of database which overwrite defined one in connection
|
71
67
|
:param replace: Whether to replace instead of insert
|
@@ -93,7 +89,6 @@ class BigQueryToMsSqlOperator(BaseOperator):
|
|
93
89
|
gcp_conn_id: str = "google_cloud_default",
|
94
90
|
mssql_conn_id: str = "mssql_default",
|
95
91
|
database: str | None = None,
|
96
|
-
delegate_to: str | None = None,
|
97
92
|
replace: bool = False,
|
98
93
|
batch_size: int = 1000,
|
99
94
|
location: str | None = None,
|
@@ -107,11 +102,6 @@ class BigQueryToMsSqlOperator(BaseOperator):
|
|
107
102
|
self.database = database
|
108
103
|
self.mssql_table = mssql_table
|
109
104
|
self.replace = replace
|
110
|
-
if delegate_to:
|
111
|
-
warnings.warn(
|
112
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
113
|
-
)
|
114
|
-
self.delegate_to = delegate_to
|
115
105
|
self.batch_size = batch_size
|
116
106
|
self.location = location
|
117
107
|
self.impersonation_chain = impersonation_chain
|
@@ -126,7 +116,6 @@ class BigQueryToMsSqlOperator(BaseOperator):
|
|
126
116
|
def execute(self, context: Context) -> None:
|
127
117
|
big_query_hook = BigQueryHook(
|
128
118
|
gcp_conn_id=self.gcp_conn_id,
|
129
|
-
delegate_to=self.delegate_to,
|
130
119
|
location=self.location,
|
131
120
|
impersonation_chain=self.impersonation_chain,
|
132
121
|
)
|
@@ -18,7 +18,6 @@
|
|
18
18
|
"""This module contains Google BigQuery to MySQL operator."""
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
|
-
import warnings
|
22
21
|
from typing import TYPE_CHECKING, Sequence
|
23
22
|
|
24
23
|
from airflow.models import BaseOperator
|
@@ -63,9 +62,6 @@ class BigQueryToMySqlOperator(BaseOperator):
|
|
63
62
|
:param selected_fields: List of fields to return (comma-separated). If
|
64
63
|
unspecified, all fields are returned.
|
65
64
|
:param gcp_conn_id: reference to a specific Google Cloud hook.
|
66
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
67
|
-
if any. For this to work, the service account making the request must have
|
68
|
-
domain-wide delegation enabled.
|
69
65
|
:param mysql_conn_id: Reference to :ref:`mysql connection id <howto/connection:mysql>`.
|
70
66
|
:param database: name of database which overwrite defined one in connection
|
71
67
|
:param replace: Whether to replace instead of insert
|
@@ -97,7 +93,6 @@ class BigQueryToMySqlOperator(BaseOperator):
|
|
97
93
|
gcp_conn_id: str = "google_cloud_default",
|
98
94
|
mysql_conn_id: str = "mysql_default",
|
99
95
|
database: str | None = None,
|
100
|
-
delegate_to: str | None = None,
|
101
96
|
replace: bool = False,
|
102
97
|
batch_size: int = 1000,
|
103
98
|
location: str | None = None,
|
@@ -111,11 +106,6 @@ class BigQueryToMySqlOperator(BaseOperator):
|
|
111
106
|
self.database = database
|
112
107
|
self.mysql_table = mysql_table
|
113
108
|
self.replace = replace
|
114
|
-
if delegate_to:
|
115
|
-
warnings.warn(
|
116
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
117
|
-
)
|
118
|
-
self.delegate_to = delegate_to
|
119
109
|
self.batch_size = batch_size
|
120
110
|
self.location = location
|
121
111
|
self.impersonation_chain = impersonation_chain
|
@@ -127,7 +117,6 @@ class BigQueryToMySqlOperator(BaseOperator):
|
|
127
117
|
def execute(self, context: Context) -> None:
|
128
118
|
big_query_hook = BigQueryHook(
|
129
119
|
gcp_conn_id=self.gcp_conn_id,
|
130
|
-
delegate_to=self.delegate_to,
|
131
120
|
location=self.location,
|
132
121
|
impersonation_chain=self.impersonation_chain,
|
133
122
|
)
|
@@ -17,7 +17,6 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import json
|
20
|
-
import warnings
|
21
20
|
from datetime import datetime
|
22
21
|
from tempfile import NamedTemporaryFile
|
23
22
|
from typing import Any, Sequence
|
@@ -65,9 +64,6 @@ class GoogleCalendarToGCSOperator(BaseOperator):
|
|
65
64
|
:param destination_path: The Google Cloud Storage URI array for the object created by the operator.
|
66
65
|
For example: ``path/to/my/files``.
|
67
66
|
:param gcp_conn_id: The connection ID to use when fetching connection info.
|
68
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
69
|
-
if any. For this to work, the service account making the request must have
|
70
|
-
domain-wide delegation enabled.
|
71
67
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
72
68
|
credentials, or chained list of accounts required to get the access_token
|
73
69
|
of the last account in the list, which will be impersonated in the request.
|
@@ -108,7 +104,6 @@ class GoogleCalendarToGCSOperator(BaseOperator):
|
|
108
104
|
updated_min: datetime | None = None,
|
109
105
|
destination_path: str | None = None,
|
110
106
|
gcp_conn_id: str = "google_cloud_default",
|
111
|
-
delegate_to: str | None = None,
|
112
107
|
impersonation_chain: str | Sequence[str] | None = None,
|
113
108
|
**kwargs,
|
114
109
|
) -> None:
|
@@ -133,11 +128,6 @@ class GoogleCalendarToGCSOperator(BaseOperator):
|
|
133
128
|
self.updated_min = updated_min
|
134
129
|
self.destination_bucket = destination_bucket
|
135
130
|
self.destination_path = destination_path
|
136
|
-
if delegate_to:
|
137
|
-
warnings.warn(
|
138
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
139
|
-
)
|
140
|
-
self.delegate_to = delegate_to
|
141
131
|
self.impersonation_chain = impersonation_chain
|
142
132
|
|
143
133
|
def _upload_data(
|
@@ -146,7 +136,6 @@ class GoogleCalendarToGCSOperator(BaseOperator):
|
|
146
136
|
) -> str:
|
147
137
|
gcs_hook = GCSHook(
|
148
138
|
gcp_conn_id=self.gcp_conn_id,
|
149
|
-
delegate_to=self.delegate_to,
|
150
139
|
impersonation_chain=self.impersonation_chain,
|
151
140
|
)
|
152
141
|
|
@@ -173,7 +162,6 @@ class GoogleCalendarToGCSOperator(BaseOperator):
|
|
173
162
|
calendar_hook = GoogleCalendarHook(
|
174
163
|
api_version=self.api_version,
|
175
164
|
gcp_conn_id=self.gcp_conn_id,
|
176
|
-
delegate_to=self.delegate_to,
|
177
165
|
impersonation_chain=self.impersonation_chain,
|
178
166
|
)
|
179
167
|
|
@@ -22,7 +22,6 @@ data from Cassandra to Google Cloud Storage in JSON format.
|
|
22
22
|
from __future__ import annotations
|
23
23
|
|
24
24
|
import json
|
25
|
-
import warnings
|
26
25
|
from base64 import b64encode
|
27
26
|
from datetime import datetime
|
28
27
|
from decimal import Decimal
|
@@ -67,9 +66,6 @@ class CassandraToGCSOperator(BaseOperator):
|
|
67
66
|
:param cassandra_conn_id: Reference to a specific Cassandra hook.
|
68
67
|
:param gzip: Option to compress file for upload
|
69
68
|
:param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud.
|
70
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
71
|
-
if any. For this to work, the service account making the request must have
|
72
|
-
domain-wide delegation enabled.
|
73
69
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
74
70
|
credentials, or chained list of accounts required to get the access_token
|
75
71
|
of the last account in the list, which will be impersonated in the request.
|
@@ -106,7 +102,6 @@ class CassandraToGCSOperator(BaseOperator):
|
|
106
102
|
gzip: bool = False,
|
107
103
|
cassandra_conn_id: str = "cassandra_default",
|
108
104
|
gcp_conn_id: str = "google_cloud_default",
|
109
|
-
delegate_to: str | None = None,
|
110
105
|
impersonation_chain: str | Sequence[str] | None = None,
|
111
106
|
query_timeout: float | None | NotSetType = NOT_SET,
|
112
107
|
encode_uuid: bool = True,
|
@@ -121,11 +116,6 @@ class CassandraToGCSOperator(BaseOperator):
|
|
121
116
|
self.approx_max_file_size_bytes = approx_max_file_size_bytes
|
122
117
|
self.cassandra_conn_id = cassandra_conn_id
|
123
118
|
self.gcp_conn_id = gcp_conn_id
|
124
|
-
if delegate_to:
|
125
|
-
warnings.warn(
|
126
|
-
"'delegate_to' parameter is deprecated, please use 'mpersonation_chain'", DeprecationWarning
|
127
|
-
)
|
128
|
-
self.delegate_to = delegate_to
|
129
119
|
self.gzip = gzip
|
130
120
|
self.impersonation_chain = impersonation_chain
|
131
121
|
self.query_timeout = query_timeout
|
@@ -255,7 +245,6 @@ class CassandraToGCSOperator(BaseOperator):
|
|
255
245
|
"""Upload a file (data split or schema .json file) to Google Cloud Storage."""
|
256
246
|
hook = GCSHook(
|
257
247
|
gcp_conn_id=self.gcp_conn_id,
|
258
|
-
delegate_to=self.delegate_to,
|
259
248
|
impersonation_chain=self.impersonation_chain,
|
260
249
|
)
|
261
250
|
hook.upload(
|