apache-airflow-providers-google 9.0.0rc2__py3-none-any.whl → 10.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/cloud/hooks/automl.py +5 -6
- airflow/providers/google/cloud/hooks/bigquery.py +5 -6
- airflow/providers/google/cloud/hooks/bigquery_dts.py +11 -8
- airflow/providers/google/cloud/hooks/bigtable.py +5 -6
- airflow/providers/google/cloud/hooks/cloud_build.py +14 -11
- airflow/providers/google/cloud/hooks/cloud_composer.py +16 -0
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +5 -14
- airflow/providers/google/cloud/hooks/cloud_sql.py +5 -7
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +5 -5
- airflow/providers/google/cloud/hooks/compute.py +5 -6
- airflow/providers/google/cloud/hooks/compute_ssh.py +8 -12
- airflow/providers/google/cloud/hooks/datacatalog.py +5 -9
- airflow/providers/google/cloud/hooks/dataflow.py +14 -6
- airflow/providers/google/cloud/hooks/dataform.py +8 -0
- airflow/providers/google/cloud/hooks/datafusion.py +13 -6
- airflow/providers/google/cloud/hooks/dataplex.py +5 -8
- airflow/providers/google/cloud/hooks/dataproc.py +13 -8
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +8 -0
- airflow/providers/google/cloud/hooks/datastore.py +5 -6
- airflow/providers/google/cloud/hooks/dlp.py +5 -9
- airflow/providers/google/cloud/hooks/functions.py +5 -6
- airflow/providers/google/cloud/hooks/gcs.py +6 -6
- airflow/providers/google/cloud/hooks/gdm.py +5 -6
- airflow/providers/google/cloud/hooks/kms.py +5 -9
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +6 -4
- airflow/providers/google/cloud/hooks/life_sciences.py +5 -9
- airflow/providers/google/cloud/hooks/mlengine.py +8 -0
- airflow/providers/google/cloud/hooks/natural_language.py +5 -9
- airflow/providers/google/cloud/hooks/os_login.py +5 -6
- airflow/providers/google/cloud/hooks/pubsub.py +5 -5
- airflow/providers/google/cloud/hooks/secret_manager.py +5 -9
- airflow/providers/google/cloud/hooks/spanner.py +5 -6
- airflow/providers/google/cloud/hooks/speech_to_text.py +5 -9
- airflow/providers/google/cloud/hooks/stackdriver.py +5 -6
- airflow/providers/google/cloud/hooks/tasks.py +5 -9
- airflow/providers/google/cloud/hooks/text_to_speech.py +5 -9
- airflow/providers/google/cloud/hooks/translate.py +5 -6
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +5 -5
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +5 -6
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +5 -6
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +8 -0
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +8 -0
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +5 -6
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +8 -0
- airflow/providers/google/cloud/hooks/video_intelligence.py +5 -9
- airflow/providers/google/cloud/hooks/vision.py +5 -6
- airflow/providers/google/cloud/hooks/workflows.py +8 -0
- airflow/providers/google/cloud/operators/bigquery.py +1 -148
- airflow/providers/google/cloud/operators/cloud_build.py +0 -13
- airflow/providers/google/cloud/operators/cloud_composer.py +2 -68
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +0 -21
- airflow/providers/google/cloud/operators/dataflow.py +2 -72
- airflow/providers/google/cloud/operators/dataform.py +5 -141
- airflow/providers/google/cloud/operators/datafusion.py +0 -102
- airflow/providers/google/cloud/operators/dataplex.py +3 -61
- airflow/providers/google/cloud/operators/dataproc.py +5 -13
- airflow/providers/google/cloud/operators/datastore.py +2 -86
- airflow/providers/google/cloud/operators/gcs.py +1 -43
- airflow/providers/google/cloud/operators/mlengine.py +1 -123
- airflow/providers/google/cloud/operators/pubsub.py +0 -61
- airflow/providers/google/cloud/operators/stackdriver.py +0 -101
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +0 -26
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +0 -41
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +0 -39
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +0 -71
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +0 -71
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +0 -35
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +0 -41
- airflow/providers/google/cloud/sensors/bigquery.py +0 -24
- airflow/providers/google/cloud/sensors/cloud_composer.py +0 -10
- airflow/providers/google/cloud/sensors/dataflow.py +0 -42
- airflow/providers/google/cloud/sensors/dataform.py +0 -12
- airflow/providers/google/cloud/sensors/datafusion.py +0 -11
- airflow/providers/google/cloud/sensors/dataplex.py +0 -10
- airflow/providers/google/cloud/sensors/gcs.py +0 -44
- airflow/providers/google/cloud/sensors/pubsub.py +0 -11
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +0 -12
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +0 -12
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +0 -10
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +0 -11
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +0 -11
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +0 -11
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +0 -12
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +0 -11
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +0 -13
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +0 -11
- airflow/providers/google/cloud/transfers/gcs_to_local.py +0 -11
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +0 -11
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +0 -12
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +0 -11
- airflow/providers/google/cloud/transfers/local_to_gcs.py +0 -9
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +0 -11
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +0 -11
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +0 -12
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +0 -11
- airflow/providers/google/cloud/triggers/bigquery_dts.py +0 -12
- airflow/providers/google/cloud/triggers/cloud_build.py +0 -11
- airflow/providers/google/cloud/triggers/cloud_composer.py +0 -9
- airflow/providers/google/cloud/triggers/dataflow.py +0 -7
- airflow/providers/google/cloud/triggers/datafusion.py +0 -11
- airflow/providers/google/cloud/triggers/dataproc.py +2 -8
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +0 -4
- airflow/providers/google/cloud/triggers/mlengine.py +0 -3
- airflow/providers/google/common/hooks/discovery_api.py +0 -7
- airflow/providers/google/firebase/hooks/firestore.py +0 -10
- airflow/providers/google/get_provider_info.py +1 -0
- airflow/providers/google/marketing_platform/example_dags/example_display_video.py +0 -64
- airflow/providers/google/marketing_platform/hooks/campaign_manager.py +3 -3
- airflow/providers/google/marketing_platform/hooks/display_video.py +7 -38
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +22 -22
- airflow/providers/google/marketing_platform/operators/display_video.py +0 -284
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +3 -3
- airflow/providers/google/marketing_platform/sensors/display_video.py +0 -65
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +0 -4
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +0 -4
- {apache_airflow_providers_google-9.0.0rc2.dist-info → apache_airflow_providers_google-10.0.0rc1.dist-info}/METADATA +33 -4
- {apache_airflow_providers_google-9.0.0rc2.dist-info → apache_airflow_providers_google-10.0.0rc1.dist-info}/RECORD +122 -122
- {apache_airflow_providers_google-9.0.0rc2.dist-info → apache_airflow_providers_google-10.0.0rc1.dist-info}/LICENSE +0 -0
- {apache_airflow_providers_google-9.0.0rc2.dist-info → apache_airflow_providers_google-10.0.0rc1.dist-info}/NOTICE +0 -0
- {apache_airflow_providers_google-9.0.0rc2.dist-info → apache_airflow_providers_google-10.0.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-9.0.0rc2.dist-info → apache_airflow_providers_google-10.0.0rc1.dist-info}/entry_points.txt +0 -0
- {apache_airflow_providers_google-9.0.0rc2.dist-info → apache_airflow_providers_google-10.0.0rc1.dist-info}/top_level.txt +0 -0
@@ -19,7 +19,6 @@
|
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
21
|
import json
|
22
|
-
import warnings
|
23
22
|
from typing import TYPE_CHECKING, Any, Sequence
|
24
23
|
|
25
24
|
from google.api_core.exceptions import BadRequest, Conflict
|
@@ -128,9 +127,6 @@ class GCSToBigQueryOperator(BaseOperator):
|
|
128
127
|
execute() command, which in turn gets stored in XCom for future
|
129
128
|
operators to use. This can be helpful with incremental loads--during
|
130
129
|
future executions, you can pick up from the max ID.
|
131
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
132
|
-
if any. For this to work, the service account making the request must have
|
133
|
-
domain-wide delegation enabled.
|
134
130
|
:param schema_update_options: Allows the schema of the destination
|
135
131
|
table to be updated as a side effect of the load job.
|
136
132
|
:param src_fmt_configs: configure optional fields specific to the source format
|
@@ -208,7 +204,6 @@ class GCSToBigQueryOperator(BaseOperator):
|
|
208
204
|
encoding="UTF-8",
|
209
205
|
max_id_key=None,
|
210
206
|
gcp_conn_id="google_cloud_default",
|
211
|
-
delegate_to=None,
|
212
207
|
schema_update_options=(),
|
213
208
|
src_fmt_configs=None,
|
214
209
|
external_table=False,
|
@@ -274,11 +269,6 @@ class GCSToBigQueryOperator(BaseOperator):
|
|
274
269
|
|
275
270
|
self.max_id_key = max_id_key
|
276
271
|
self.gcp_conn_id = gcp_conn_id
|
277
|
-
if delegate_to:
|
278
|
-
warnings.warn(
|
279
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
280
|
-
)
|
281
|
-
self.delegate_to = delegate_to
|
282
272
|
|
283
273
|
self.schema_update_options = schema_update_options
|
284
274
|
self.src_fmt_configs = src_fmt_configs
|
@@ -324,7 +314,6 @@ class GCSToBigQueryOperator(BaseOperator):
|
|
324
314
|
def execute(self, context: Context):
|
325
315
|
hook = BigQueryHook(
|
326
316
|
gcp_conn_id=self.gcp_conn_id,
|
327
|
-
delegate_to=self.delegate_to,
|
328
317
|
location=self.location,
|
329
318
|
impersonation_chain=self.impersonation_chain,
|
330
319
|
)
|
@@ -356,7 +345,6 @@ class GCSToBigQueryOperator(BaseOperator):
|
|
356
345
|
if self.schema_object and self.source_format != "DATASTORE_BACKUP":
|
357
346
|
gcs_hook = GCSHook(
|
358
347
|
gcp_conn_id=self.gcp_conn_id,
|
359
|
-
delegate_to=self.delegate_to,
|
360
348
|
impersonation_chain=self.impersonation_chain,
|
361
349
|
)
|
362
350
|
self.schema_fields = json.loads(
|
@@ -467,7 +455,6 @@ class GCSToBigQueryOperator(BaseOperator):
|
|
467
455
|
def _find_max_value_in_column(self):
|
468
456
|
hook = BigQueryHook(
|
469
457
|
gcp_conn_id=self.gcp_conn_id,
|
470
|
-
delegate_to=self.delegate_to,
|
471
458
|
location=self.location,
|
472
459
|
impersonation_chain=self.impersonation_chain,
|
473
460
|
)
|
@@ -18,7 +18,6 @@
|
|
18
18
|
"""This module contains a Google Cloud Storage operator."""
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
|
-
import warnings
|
22
21
|
from typing import TYPE_CHECKING, Sequence
|
23
22
|
|
24
23
|
from airflow.exceptions import AirflowException
|
@@ -71,9 +70,6 @@ class GCSToGCSOperator(BaseOperator):
|
|
71
70
|
If source_objects = ['foo/bah/'] and delimiter = '.avro', then only the 'files' in the
|
72
71
|
folder 'foo/bah/' with '.avro' delimiter will be copied to the destination object.
|
73
72
|
:param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud.
|
74
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
75
|
-
if any. For this to work, the service account making the request must have
|
76
|
-
domain-wide delegation enabled.
|
77
73
|
:param last_modified_time: When specified, the objects will be copied or moved,
|
78
74
|
only if they were modified after last_modified_time.
|
79
75
|
If tzinfo has not been set, UTC will be assumed.
|
@@ -188,7 +184,6 @@ class GCSToGCSOperator(BaseOperator):
|
|
188
184
|
move_object=False,
|
189
185
|
replace=True,
|
190
186
|
gcp_conn_id="google_cloud_default",
|
191
|
-
delegate_to=None,
|
192
187
|
last_modified_time=None,
|
193
188
|
maximum_modified_time=None,
|
194
189
|
is_older_than=None,
|
@@ -208,11 +203,6 @@ class GCSToGCSOperator(BaseOperator):
|
|
208
203
|
self.move_object = move_object
|
209
204
|
self.replace = replace
|
210
205
|
self.gcp_conn_id = gcp_conn_id
|
211
|
-
if delegate_to:
|
212
|
-
warnings.warn(
|
213
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
214
|
-
)
|
215
|
-
self.delegate_to = delegate_to
|
216
206
|
self.last_modified_time = last_modified_time
|
217
207
|
self.maximum_modified_time = maximum_modified_time
|
218
208
|
self.is_older_than = is_older_than
|
@@ -224,7 +214,6 @@ class GCSToGCSOperator(BaseOperator):
|
|
224
214
|
|
225
215
|
hook = GCSHook(
|
226
216
|
gcp_conn_id=self.gcp_conn_id,
|
227
|
-
delegate_to=self.delegate_to,
|
228
217
|
impersonation_chain=self.impersonation_chain,
|
229
218
|
)
|
230
219
|
if self.source_objects and self.source_object:
|
@@ -16,7 +16,6 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
import warnings
|
20
19
|
from typing import TYPE_CHECKING, Sequence
|
21
20
|
|
22
21
|
from airflow.exceptions import AirflowException
|
@@ -52,9 +51,6 @@ class GCSToLocalFilesystemOperator(BaseOperator):
|
|
52
51
|
the contents of the downloaded file to XCom with the key set in this
|
53
52
|
parameter. If not set, the downloaded data will not be pushed to XCom. (templated)
|
54
53
|
:param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud.
|
55
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
56
|
-
if any. For this to work, the service account making the request must have
|
57
|
-
domain-wide delegation enabled.
|
58
54
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
59
55
|
credentials, or chained list of accounts required to get the access_token
|
60
56
|
of the last account in the list, which will be impersonated in the request.
|
@@ -85,7 +81,6 @@ class GCSToLocalFilesystemOperator(BaseOperator):
|
|
85
81
|
filename: str | None = None,
|
86
82
|
store_to_xcom_key: str | None = None,
|
87
83
|
gcp_conn_id: str = "google_cloud_default",
|
88
|
-
delegate_to: str | None = None,
|
89
84
|
impersonation_chain: str | Sequence[str] | None = None,
|
90
85
|
file_encoding: str = "utf-8",
|
91
86
|
**kwargs,
|
@@ -109,11 +104,6 @@ class GCSToLocalFilesystemOperator(BaseOperator):
|
|
109
104
|
self.object_name = object_name
|
110
105
|
self.store_to_xcom_key = store_to_xcom_key
|
111
106
|
self.gcp_conn_id = gcp_conn_id
|
112
|
-
if delegate_to:
|
113
|
-
warnings.warn(
|
114
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
115
|
-
)
|
116
|
-
self.delegate_to = delegate_to
|
117
107
|
self.impersonation_chain = impersonation_chain
|
118
108
|
self.file_encoding = file_encoding
|
119
109
|
|
@@ -121,7 +111,6 @@ class GCSToLocalFilesystemOperator(BaseOperator):
|
|
121
111
|
self.log.info("Executing download: %s, %s, %s", self.bucket, self.object_name, self.filename)
|
122
112
|
hook = GCSHook(
|
123
113
|
gcp_conn_id=self.gcp_conn_id,
|
124
|
-
delegate_to=self.delegate_to,
|
125
114
|
impersonation_chain=self.impersonation_chain,
|
126
115
|
)
|
127
116
|
|
@@ -19,7 +19,6 @@
|
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
21
|
import os
|
22
|
-
import warnings
|
23
22
|
from tempfile import NamedTemporaryFile
|
24
23
|
from typing import TYPE_CHECKING, Sequence
|
25
24
|
|
@@ -85,9 +84,6 @@ class GCSToSFTPOperator(BaseOperator):
|
|
85
84
|
:param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud.
|
86
85
|
:param sftp_conn_id: The sftp connection id. The name or identifier for
|
87
86
|
establishing a connection to the SFTP server.
|
88
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
89
|
-
if any. For this to work, the service account making the request must have
|
90
|
-
domain-wide delegation enabled.
|
91
87
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
92
88
|
credentials, or chained list of accounts required to get the access_token
|
93
89
|
of the last account in the list, which will be impersonated in the request.
|
@@ -116,7 +112,6 @@ class GCSToSFTPOperator(BaseOperator):
|
|
116
112
|
move_object: bool = False,
|
117
113
|
gcp_conn_id: str = "google_cloud_default",
|
118
114
|
sftp_conn_id: str = "ssh_default",
|
119
|
-
delegate_to: str | None = None,
|
120
115
|
impersonation_chain: str | Sequence[str] | None = None,
|
121
116
|
**kwargs,
|
122
117
|
) -> None:
|
@@ -129,18 +124,12 @@ class GCSToSFTPOperator(BaseOperator):
|
|
129
124
|
self.move_object = move_object
|
130
125
|
self.gcp_conn_id = gcp_conn_id
|
131
126
|
self.sftp_conn_id = sftp_conn_id
|
132
|
-
if delegate_to:
|
133
|
-
warnings.warn(
|
134
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
135
|
-
)
|
136
|
-
self.delegate_to = delegate_to
|
137
127
|
self.impersonation_chain = impersonation_chain
|
138
128
|
self.sftp_dirs = None
|
139
129
|
|
140
130
|
def execute(self, context: Context):
|
141
131
|
gcs_hook = GCSHook(
|
142
132
|
gcp_conn_id=self.gcp_conn_id,
|
143
|
-
delegate_to=self.delegate_to,
|
144
133
|
impersonation_chain=self.impersonation_chain,
|
145
134
|
)
|
146
135
|
|
@@ -16,7 +16,6 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
import warnings
|
20
19
|
from typing import TYPE_CHECKING, Sequence
|
21
20
|
|
22
21
|
from airflow.models import BaseOperator
|
@@ -43,9 +42,6 @@ class GoogleDriveToGCSOperator(BaseOperator):
|
|
43
42
|
:param file_name: The name of the file residing in Google Drive
|
44
43
|
:param drive_id: Optional. The id of the shared Google Drive in which the file resides.
|
45
44
|
:param gcp_conn_id: The GCP connection ID to use when fetching connection info.
|
46
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
47
|
-
if any. For this to work, the service account making the request must have
|
48
|
-
domain-wide delegation enabled.
|
49
45
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
50
46
|
credentials, or chained list of accounts required to get the access_token
|
51
47
|
of the last account in the list, which will be impersonated in the request.
|
@@ -74,7 +70,6 @@ class GoogleDriveToGCSOperator(BaseOperator):
|
|
74
70
|
folder_id: str,
|
75
71
|
drive_id: str | None = None,
|
76
72
|
gcp_conn_id: str = "google_cloud_default",
|
77
|
-
delegate_to: str | None = None,
|
78
73
|
impersonation_chain: str | Sequence[str] | None = None,
|
79
74
|
**kwargs,
|
80
75
|
) -> None:
|
@@ -85,22 +80,15 @@ class GoogleDriveToGCSOperator(BaseOperator):
|
|
85
80
|
self.drive_id = drive_id
|
86
81
|
self.file_name = file_name
|
87
82
|
self.gcp_conn_id = gcp_conn_id
|
88
|
-
if delegate_to:
|
89
|
-
warnings.warn(
|
90
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
91
|
-
)
|
92
|
-
self.delegate_to = delegate_to
|
93
83
|
self.impersonation_chain = impersonation_chain
|
94
84
|
|
95
85
|
def execute(self, context: Context):
|
96
86
|
gdrive_hook = GoogleDriveHook(
|
97
87
|
gcp_conn_id=self.gcp_conn_id,
|
98
|
-
delegate_to=self.delegate_to,
|
99
88
|
impersonation_chain=self.impersonation_chain,
|
100
89
|
)
|
101
90
|
gcs_hook = GCSHook(
|
102
91
|
gcp_conn_id=self.gcp_conn_id,
|
103
|
-
delegate_to=self.delegate_to,
|
104
92
|
impersonation_chain=self.impersonation_chain,
|
105
93
|
)
|
106
94
|
file_metadata = gdrive_hook.get_file_id(
|
@@ -16,7 +16,6 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
import warnings
|
20
19
|
from typing import TYPE_CHECKING, Sequence
|
21
20
|
|
22
21
|
from airflow.models import BaseOperator
|
@@ -39,9 +38,6 @@ class GoogleDriveToLocalOperator(BaseOperator):
|
|
39
38
|
:param file_name: The name of the file residing in Google Drive
|
40
39
|
:param gcp_conn_id: The GCP connection ID to use when fetching connection info.
|
41
40
|
:param drive_id: Optional. The id of the shared Google Drive in which the file resides.
|
42
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
43
|
-
if any. For this to work, the service account making the request must have
|
44
|
-
domain-wide delegation enabled.
|
45
41
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
46
42
|
credentials, or chained list of accounts required to get the access_token
|
47
43
|
of the last account in the list, which will be impersonated in the request.
|
@@ -68,7 +64,6 @@ class GoogleDriveToLocalOperator(BaseOperator):
|
|
68
64
|
folder_id: str,
|
69
65
|
drive_id: str | None = None,
|
70
66
|
gcp_conn_id: str = "google_cloud_default",
|
71
|
-
delegate_to: str | None = None,
|
72
67
|
impersonation_chain: str | Sequence[str] | None = None,
|
73
68
|
**kwargs,
|
74
69
|
) -> None:
|
@@ -78,18 +73,12 @@ class GoogleDriveToLocalOperator(BaseOperator):
|
|
78
73
|
self.drive_id = drive_id
|
79
74
|
self.file_name = file_name
|
80
75
|
self.gcp_conn_id = gcp_conn_id
|
81
|
-
if delegate_to:
|
82
|
-
warnings.warn(
|
83
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
84
|
-
)
|
85
|
-
self.delegate_to = delegate_to
|
86
76
|
self.impersonation_chain = impersonation_chain
|
87
77
|
|
88
78
|
def execute(self, context: Context):
|
89
79
|
self.log.info("Executing download: %s into %s", self.file_name, self.output_file)
|
90
80
|
gdrive_hook = GoogleDriveHook(
|
91
81
|
gcp_conn_id=self.gcp_conn_id,
|
92
|
-
delegate_to=self.delegate_to,
|
93
82
|
impersonation_chain=self.impersonation_chain,
|
94
83
|
)
|
95
84
|
file_metadata = gdrive_hook.get_file_id(
|
@@ -19,7 +19,6 @@
|
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
21
|
import os
|
22
|
-
import warnings
|
23
22
|
from glob import glob
|
24
23
|
from typing import TYPE_CHECKING, Sequence
|
25
24
|
|
@@ -47,7 +46,6 @@ class LocalFilesystemToGCSOperator(BaseOperator):
|
|
47
46
|
:param bucket: The bucket to upload to. (templated)
|
48
47
|
:param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud.
|
49
48
|
:param mime_type: The mime-type string
|
50
|
-
:param delegate_to: The account to impersonate, if any
|
51
49
|
:param gzip: Allows for file to be compressed and uploaded as gzip
|
52
50
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
53
51
|
credentials, or chained list of accounts required to get the access_token
|
@@ -74,7 +72,6 @@ class LocalFilesystemToGCSOperator(BaseOperator):
|
|
74
72
|
bucket,
|
75
73
|
gcp_conn_id="google_cloud_default",
|
76
74
|
mime_type="application/octet-stream",
|
77
|
-
delegate_to=None,
|
78
75
|
gzip=False,
|
79
76
|
impersonation_chain: str | Sequence[str] | None = None,
|
80
77
|
**kwargs,
|
@@ -86,11 +83,6 @@ class LocalFilesystemToGCSOperator(BaseOperator):
|
|
86
83
|
self.bucket = bucket
|
87
84
|
self.gcp_conn_id = gcp_conn_id
|
88
85
|
self.mime_type = mime_type
|
89
|
-
if delegate_to:
|
90
|
-
warnings.warn(
|
91
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
92
|
-
)
|
93
|
-
self.delegate_to = delegate_to
|
94
86
|
self.gzip = gzip
|
95
87
|
self.impersonation_chain = impersonation_chain
|
96
88
|
|
@@ -98,7 +90,6 @@ class LocalFilesystemToGCSOperator(BaseOperator):
|
|
98
90
|
"""Uploads a file or list of files to Google Cloud Storage"""
|
99
91
|
hook = GCSHook(
|
100
92
|
gcp_conn_id=self.gcp_conn_id,
|
101
|
-
delegate_to=self.delegate_to,
|
102
93
|
impersonation_chain=self.impersonation_chain,
|
103
94
|
)
|
104
95
|
|
@@ -17,7 +17,6 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
import warnings
|
21
20
|
from tempfile import NamedTemporaryFile
|
22
21
|
from typing import TYPE_CHECKING, Sequence
|
23
22
|
|
@@ -61,9 +60,6 @@ class S3ToGCSOperator(S3ListOperator):
|
|
61
60
|
:param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud.
|
62
61
|
:param dest_gcs: The destination Google Cloud Storage bucket and prefix
|
63
62
|
where you want to store the files. (templated)
|
64
|
-
:param delegate_to: Google account to impersonate using domain-wide delegation of authority,
|
65
|
-
if any. For this to work, the service account making the request must have
|
66
|
-
domain-wide delegation enabled.
|
67
63
|
:param replace: Whether you want to replace existing destination files
|
68
64
|
or not.
|
69
65
|
:param gzip: Option to compress file for upload
|
@@ -115,7 +111,6 @@ class S3ToGCSOperator(S3ListOperator):
|
|
115
111
|
verify=None,
|
116
112
|
gcp_conn_id="google_cloud_default",
|
117
113
|
dest_gcs=None,
|
118
|
-
delegate_to=None,
|
119
114
|
replace=False,
|
120
115
|
gzip=False,
|
121
116
|
google_impersonation_chain: str | Sequence[str] | None = None,
|
@@ -125,11 +120,6 @@ class S3ToGCSOperator(S3ListOperator):
|
|
125
120
|
super().__init__(bucket=bucket, prefix=prefix, delimiter=delimiter, aws_conn_id=aws_conn_id, **kwargs)
|
126
121
|
self.gcp_conn_id = gcp_conn_id
|
127
122
|
self.dest_gcs = dest_gcs
|
128
|
-
if delegate_to:
|
129
|
-
warnings.warn(
|
130
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
131
|
-
)
|
132
|
-
self.delegate_to = delegate_to
|
133
123
|
self.replace = replace
|
134
124
|
self.verify = verify
|
135
125
|
self.gzip = gzip
|
@@ -153,7 +143,6 @@ class S3ToGCSOperator(S3ListOperator):
|
|
153
143
|
|
154
144
|
gcs_hook = GCSHook(
|
155
145
|
gcp_conn_id=self.gcp_conn_id,
|
156
|
-
delegate_to=self.delegate_to,
|
157
146
|
impersonation_chain=self.google_impersonation_chain,
|
158
147
|
)
|
159
148
|
|
@@ -19,7 +19,6 @@
|
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
21
|
import os
|
22
|
-
import warnings
|
23
22
|
from tempfile import NamedTemporaryFile
|
24
23
|
from typing import TYPE_CHECKING, Sequence
|
25
24
|
|
@@ -57,9 +56,6 @@ class SFTPToGCSOperator(BaseOperator):
|
|
57
56
|
:param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud.
|
58
57
|
:param sftp_conn_id: The sftp connection id. The name or identifier for
|
59
58
|
establishing a connection to the SFTP server.
|
60
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
61
|
-
if any. For this to work, the service account making the request must have
|
62
|
-
domain-wide delegation enabled.
|
63
59
|
:param mime_type: The mime-type string
|
64
60
|
:param gzip: Allows for file to be compressed and uploaded as gzip
|
65
61
|
:param move_object: When move object is True, the object is moved instead
|
@@ -90,7 +86,6 @@ class SFTPToGCSOperator(BaseOperator):
|
|
90
86
|
destination_path: str | None = None,
|
91
87
|
gcp_conn_id: str = "google_cloud_default",
|
92
88
|
sftp_conn_id: str = "ssh_default",
|
93
|
-
delegate_to: str | None = None,
|
94
89
|
mime_type: str = "application/octet-stream",
|
95
90
|
gzip: bool = False,
|
96
91
|
move_object: bool = False,
|
@@ -104,11 +99,6 @@ class SFTPToGCSOperator(BaseOperator):
|
|
104
99
|
self.destination_bucket = self._set_bucket_name(destination_bucket)
|
105
100
|
self.gcp_conn_id = gcp_conn_id
|
106
101
|
self.mime_type = mime_type
|
107
|
-
if delegate_to:
|
108
|
-
warnings.warn(
|
109
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
110
|
-
)
|
111
|
-
self.delegate_to = delegate_to
|
112
102
|
self.gzip = gzip
|
113
103
|
self.sftp_conn_id = sftp_conn_id
|
114
104
|
self.move_object = move_object
|
@@ -117,7 +107,6 @@ class SFTPToGCSOperator(BaseOperator):
|
|
117
107
|
def execute(self, context: Context):
|
118
108
|
gcs_hook = GCSHook(
|
119
109
|
gcp_conn_id=self.gcp_conn_id,
|
120
|
-
delegate_to=self.delegate_to,
|
121
110
|
impersonation_chain=self.impersonation_chain,
|
122
111
|
)
|
123
112
|
|
@@ -17,7 +17,6 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import csv
|
20
|
-
import warnings
|
21
20
|
from tempfile import NamedTemporaryFile
|
22
21
|
from typing import TYPE_CHECKING, Any, Sequence
|
23
22
|
|
@@ -45,9 +44,6 @@ class GoogleSheetsToGCSOperator(BaseOperator):
|
|
45
44
|
:param destination_path: The Google cloud storage URI array for the object created by the operator.
|
46
45
|
For example: ``path/to/my/files``.
|
47
46
|
:param gcp_conn_id: The connection ID to use when fetching connection info.
|
48
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
49
|
-
if any. For this to work, the service account making the request must have
|
50
|
-
domain-wide delegation enabled.
|
51
47
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
52
48
|
credentials, or chained list of accounts required to get the access_token
|
53
49
|
of the last account in the list, which will be impersonated in the request.
|
@@ -74,7 +70,6 @@ class GoogleSheetsToGCSOperator(BaseOperator):
|
|
74
70
|
sheet_filter: list[str] | None = None,
|
75
71
|
destination_path: str | None = None,
|
76
72
|
gcp_conn_id: str = "google_cloud_default",
|
77
|
-
delegate_to: str | None = None,
|
78
73
|
impersonation_chain: str | Sequence[str] | None = None,
|
79
74
|
**kwargs,
|
80
75
|
) -> None:
|
@@ -84,11 +79,6 @@ class GoogleSheetsToGCSOperator(BaseOperator):
|
|
84
79
|
self.sheet_filter = sheet_filter
|
85
80
|
self.destination_bucket = destination_bucket
|
86
81
|
self.destination_path = destination_path
|
87
|
-
if delegate_to:
|
88
|
-
warnings.warn(
|
89
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
90
|
-
)
|
91
|
-
self.delegate_to = delegate_to
|
92
82
|
self.impersonation_chain = impersonation_chain
|
93
83
|
|
94
84
|
def _upload_data(
|
@@ -122,12 +112,10 @@ class GoogleSheetsToGCSOperator(BaseOperator):
|
|
122
112
|
def execute(self, context: Context):
|
123
113
|
sheet_hook = GSheetsHook(
|
124
114
|
gcp_conn_id=self.gcp_conn_id,
|
125
|
-
delegate_to=self.delegate_to,
|
126
115
|
impersonation_chain=self.impersonation_chain,
|
127
116
|
)
|
128
117
|
gcs_hook = GCSHook(
|
129
118
|
gcp_conn_id=self.gcp_conn_id,
|
130
|
-
delegate_to=self.delegate_to,
|
131
119
|
impersonation_chain=self.impersonation_chain,
|
132
120
|
)
|
133
121
|
|
@@ -21,7 +21,6 @@ from __future__ import annotations
|
|
21
21
|
import abc
|
22
22
|
import json
|
23
23
|
import os
|
24
|
-
import warnings
|
25
24
|
from tempfile import NamedTemporaryFile
|
26
25
|
from typing import TYPE_CHECKING, Sequence
|
27
26
|
|
@@ -65,9 +64,6 @@ class BaseSQLToGCSOperator(BaseOperator):
|
|
65
64
|
dict. Examples could be seen: https://cloud.google.com/bigquery/docs
|
66
65
|
/schemas#specifying_a_json_schema_file
|
67
66
|
:param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud.
|
68
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
69
|
-
if any. For this to work, the service account making the request must have
|
70
|
-
domain-wide delegation enabled.
|
71
67
|
:param parameters: a parameters dict that is substituted at query runtime.
|
72
68
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
73
69
|
credentials, or chained list of accounts required to get the access_token
|
@@ -118,7 +114,6 @@ class BaseSQLToGCSOperator(BaseOperator):
|
|
118
114
|
schema: str | list | None = None,
|
119
115
|
parameters: dict | None = None,
|
120
116
|
gcp_conn_id: str = "google_cloud_default",
|
121
|
-
delegate_to: str | None = None,
|
122
117
|
impersonation_chain: str | Sequence[str] | None = None,
|
123
118
|
upload_metadata: bool = False,
|
124
119
|
exclude_columns: set | None = None,
|
@@ -143,11 +138,6 @@ class BaseSQLToGCSOperator(BaseOperator):
|
|
143
138
|
self.schema = schema
|
144
139
|
self.parameters = parameters
|
145
140
|
self.gcp_conn_id = gcp_conn_id
|
146
|
-
if delegate_to:
|
147
|
-
warnings.warn(
|
148
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
149
|
-
)
|
150
|
-
self.delegate_to = delegate_to
|
151
141
|
self.impersonation_chain = impersonation_chain
|
152
142
|
self.upload_metadata = upload_metadata
|
153
143
|
self.exclude_columns = exclude_columns
|
@@ -459,7 +449,6 @@ class BaseSQLToGCSOperator(BaseOperator):
|
|
459
449
|
"""Upload a file (data split or schema .json file) to Google Cloud Storage."""
|
460
450
|
hook = GCSHook(
|
461
451
|
gcp_conn_id=self.gcp_conn_id,
|
462
|
-
delegate_to=self.delegate_to,
|
463
452
|
impersonation_chain=self.impersonation_chain,
|
464
453
|
)
|
465
454
|
is_data_file = file_to_upload.get("file_name") != self.schema_filename
|
@@ -18,7 +18,6 @@
|
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
20
|
import asyncio
|
21
|
-
import warnings
|
22
21
|
from typing import Any, AsyncIterator, Sequence
|
23
22
|
|
24
23
|
from google.cloud.bigquery_datatransfer_v1 import TransferRun, TransferState
|
@@ -34,9 +33,6 @@ class BigQueryDataTransferRunTrigger(BaseTrigger):
|
|
34
33
|
:param run_id: ID of the Transfer Run which should be watched.
|
35
34
|
:param poll_interval: Optional. Interval which defines how often triggers check status of the job.
|
36
35
|
:param gcp_conn_id: The connection ID used to connect to Google Cloud.
|
37
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
38
|
-
if any. For this to work, the service account making the request must have
|
39
|
-
domain-wide delegation enabled.
|
40
36
|
:param location: BigQuery Transfer Service location for regional transfers.
|
41
37
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
42
38
|
credentials, or chained list of accounts required to get the access_token
|
@@ -55,7 +51,6 @@ class BigQueryDataTransferRunTrigger(BaseTrigger):
|
|
55
51
|
run_id: str,
|
56
52
|
poll_interval: int = 10,
|
57
53
|
gcp_conn_id: str = "google_cloud_default",
|
58
|
-
delegate_to: str | None = None,
|
59
54
|
location: str | None = None,
|
60
55
|
impersonation_chain: str | Sequence[str] | None = None,
|
61
56
|
):
|
@@ -65,11 +60,6 @@ class BigQueryDataTransferRunTrigger(BaseTrigger):
|
|
65
60
|
self.run_id = run_id
|
66
61
|
self.poll_interval = poll_interval
|
67
62
|
self.gcp_conn_id = gcp_conn_id
|
68
|
-
if delegate_to:
|
69
|
-
warnings.warn(
|
70
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
71
|
-
)
|
72
|
-
self.delegate_to = delegate_to
|
73
63
|
self.location = location
|
74
64
|
self.impersonation_chain = impersonation_chain
|
75
65
|
|
@@ -83,7 +73,6 @@ class BigQueryDataTransferRunTrigger(BaseTrigger):
|
|
83
73
|
"run_id": self.run_id,
|
84
74
|
"poll_interval": self.poll_interval,
|
85
75
|
"gcp_conn_id": self.gcp_conn_id,
|
86
|
-
"delegate_to": self.delegate_to,
|
87
76
|
"location": self.location,
|
88
77
|
"impersonation_chain": self.impersonation_chain,
|
89
78
|
},
|
@@ -157,7 +146,6 @@ class BigQueryDataTransferRunTrigger(BaseTrigger):
|
|
157
146
|
def _get_async_hook(self) -> AsyncBiqQueryDataTransferServiceHook:
|
158
147
|
return AsyncBiqQueryDataTransferServiceHook(
|
159
148
|
gcp_conn_id=self.gcp_conn_id,
|
160
|
-
delegate_to=self.delegate_to,
|
161
149
|
location=self.location,
|
162
150
|
impersonation_chain=self.impersonation_chain,
|
163
151
|
)
|
@@ -17,7 +17,6 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import asyncio
|
20
|
-
import warnings
|
21
20
|
from typing import Any, AsyncIterator, Sequence
|
22
21
|
|
23
22
|
from google.cloud.devtools.cloudbuild_v1.types import Build
|
@@ -41,9 +40,6 @@ class CloudBuildCreateBuildTrigger(BaseTrigger):
|
|
41
40
|
If set as a sequence, the identities from the list must grant
|
42
41
|
Service Account Token Creator IAM role to the directly preceding identity, with first
|
43
42
|
account from the list granting this role to the originating account (templated).
|
44
|
-
:param delegate_to: The account to impersonate using domain-wide delegation of authority,
|
45
|
-
if any. For this to work, the service account making the request must have
|
46
|
-
domain-wide delegation enabled.
|
47
43
|
:param poll_interval: polling period in seconds to check for the status
|
48
44
|
:param location: The location of the project.
|
49
45
|
"""
|
@@ -54,7 +50,6 @@ class CloudBuildCreateBuildTrigger(BaseTrigger):
|
|
54
50
|
project_id: str | None,
|
55
51
|
gcp_conn_id: str = "google_cloud_default",
|
56
52
|
impersonation_chain: str | Sequence[str] | None = None,
|
57
|
-
delegate_to: str | None = None,
|
58
53
|
poll_interval: float = 4.0,
|
59
54
|
location: str = "global",
|
60
55
|
):
|
@@ -63,11 +58,6 @@ class CloudBuildCreateBuildTrigger(BaseTrigger):
|
|
63
58
|
self.project_id = project_id
|
64
59
|
self.gcp_conn_id = gcp_conn_id
|
65
60
|
self.impersonation_chain = impersonation_chain
|
66
|
-
if delegate_to:
|
67
|
-
warnings.warn(
|
68
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
69
|
-
)
|
70
|
-
self.delegate_to = delegate_to
|
71
61
|
self.poll_interval = poll_interval
|
72
62
|
self.location = location
|
73
63
|
|
@@ -80,7 +70,6 @@ class CloudBuildCreateBuildTrigger(BaseTrigger):
|
|
80
70
|
"project_id": self.project_id,
|
81
71
|
"gcp_conn_id": self.gcp_conn_id,
|
82
72
|
"impersonation_chain": self.impersonation_chain,
|
83
|
-
"delegate_to": self.delegate_to,
|
84
73
|
"poll_interval": self.poll_interval,
|
85
74
|
"location": self.location,
|
86
75
|
},
|
@@ -19,7 +19,6 @@
|
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
21
|
import asyncio
|
22
|
-
import warnings
|
23
22
|
from typing import Any, Sequence
|
24
23
|
|
25
24
|
from airflow import AirflowException
|
@@ -37,7 +36,6 @@ class CloudComposerExecutionTrigger(BaseTrigger):
|
|
37
36
|
operation_name: str,
|
38
37
|
gcp_conn_id: str = "google_cloud_default",
|
39
38
|
impersonation_chain: str | Sequence[str] | None = None,
|
40
|
-
delegate_to: str | None = None,
|
41
39
|
pooling_period_seconds: int = 30,
|
42
40
|
):
|
43
41
|
super().__init__()
|
@@ -46,17 +44,11 @@ class CloudComposerExecutionTrigger(BaseTrigger):
|
|
46
44
|
self.operation_name = operation_name
|
47
45
|
self.gcp_conn_id = gcp_conn_id
|
48
46
|
self.impersonation_chain = impersonation_chain
|
49
|
-
if delegate_to:
|
50
|
-
warnings.warn(
|
51
|
-
"'delegate_to' parameter is deprecated, please use 'impersonation_chain'", DeprecationWarning
|
52
|
-
)
|
53
|
-
self.delegate_to = delegate_to
|
54
47
|
self.pooling_period_seconds = pooling_period_seconds
|
55
48
|
|
56
49
|
self.gcp_hook = CloudComposerAsyncHook(
|
57
50
|
gcp_conn_id=self.gcp_conn_id,
|
58
51
|
impersonation_chain=self.impersonation_chain,
|
59
|
-
delegate_to=self.delegate_to,
|
60
52
|
)
|
61
53
|
|
62
54
|
def serialize(self) -> tuple[str, dict[str, Any]]:
|
@@ -68,7 +60,6 @@ class CloudComposerExecutionTrigger(BaseTrigger):
|
|
68
60
|
"operation_name": self.operation_name,
|
69
61
|
"gcp_conn_id": self.gcp_conn_id,
|
70
62
|
"impersonation_chain": self.impersonation_chain,
|
71
|
-
"delegate_to": self.delegate_to,
|
72
63
|
"pooling_period_seconds": self.pooling_period_seconds,
|
73
64
|
},
|
74
65
|
)
|