apache-airflow-providers-google 16.0.0a1__py3-none-any.whl → 16.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/ads/hooks/ads.py +43 -5
- airflow/providers/google/ads/operators/ads.py +1 -1
- airflow/providers/google/ads/transfers/ads_to_gcs.py +1 -1
- airflow/providers/google/cloud/hooks/bigquery.py +63 -77
- airflow/providers/google/cloud/hooks/cloud_sql.py +8 -4
- airflow/providers/google/cloud/hooks/datacatalog.py +9 -1
- airflow/providers/google/cloud/hooks/dataflow.py +2 -2
- airflow/providers/google/cloud/hooks/dataplex.py +1 -1
- airflow/providers/google/cloud/hooks/dataprep.py +4 -1
- airflow/providers/google/cloud/hooks/gcs.py +5 -5
- airflow/providers/google/cloud/hooks/looker.py +10 -1
- airflow/providers/google/cloud/hooks/mlengine.py +2 -1
- airflow/providers/google/cloud/hooks/secret_manager.py +102 -10
- airflow/providers/google/cloud/hooks/spanner.py +2 -2
- airflow/providers/google/cloud/hooks/translate.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +0 -36
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +307 -7
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +44 -80
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +11 -2
- airflow/providers/google/cloud/hooks/vision.py +2 -2
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +75 -11
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/cloud_run.py +27 -0
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +16 -43
- airflow/providers/google/cloud/links/cloud_tasks.py +6 -25
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -96
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +14 -90
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +5 -59
- airflow/providers/google/cloud/links/life_sciences.py +0 -19
- airflow/providers/google/cloud/links/managed_kafka.py +0 -70
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +16 -186
- airflow/providers/google/cloud/links/vertex_ai.py +8 -224
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/log/gcs_task_handler.py +4 -4
- airflow/providers/google/cloud/operators/alloy_db.py +69 -54
- airflow/providers/google/cloud/operators/automl.py +16 -14
- airflow/providers/google/cloud/operators/bigquery.py +49 -25
- airflow/providers/google/cloud/operators/bigquery_dts.py +2 -4
- airflow/providers/google/cloud/operators/bigtable.py +35 -6
- airflow/providers/google/cloud/operators/cloud_base.py +21 -1
- airflow/providers/google/cloud/operators/cloud_build.py +74 -31
- airflow/providers/google/cloud/operators/cloud_composer.py +34 -35
- airflow/providers/google/cloud/operators/cloud_memorystore.py +68 -42
- airflow/providers/google/cloud/operators/cloud_run.py +9 -1
- airflow/providers/google/cloud/operators/cloud_sql.py +11 -15
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +0 -2
- airflow/providers/google/cloud/operators/compute.py +7 -39
- airflow/providers/google/cloud/operators/datacatalog.py +156 -20
- airflow/providers/google/cloud/operators/dataflow.py +37 -14
- airflow/providers/google/cloud/operators/dataform.py +14 -4
- airflow/providers/google/cloud/operators/datafusion.py +4 -12
- airflow/providers/google/cloud/operators/dataplex.py +180 -96
- airflow/providers/google/cloud/operators/dataprep.py +0 -4
- airflow/providers/google/cloud/operators/dataproc.py +10 -16
- airflow/providers/google/cloud/operators/dataproc_metastore.py +95 -87
- airflow/providers/google/cloud/operators/datastore.py +21 -5
- airflow/providers/google/cloud/operators/dlp.py +3 -26
- airflow/providers/google/cloud/operators/functions.py +15 -6
- airflow/providers/google/cloud/operators/gcs.py +1 -7
- airflow/providers/google/cloud/operators/kubernetes_engine.py +53 -92
- airflow/providers/google/cloud/operators/life_sciences.py +0 -1
- airflow/providers/google/cloud/operators/managed_kafka.py +106 -51
- airflow/providers/google/cloud/operators/mlengine.py +0 -1
- airflow/providers/google/cloud/operators/pubsub.py +4 -5
- airflow/providers/google/cloud/operators/spanner.py +0 -4
- airflow/providers/google/cloud/operators/speech_to_text.py +0 -1
- airflow/providers/google/cloud/operators/stackdriver.py +0 -8
- airflow/providers/google/cloud/operators/tasks.py +0 -11
- airflow/providers/google/cloud/operators/text_to_speech.py +0 -1
- airflow/providers/google/cloud/operators/translate.py +37 -13
- airflow/providers/google/cloud/operators/translate_speech.py +0 -1
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +31 -18
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +28 -8
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +38 -25
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +69 -7
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +42 -8
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +531 -0
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +93 -117
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +10 -8
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +56 -10
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +29 -6
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +9 -6
- airflow/providers/google/cloud/operators/workflows.py +1 -9
- airflow/providers/google/cloud/sensors/bigquery.py +1 -1
- airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -1
- airflow/providers/google/cloud/sensors/bigtable.py +15 -3
- airflow/providers/google/cloud/sensors/cloud_composer.py +6 -1
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -1
- airflow/providers/google/cloud/sensors/dataflow.py +3 -3
- airflow/providers/google/cloud/sensors/dataform.py +6 -1
- airflow/providers/google/cloud/sensors/datafusion.py +6 -1
- airflow/providers/google/cloud/sensors/dataplex.py +6 -1
- airflow/providers/google/cloud/sensors/dataprep.py +6 -1
- airflow/providers/google/cloud/sensors/dataproc.py +6 -1
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +6 -1
- airflow/providers/google/cloud/sensors/gcs.py +9 -3
- airflow/providers/google/cloud/sensors/looker.py +6 -1
- airflow/providers/google/cloud/sensors/pubsub.py +8 -3
- airflow/providers/google/cloud/sensors/tasks.py +6 -1
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +6 -1
- airflow/providers/google/cloud/sensors/workflows.py +6 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +10 -7
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +1 -2
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +0 -1
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +1 -1
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +1 -2
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/gcs_to_local.py +1 -1
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +5 -1
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +1 -1
- airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
- airflow/providers/google/cloud/transfers/local_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +11 -5
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +1 -1
- airflow/providers/google/cloud/triggers/bigquery.py +32 -5
- airflow/providers/google/cloud/triggers/dataproc.py +62 -10
- airflow/providers/google/cloud/utils/field_validator.py +1 -2
- airflow/providers/google/common/auth_backend/google_openid.py +2 -1
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +7 -3
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/firebase/operators/firestore.py +1 -1
- airflow/providers/google/get_provider_info.py +14 -16
- airflow/providers/google/leveldb/hooks/leveldb.py +30 -1
- airflow/providers/google/leveldb/operators/leveldb.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +3 -6
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +0 -1
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +4 -4
- airflow/providers/google/marketing_platform/operators/display_video.py +6 -6
- airflow/providers/google/marketing_platform/operators/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +6 -1
- airflow/providers/google/marketing_platform/sensors/display_video.py +6 -1
- airflow/providers/google/suite/operators/sheets.py +3 -3
- airflow/providers/google/suite/sensors/drive.py +6 -1
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +1 -1
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +1 -1
- airflow/providers/google/version_compat.py +28 -0
- {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0.dist-info}/METADATA +35 -35
- {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0.dist-info}/RECORD +171 -170
- airflow/providers/google/cloud/links/automl.py +0 -193
- {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0.dist-info}/entry_points.txt +0 -0
@@ -153,21 +153,24 @@ class AutoMLTrainModelOperator(GoogleCloudBaseOperator):
|
|
153
153
|
project_id = self.project_id or hook.project_id
|
154
154
|
if project_id:
|
155
155
|
TranslationLegacyModelTrainLink.persist(
|
156
|
-
context=context,
|
156
|
+
context=context,
|
157
|
+
dataset_id=self.model["dataset_id"],
|
158
|
+
project_id=project_id,
|
159
|
+
location=self.location,
|
157
160
|
)
|
158
161
|
operation_result = hook.wait_for_operation(timeout=self.timeout, operation=operation)
|
159
162
|
result = Model.to_dict(operation_result)
|
160
163
|
model_id = hook.extract_object_id(result)
|
161
164
|
self.log.info("Model is created, model_id: %s", model_id)
|
162
165
|
|
163
|
-
|
166
|
+
context["task_instance"].xcom_push(key="model_id", value=model_id)
|
164
167
|
if project_id:
|
165
168
|
TranslationLegacyModelLink.persist(
|
166
169
|
context=context,
|
167
|
-
task_instance=self,
|
168
170
|
dataset_id=self.model["dataset_id"] or "-",
|
169
171
|
model_id=model_id,
|
170
172
|
project_id=project_id,
|
173
|
+
location=self.location,
|
171
174
|
)
|
172
175
|
return result
|
173
176
|
|
@@ -313,10 +316,10 @@ class AutoMLPredictOperator(GoogleCloudBaseOperator):
|
|
313
316
|
if project_id and self.model_id and dataset_id:
|
314
317
|
TranslationLegacyModelPredictLink.persist(
|
315
318
|
context=context,
|
316
|
-
task_instance=self,
|
317
319
|
model_id=self.model_id,
|
318
320
|
dataset_id=dataset_id,
|
319
321
|
project_id=project_id,
|
322
|
+
location=self.location,
|
320
323
|
)
|
321
324
|
return PredictResponse.to_dict(result)
|
322
325
|
|
@@ -412,14 +415,14 @@ class AutoMLCreateDatasetOperator(GoogleCloudBaseOperator):
|
|
412
415
|
dataset_id = hook.extract_object_id(result)
|
413
416
|
self.log.info("Creating completed. Dataset id: %s", dataset_id)
|
414
417
|
|
415
|
-
|
418
|
+
context["task_instance"].xcom_push(key="dataset_id", value=dataset_id)
|
416
419
|
project_id = self.project_id or hook.project_id
|
417
420
|
if project_id:
|
418
421
|
TranslationLegacyDatasetLink.persist(
|
419
422
|
context=context,
|
420
|
-
task_instance=self,
|
421
423
|
dataset_id=dataset_id,
|
422
424
|
project_id=project_id,
|
425
|
+
location=self.location,
|
423
426
|
)
|
424
427
|
return result
|
425
428
|
|
@@ -530,9 +533,9 @@ class AutoMLImportDataOperator(GoogleCloudBaseOperator):
|
|
530
533
|
if project_id:
|
531
534
|
TranslationLegacyDatasetLink.persist(
|
532
535
|
context=context,
|
533
|
-
task_instance=self,
|
534
536
|
dataset_id=self.dataset_id,
|
535
537
|
project_id=project_id,
|
538
|
+
location=self.location,
|
536
539
|
)
|
537
540
|
|
538
541
|
|
@@ -649,9 +652,9 @@ class AutoMLTablesListColumnSpecsOperator(GoogleCloudBaseOperator):
|
|
649
652
|
if project_id:
|
650
653
|
TranslationLegacyDatasetLink.persist(
|
651
654
|
context=context,
|
652
|
-
task_instance=self,
|
653
655
|
dataset_id=self.dataset_id,
|
654
656
|
project_id=project_id,
|
657
|
+
location=self.location,
|
655
658
|
)
|
656
659
|
return result
|
657
660
|
|
@@ -749,9 +752,9 @@ class AutoMLTablesUpdateDatasetOperator(GoogleCloudBaseOperator):
|
|
749
752
|
if project_id:
|
750
753
|
TranslationLegacyDatasetLink.persist(
|
751
754
|
context=context,
|
752
|
-
task_instance=self,
|
753
755
|
dataset_id=hook.extract_object_id(self.dataset),
|
754
756
|
project_id=project_id,
|
757
|
+
location=self.location,
|
755
758
|
)
|
756
759
|
return Dataset.to_dict(result)
|
757
760
|
|
@@ -845,10 +848,10 @@ class AutoMLGetModelOperator(GoogleCloudBaseOperator):
|
|
845
848
|
if project_id:
|
846
849
|
TranslationLegacyModelLink.persist(
|
847
850
|
context=context,
|
848
|
-
task_instance=self,
|
849
851
|
dataset_id=model["dataset_id"],
|
850
852
|
model_id=self.model_id,
|
851
853
|
project_id=project_id,
|
854
|
+
location=self.location,
|
852
855
|
)
|
853
856
|
return model
|
854
857
|
|
@@ -1154,9 +1157,9 @@ class AutoMLTablesListTableSpecsOperator(GoogleCloudBaseOperator):
|
|
1154
1157
|
if project_id:
|
1155
1158
|
TranslationLegacyDatasetLink.persist(
|
1156
1159
|
context=context,
|
1157
|
-
task_instance=self,
|
1158
1160
|
dataset_id=self.dataset_id,
|
1159
1161
|
project_id=project_id,
|
1162
|
+
location=self.location,
|
1160
1163
|
)
|
1161
1164
|
return result
|
1162
1165
|
|
@@ -1245,14 +1248,13 @@ class AutoMLListDatasetOperator(GoogleCloudBaseOperator):
|
|
1245
1248
|
result.append(Dataset.to_dict(dataset))
|
1246
1249
|
self.log.info("Datasets obtained.")
|
1247
1250
|
|
1248
|
-
|
1249
|
-
context,
|
1251
|
+
context["task_instance"].xcom_push(
|
1250
1252
|
key="dataset_id_list",
|
1251
1253
|
value=[hook.extract_object_id(d) for d in result],
|
1252
1254
|
)
|
1253
1255
|
project_id = self.project_id or hook.project_id
|
1254
1256
|
if project_id:
|
1255
|
-
TranslationDatasetListLink.persist(context=context,
|
1257
|
+
TranslationDatasetListLink.persist(context=context, project_id=project_id)
|
1256
1258
|
return result
|
1257
1259
|
|
1258
1260
|
|
@@ -93,16 +93,32 @@ class IfExistAction(enum.Enum):
|
|
93
93
|
SKIP = "skip"
|
94
94
|
|
95
95
|
|
96
|
+
class _BigQueryHookWithFlexibleProjectId(BigQueryHook):
|
97
|
+
@property
|
98
|
+
def project_id(self) -> str:
|
99
|
+
_, project_id = self.get_credentials_and_project_id()
|
100
|
+
return project_id or PROVIDE_PROJECT_ID
|
101
|
+
|
102
|
+
@project_id.setter
|
103
|
+
def project_id(self, value: str) -> None:
|
104
|
+
cached_creds, _ = self.get_credentials_and_project_id()
|
105
|
+
self._cached_project_id = value or PROVIDE_PROJECT_ID
|
106
|
+
self._cached_credntials = cached_creds
|
107
|
+
|
108
|
+
|
96
109
|
class _BigQueryDbHookMixin:
|
97
|
-
def get_db_hook(self: BigQueryCheckOperator) ->
|
110
|
+
def get_db_hook(self: BigQueryCheckOperator) -> _BigQueryHookWithFlexibleProjectId: # type:ignore[misc]
|
98
111
|
"""Get BigQuery DB Hook."""
|
99
|
-
|
112
|
+
hook = _BigQueryHookWithFlexibleProjectId(
|
100
113
|
gcp_conn_id=self.gcp_conn_id,
|
101
114
|
use_legacy_sql=self.use_legacy_sql,
|
102
115
|
location=self.location,
|
103
116
|
impersonation_chain=self.impersonation_chain,
|
104
117
|
labels=self.labels,
|
105
118
|
)
|
119
|
+
if self.project_id:
|
120
|
+
hook.project_id = self.project_id
|
121
|
+
return hook
|
106
122
|
|
107
123
|
|
108
124
|
class _BigQueryOperatorsEncryptionConfigurationMixin:
|
@@ -190,6 +206,7 @@ class BigQueryCheckOperator(
|
|
190
206
|
https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs.
|
191
207
|
For example, [{ 'name': 'corpus', 'parameterType': { 'type': 'STRING' },
|
192
208
|
'parameterValue': { 'value': 'romeoandjuliet' } }]. (templated)
|
209
|
+
:param project_id: Google Cloud Project where the job is running
|
193
210
|
"""
|
194
211
|
|
195
212
|
template_fields: Sequence[str] = (
|
@@ -208,6 +225,7 @@ class BigQueryCheckOperator(
|
|
208
225
|
*,
|
209
226
|
sql: str,
|
210
227
|
gcp_conn_id: str = "google_cloud_default",
|
228
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
211
229
|
use_legacy_sql: bool = True,
|
212
230
|
location: str | None = None,
|
213
231
|
impersonation_chain: str | Sequence[str] | None = None,
|
@@ -228,6 +246,7 @@ class BigQueryCheckOperator(
|
|
228
246
|
self.deferrable = deferrable
|
229
247
|
self.poll_interval = poll_interval
|
230
248
|
self.query_params = query_params
|
249
|
+
self.project_id = project_id
|
231
250
|
|
232
251
|
def _submit_job(
|
233
252
|
self,
|
@@ -243,7 +262,7 @@ class BigQueryCheckOperator(
|
|
243
262
|
|
244
263
|
return hook.insert_job(
|
245
264
|
configuration=configuration,
|
246
|
-
project_id=
|
265
|
+
project_id=self.project_id,
|
247
266
|
location=self.location,
|
248
267
|
job_id=job_id,
|
249
268
|
nowait=True,
|
@@ -257,6 +276,8 @@ class BigQueryCheckOperator(
|
|
257
276
|
gcp_conn_id=self.gcp_conn_id,
|
258
277
|
impersonation_chain=self.impersonation_chain,
|
259
278
|
)
|
279
|
+
if self.project_id is None:
|
280
|
+
self.project_id = hook.project_id
|
260
281
|
job = self._submit_job(hook, job_id="")
|
261
282
|
context["ti"].xcom_push(key="job_id", value=job.job_id)
|
262
283
|
if job.running():
|
@@ -265,7 +286,7 @@ class BigQueryCheckOperator(
|
|
265
286
|
trigger=BigQueryCheckTrigger(
|
266
287
|
conn_id=self.gcp_conn_id,
|
267
288
|
job_id=job.job_id,
|
268
|
-
project_id=
|
289
|
+
project_id=self.project_id,
|
269
290
|
location=self.location or hook.location,
|
270
291
|
poll_interval=self.poll_interval,
|
271
292
|
impersonation_chain=self.impersonation_chain,
|
@@ -342,6 +363,7 @@ class BigQueryValueCheckOperator(
|
|
342
363
|
:param deferrable: Run operator in the deferrable mode.
|
343
364
|
:param poll_interval: (Deferrable mode only) polling period in seconds to
|
344
365
|
check for the status of job.
|
366
|
+
:param project_id: Google Cloud Project where the job is running
|
345
367
|
"""
|
346
368
|
|
347
369
|
template_fields: Sequence[str] = (
|
@@ -363,6 +385,7 @@ class BigQueryValueCheckOperator(
|
|
363
385
|
tolerance: Any = None,
|
364
386
|
encryption_configuration: dict | None = None,
|
365
387
|
gcp_conn_id: str = "google_cloud_default",
|
388
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
366
389
|
use_legacy_sql: bool = True,
|
367
390
|
location: str | None = None,
|
368
391
|
impersonation_chain: str | Sequence[str] | None = None,
|
@@ -380,6 +403,7 @@ class BigQueryValueCheckOperator(
|
|
380
403
|
self.labels = labels
|
381
404
|
self.deferrable = deferrable
|
382
405
|
self.poll_interval = poll_interval
|
406
|
+
self.project_id = project_id
|
383
407
|
|
384
408
|
def _submit_job(
|
385
409
|
self,
|
@@ -398,7 +422,7 @@ class BigQueryValueCheckOperator(
|
|
398
422
|
|
399
423
|
return hook.insert_job(
|
400
424
|
configuration=configuration,
|
401
|
-
project_id=
|
425
|
+
project_id=self.project_id,
|
402
426
|
location=self.location,
|
403
427
|
job_id=job_id,
|
404
428
|
nowait=True,
|
@@ -409,7 +433,8 @@ class BigQueryValueCheckOperator(
|
|
409
433
|
super().execute(context=context)
|
410
434
|
else:
|
411
435
|
hook = BigQueryHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
|
412
|
-
|
436
|
+
if self.project_id is None:
|
437
|
+
self.project_id = hook.project_id
|
413
438
|
job = self._submit_job(hook, job_id="")
|
414
439
|
context["ti"].xcom_push(key="job_id", value=job.job_id)
|
415
440
|
if job.running():
|
@@ -418,7 +443,7 @@ class BigQueryValueCheckOperator(
|
|
418
443
|
trigger=BigQueryValueCheckTrigger(
|
419
444
|
conn_id=self.gcp_conn_id,
|
420
445
|
job_id=job.job_id,
|
421
|
-
project_id=
|
446
|
+
project_id=self.project_id,
|
422
447
|
location=self.location or hook.location,
|
423
448
|
sql=self.sql,
|
424
449
|
pass_value=self.pass_value,
|
@@ -575,6 +600,9 @@ class BigQueryIntervalCheckOperator(
|
|
575
600
|
hook = BigQueryHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
|
576
601
|
self.log.info("Using ratio formula: %s", self.ratio_formula)
|
577
602
|
|
603
|
+
if self.project_id is None:
|
604
|
+
self.project_id = hook.project_id
|
605
|
+
|
578
606
|
self.log.info("Executing SQL check: %s", self.sql1)
|
579
607
|
job_1 = self._submit_job(hook, sql=self.sql1, job_id="")
|
580
608
|
context["ti"].xcom_push(key="job_id", value=job_1.job_id)
|
@@ -587,7 +615,7 @@ class BigQueryIntervalCheckOperator(
|
|
587
615
|
conn_id=self.gcp_conn_id,
|
588
616
|
first_job_id=job_1.job_id,
|
589
617
|
second_job_id=job_2.job_id,
|
590
|
-
project_id=
|
618
|
+
project_id=self.project_id,
|
591
619
|
table=self.table,
|
592
620
|
location=self.location or hook.location,
|
593
621
|
metrics_thresholds=self.metrics_thresholds,
|
@@ -654,6 +682,7 @@ class BigQueryColumnCheckOperator(
|
|
654
682
|
Service Account Token Creator IAM role to the directly preceding identity, with first
|
655
683
|
account from the list granting this role to the originating account (templated).
|
656
684
|
:param labels: a dictionary containing labels for the table, passed to BigQuery
|
685
|
+
:param project_id: Google Cloud Project where the job is running
|
657
686
|
"""
|
658
687
|
|
659
688
|
template_fields: Sequence[str] = tuple(set(SQLColumnCheckOperator.template_fields) | {"gcp_conn_id"})
|
@@ -670,6 +699,7 @@ class BigQueryColumnCheckOperator(
|
|
670
699
|
accept_none: bool = True,
|
671
700
|
encryption_configuration: dict | None = None,
|
672
701
|
gcp_conn_id: str = "google_cloud_default",
|
702
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
673
703
|
use_legacy_sql: bool = True,
|
674
704
|
location: str | None = None,
|
675
705
|
impersonation_chain: str | Sequence[str] | None = None,
|
@@ -695,6 +725,7 @@ class BigQueryColumnCheckOperator(
|
|
695
725
|
self.location = location
|
696
726
|
self.impersonation_chain = impersonation_chain
|
697
727
|
self.labels = labels
|
728
|
+
self.project_id = project_id
|
698
729
|
|
699
730
|
def _submit_job(
|
700
731
|
self,
|
@@ -706,7 +737,7 @@ class BigQueryColumnCheckOperator(
|
|
706
737
|
self.include_encryption_configuration(configuration, "query")
|
707
738
|
return hook.insert_job(
|
708
739
|
configuration=configuration,
|
709
|
-
project_id=
|
740
|
+
project_id=self.project_id,
|
710
741
|
location=self.location,
|
711
742
|
job_id=job_id,
|
712
743
|
nowait=False,
|
@@ -715,6 +746,9 @@ class BigQueryColumnCheckOperator(
|
|
715
746
|
def execute(self, context=None):
|
716
747
|
"""Perform checks on the given columns."""
|
717
748
|
hook = self.get_db_hook()
|
749
|
+
|
750
|
+
if self.project_id is None:
|
751
|
+
self.project_id = hook.project_id
|
718
752
|
failed_tests = []
|
719
753
|
|
720
754
|
job = self._submit_job(hook, job_id="")
|
@@ -786,6 +820,7 @@ class BigQueryTableCheckOperator(
|
|
786
820
|
account from the list granting this role to the originating account (templated).
|
787
821
|
:param labels: a dictionary containing labels for the table, passed to BigQuery
|
788
822
|
:param encryption_configuration: (Optional) Custom encryption configuration (e.g., Cloud KMS keys).
|
823
|
+
:param project_id: Google Cloud Project where the job is running
|
789
824
|
|
790
825
|
.. code-block:: python
|
791
826
|
|
@@ -805,6 +840,7 @@ class BigQueryTableCheckOperator(
|
|
805
840
|
checks: dict,
|
806
841
|
partition_clause: str | None = None,
|
807
842
|
gcp_conn_id: str = "google_cloud_default",
|
843
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
808
844
|
use_legacy_sql: bool = True,
|
809
845
|
location: str | None = None,
|
810
846
|
impersonation_chain: str | Sequence[str] | None = None,
|
@@ -819,6 +855,7 @@ class BigQueryTableCheckOperator(
|
|
819
855
|
self.impersonation_chain = impersonation_chain
|
820
856
|
self.labels = labels
|
821
857
|
self.encryption_configuration = encryption_configuration
|
858
|
+
self.project_id = project_id
|
822
859
|
|
823
860
|
def _submit_job(
|
824
861
|
self,
|
@@ -832,7 +869,7 @@ class BigQueryTableCheckOperator(
|
|
832
869
|
|
833
870
|
return hook.insert_job(
|
834
871
|
configuration=configuration,
|
835
|
-
project_id=
|
872
|
+
project_id=self.project_id,
|
836
873
|
location=self.location,
|
837
874
|
job_id=job_id,
|
838
875
|
nowait=False,
|
@@ -841,6 +878,8 @@ class BigQueryTableCheckOperator(
|
|
841
878
|
def execute(self, context=None):
|
842
879
|
"""Execute the given checks on the table."""
|
843
880
|
hook = self.get_db_hook()
|
881
|
+
if self.project_id is None:
|
882
|
+
self.project_id = hook.project_id
|
844
883
|
job = self._submit_job(hook, job_id="")
|
845
884
|
context["ti"].xcom_push(key="job_id", value=job.job_id)
|
846
885
|
records = job.result().to_dataframe()
|
@@ -1285,7 +1324,6 @@ class BigQueryCreateTableOperator(GoogleCloudBaseOperator):
|
|
1285
1324
|
if self._table:
|
1286
1325
|
persist_kwargs = {
|
1287
1326
|
"context": context,
|
1288
|
-
"task_instance": self,
|
1289
1327
|
"project_id": self._table.to_api_repr()["tableReference"]["projectId"],
|
1290
1328
|
"dataset_id": self._table.to_api_repr()["tableReference"]["datasetId"],
|
1291
1329
|
"table_id": self._table.to_api_repr()["tableReference"]["tableId"],
|
@@ -1304,7 +1342,6 @@ class BigQueryCreateTableOperator(GoogleCloudBaseOperator):
|
|
1304
1342
|
self.log.info(error_msg)
|
1305
1343
|
persist_kwargs = {
|
1306
1344
|
"context": context,
|
1307
|
-
"task_instance": self,
|
1308
1345
|
"project_id": self.project_id or bq_hook.project_id,
|
1309
1346
|
"dataset_id": self.dataset_id,
|
1310
1347
|
"table_id": self.table_id,
|
@@ -1569,7 +1606,6 @@ class BigQueryCreateEmptyTableOperator(GoogleCloudBaseOperator):
|
|
1569
1606
|
if self._table:
|
1570
1607
|
persist_kwargs = {
|
1571
1608
|
"context": context,
|
1572
|
-
"task_instance": self,
|
1573
1609
|
"project_id": self._table.to_api_repr()["tableReference"]["projectId"],
|
1574
1610
|
"dataset_id": self._table.to_api_repr()["tableReference"]["datasetId"],
|
1575
1611
|
"table_id": self._table.to_api_repr()["tableReference"]["tableId"],
|
@@ -1588,7 +1624,6 @@ class BigQueryCreateEmptyTableOperator(GoogleCloudBaseOperator):
|
|
1588
1624
|
self.log.info(error_msg)
|
1589
1625
|
persist_kwargs = {
|
1590
1626
|
"context": context,
|
1591
|
-
"task_instance": self,
|
1592
1627
|
"project_id": self.project_id or bq_hook.project_id,
|
1593
1628
|
"dataset_id": self.dataset_id,
|
1594
1629
|
"table_id": self.table_id,
|
@@ -1859,7 +1894,6 @@ class BigQueryCreateExternalTableOperator(GoogleCloudBaseOperator):
|
|
1859
1894
|
if self._table:
|
1860
1895
|
BigQueryTableLink.persist(
|
1861
1896
|
context=context,
|
1862
|
-
task_instance=self,
|
1863
1897
|
dataset_id=self._table.dataset_id,
|
1864
1898
|
project_id=self._table.project,
|
1865
1899
|
table_id=self._table.table_id,
|
@@ -1918,7 +1952,6 @@ class BigQueryCreateExternalTableOperator(GoogleCloudBaseOperator):
|
|
1918
1952
|
if self._table:
|
1919
1953
|
BigQueryTableLink.persist(
|
1920
1954
|
context=context,
|
1921
|
-
task_instance=self,
|
1922
1955
|
dataset_id=self._table.dataset_id,
|
1923
1956
|
project_id=self._table.project,
|
1924
1957
|
table_id=self._table.table_id,
|
@@ -2116,7 +2149,6 @@ class BigQueryCreateEmptyDatasetOperator(GoogleCloudBaseOperator):
|
|
2116
2149
|
)
|
2117
2150
|
persist_kwargs = {
|
2118
2151
|
"context": context,
|
2119
|
-
"task_instance": self,
|
2120
2152
|
"project_id": dataset["datasetReference"]["projectId"],
|
2121
2153
|
"dataset_id": dataset["datasetReference"]["datasetId"],
|
2122
2154
|
}
|
@@ -2128,7 +2160,6 @@ class BigQueryCreateEmptyDatasetOperator(GoogleCloudBaseOperator):
|
|
2128
2160
|
)
|
2129
2161
|
persist_kwargs = {
|
2130
2162
|
"context": context,
|
2131
|
-
"task_instance": self,
|
2132
2163
|
"project_id": project_id,
|
2133
2164
|
"dataset_id": dataset_id,
|
2134
2165
|
}
|
@@ -2200,7 +2231,6 @@ class BigQueryGetDatasetOperator(GoogleCloudBaseOperator):
|
|
2200
2231
|
dataset_api_repr = dataset.to_api_repr()
|
2201
2232
|
BigQueryDatasetLink.persist(
|
2202
2233
|
context=context,
|
2203
|
-
task_instance=self,
|
2204
2234
|
dataset_id=dataset_api_repr["datasetReference"]["datasetId"],
|
2205
2235
|
project_id=dataset_api_repr["datasetReference"]["projectId"],
|
2206
2236
|
)
|
@@ -2349,7 +2379,6 @@ class BigQueryUpdateTableOperator(GoogleCloudBaseOperator):
|
|
2349
2379
|
if self._table:
|
2350
2380
|
BigQueryTableLink.persist(
|
2351
2381
|
context=context,
|
2352
|
-
task_instance=self,
|
2353
2382
|
dataset_id=self._table["tableReference"]["datasetId"],
|
2354
2383
|
project_id=self._table["tableReference"]["projectId"],
|
2355
2384
|
table_id=self._table["tableReference"]["tableId"],
|
@@ -2452,7 +2481,6 @@ class BigQueryUpdateDatasetOperator(GoogleCloudBaseOperator):
|
|
2452
2481
|
dataset_api_repr = dataset.to_api_repr()
|
2453
2482
|
BigQueryDatasetLink.persist(
|
2454
2483
|
context=context,
|
2455
|
-
task_instance=self,
|
2456
2484
|
dataset_id=dataset_api_repr["datasetReference"]["datasetId"],
|
2457
2485
|
project_id=dataset_api_repr["datasetReference"]["projectId"],
|
2458
2486
|
)
|
@@ -2624,7 +2652,6 @@ class BigQueryUpsertTableOperator(GoogleCloudBaseOperator):
|
|
2624
2652
|
if self._table:
|
2625
2653
|
BigQueryTableLink.persist(
|
2626
2654
|
context=context,
|
2627
|
-
task_instance=self,
|
2628
2655
|
dataset_id=self._table["tableReference"]["datasetId"],
|
2629
2656
|
project_id=self._table["tableReference"]["projectId"],
|
2630
2657
|
table_id=self._table["tableReference"]["tableId"],
|
@@ -2754,7 +2781,6 @@ class BigQueryUpdateTableSchemaOperator(GoogleCloudBaseOperator):
|
|
2754
2781
|
if self._table:
|
2755
2782
|
BigQueryTableLink.persist(
|
2756
2783
|
context=context,
|
2757
|
-
task_instance=self,
|
2758
2784
|
dataset_id=self._table["tableReference"]["datasetId"],
|
2759
2785
|
project_id=self._table["tableReference"]["projectId"],
|
2760
2786
|
table_id=self._table["tableReference"]["tableId"],
|
@@ -3000,7 +3026,6 @@ class BigQueryInsertJobOperator(GoogleCloudBaseOperator, _BigQueryInsertJobOpera
|
|
3000
3026
|
table = job_configuration[job_type][table_prop]
|
3001
3027
|
persist_kwargs = {
|
3002
3028
|
"context": context,
|
3003
|
-
"task_instance": self,
|
3004
3029
|
"project_id": self.project_id,
|
3005
3030
|
"table_id": table,
|
3006
3031
|
}
|
@@ -3022,7 +3047,6 @@ class BigQueryInsertJobOperator(GoogleCloudBaseOperator, _BigQueryInsertJobOpera
|
|
3022
3047
|
|
3023
3048
|
persist_kwargs = {
|
3024
3049
|
"context": context,
|
3025
|
-
"task_instance": self,
|
3026
3050
|
"project_id": self.project_id,
|
3027
3051
|
"location": self.location,
|
3028
3052
|
"job_id": self.job_id,
|
@@ -134,7 +134,6 @@ class BigQueryCreateDataTransferOperator(GoogleCloudBaseOperator):
|
|
134
134
|
transfer_config = _get_transfer_config_details(response.name)
|
135
135
|
BigQueryDataTransferConfigLink.persist(
|
136
136
|
context=context,
|
137
|
-
task_instance=self,
|
138
137
|
region=transfer_config["region"],
|
139
138
|
config_id=transfer_config["config_id"],
|
140
139
|
project_id=transfer_config["project_id"],
|
@@ -142,7 +141,7 @@ class BigQueryCreateDataTransferOperator(GoogleCloudBaseOperator):
|
|
142
141
|
|
143
142
|
result = TransferConfig.to_dict(response)
|
144
143
|
self.log.info("Created DTS transfer config %s", get_object_id(result))
|
145
|
-
|
144
|
+
context["ti"].xcom_push(key="transfer_config_id", value=get_object_id(result))
|
146
145
|
# don't push AWS secret in XCOM
|
147
146
|
result.get("params", {}).pop("secret_access_key", None)
|
148
147
|
result.get("params", {}).pop("access_key_id", None)
|
@@ -329,7 +328,6 @@ class BigQueryDataTransferServiceStartTransferRunsOperator(GoogleCloudBaseOperat
|
|
329
328
|
transfer_config = _get_transfer_config_details(response.runs[0].name)
|
330
329
|
BigQueryDataTransferConfigLink.persist(
|
331
330
|
context=context,
|
332
|
-
task_instance=self,
|
333
331
|
region=transfer_config["region"],
|
334
332
|
config_id=transfer_config["config_id"],
|
335
333
|
project_id=transfer_config["project_id"],
|
@@ -337,7 +335,7 @@ class BigQueryDataTransferServiceStartTransferRunsOperator(GoogleCloudBaseOperat
|
|
337
335
|
|
338
336
|
result = StartManualTransferRunsResponse.to_dict(response)
|
339
337
|
run_id = get_object_id(result["runs"][0])
|
340
|
-
|
338
|
+
context["ti"].xcom_push(key="run_id", value=run_id)
|
341
339
|
|
342
340
|
if not self.deferrable:
|
343
341
|
# Save as attribute for further use by OpenLineage
|
@@ -20,7 +20,7 @@
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
22
|
from collections.abc import Iterable, Sequence
|
23
|
-
from typing import TYPE_CHECKING
|
23
|
+
from typing import TYPE_CHECKING, Any
|
24
24
|
|
25
25
|
import google.api_core.exceptions
|
26
26
|
|
@@ -142,6 +142,13 @@ class BigtableCreateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
|
|
142
142
|
self.impersonation_chain = impersonation_chain
|
143
143
|
super().__init__(**kwargs)
|
144
144
|
|
145
|
+
@property
|
146
|
+
def extra_links_params(self) -> dict[str, Any]:
|
147
|
+
return {
|
148
|
+
"instance_id": self.instance_id,
|
149
|
+
"project_id": self.project_id,
|
150
|
+
}
|
151
|
+
|
145
152
|
def execute(self, context: Context) -> None:
|
146
153
|
hook = BigtableHook(
|
147
154
|
gcp_conn_id=self.gcp_conn_id,
|
@@ -155,7 +162,7 @@ class BigtableCreateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
|
|
155
162
|
"The instance '%s' already exists in this project. Consider it as created",
|
156
163
|
self.instance_id,
|
157
164
|
)
|
158
|
-
BigtableInstanceLink.persist(context=context
|
165
|
+
BigtableInstanceLink.persist(context=context)
|
159
166
|
return
|
160
167
|
try:
|
161
168
|
hook.create_instance(
|
@@ -171,7 +178,7 @@ class BigtableCreateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
|
|
171
178
|
cluster_storage_type=self.cluster_storage_type,
|
172
179
|
timeout=self.timeout,
|
173
180
|
)
|
174
|
-
BigtableInstanceLink.persist(context=context
|
181
|
+
BigtableInstanceLink.persist(context=context)
|
175
182
|
except google.api_core.exceptions.GoogleAPICallError as e:
|
176
183
|
self.log.error("An error occurred. Exiting.")
|
177
184
|
raise e
|
@@ -240,6 +247,13 @@ class BigtableUpdateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
|
|
240
247
|
self.impersonation_chain = impersonation_chain
|
241
248
|
super().__init__(**kwargs)
|
242
249
|
|
250
|
+
@property
|
251
|
+
def extra_links_params(self) -> dict[str, Any]:
|
252
|
+
return {
|
253
|
+
"instance_id": self.instance_id,
|
254
|
+
"project_id": self.project_id,
|
255
|
+
}
|
256
|
+
|
243
257
|
def execute(self, context: Context) -> None:
|
244
258
|
hook = BigtableHook(
|
245
259
|
gcp_conn_id=self.gcp_conn_id,
|
@@ -258,7 +272,7 @@ class BigtableUpdateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
|
|
258
272
|
instance_labels=self.instance_labels,
|
259
273
|
timeout=self.timeout,
|
260
274
|
)
|
261
|
-
BigtableInstanceLink.persist(context=context
|
275
|
+
BigtableInstanceLink.persist(context=context)
|
262
276
|
except google.api_core.exceptions.GoogleAPICallError as e:
|
263
277
|
self.log.error("An error occurred. Exiting.")
|
264
278
|
raise e
|
@@ -414,6 +428,13 @@ class BigtableCreateTableOperator(GoogleCloudBaseOperator, BigtableValidationMix
|
|
414
428
|
return False
|
415
429
|
return True
|
416
430
|
|
431
|
+
@property
|
432
|
+
def extra_links_params(self) -> dict[str, Any]:
|
433
|
+
return {
|
434
|
+
"instance_id": self.instance_id,
|
435
|
+
"project_id": self.project_id,
|
436
|
+
}
|
437
|
+
|
417
438
|
def execute(self, context: Context) -> None:
|
418
439
|
hook = BigtableHook(
|
419
440
|
gcp_conn_id=self.gcp_conn_id,
|
@@ -431,7 +452,7 @@ class BigtableCreateTableOperator(GoogleCloudBaseOperator, BigtableValidationMix
|
|
431
452
|
initial_split_keys=self.initial_split_keys,
|
432
453
|
column_families=self.column_families,
|
433
454
|
)
|
434
|
-
BigtableTablesLink.persist(context=context
|
455
|
+
BigtableTablesLink.persist(context=context)
|
435
456
|
except google.api_core.exceptions.AlreadyExists:
|
436
457
|
if not self._compare_column_families(hook, instance):
|
437
458
|
raise AirflowException(
|
@@ -575,6 +596,14 @@ class BigtableUpdateClusterOperator(GoogleCloudBaseOperator, BigtableValidationM
|
|
575
596
|
self.impersonation_chain = impersonation_chain
|
576
597
|
super().__init__(**kwargs)
|
577
598
|
|
599
|
+
@property
|
600
|
+
def extra_links_params(self) -> dict[str, Any]:
|
601
|
+
return {
|
602
|
+
"instance_id": self.instance_id,
|
603
|
+
"cluster_id": self.cluster_id,
|
604
|
+
"project_id": self.project_id,
|
605
|
+
}
|
606
|
+
|
578
607
|
def execute(self, context: Context) -> None:
|
579
608
|
hook = BigtableHook(
|
580
609
|
gcp_conn_id=self.gcp_conn_id,
|
@@ -586,7 +615,7 @@ class BigtableUpdateClusterOperator(GoogleCloudBaseOperator, BigtableValidationM
|
|
586
615
|
|
587
616
|
try:
|
588
617
|
hook.update_cluster(instance=instance, cluster_id=self.cluster_id, nodes=self.nodes)
|
589
|
-
BigtableClusterLink.persist(context=context
|
618
|
+
BigtableClusterLink.persist(context=context)
|
590
619
|
except google.api_core.exceptions.NotFound:
|
591
620
|
raise AirflowException(
|
592
621
|
f"Dependency: cluster '{self.cluster_id}' does not exist for instance '{self.instance_id}'."
|
@@ -19,9 +19,11 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
+
from typing import Any
|
23
|
+
|
22
24
|
from google.api_core.gapic_v1.method import DEFAULT
|
23
25
|
|
24
|
-
from airflow.
|
26
|
+
from airflow.providers.google.version_compat import BaseOperator
|
25
27
|
|
26
28
|
|
27
29
|
class GoogleCloudBaseOperator(BaseOperator):
|
@@ -36,3 +38,21 @@ class GoogleCloudBaseOperator(BaseOperator):
|
|
36
38
|
"""
|
37
39
|
memo[id(DEFAULT)] = DEFAULT
|
38
40
|
return super().__deepcopy__(memo)
|
41
|
+
|
42
|
+
@property
|
43
|
+
def extra_links_params(self) -> dict[str, Any]:
|
44
|
+
"""
|
45
|
+
Override this method to include parameters for link formatting in extra links.
|
46
|
+
|
47
|
+
For example; most of the links on the Google provider require `project_id` and `location` in the Link.
|
48
|
+
To be not repeat; you can override this function and return something like the following:
|
49
|
+
|
50
|
+
.. code-block:: python
|
51
|
+
|
52
|
+
{
|
53
|
+
"project_id": self.project_id,
|
54
|
+
"location": self.location,
|
55
|
+
}
|
56
|
+
|
57
|
+
"""
|
58
|
+
return {}
|