apache-airflow-providers-google 16.0.0a1__py3-none-any.whl → 16.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. airflow/providers/google/__init__.py +1 -1
  2. airflow/providers/google/ads/hooks/ads.py +43 -5
  3. airflow/providers/google/ads/operators/ads.py +1 -1
  4. airflow/providers/google/ads/transfers/ads_to_gcs.py +1 -1
  5. airflow/providers/google/cloud/hooks/bigquery.py +63 -77
  6. airflow/providers/google/cloud/hooks/cloud_sql.py +8 -4
  7. airflow/providers/google/cloud/hooks/datacatalog.py +9 -1
  8. airflow/providers/google/cloud/hooks/dataflow.py +2 -2
  9. airflow/providers/google/cloud/hooks/dataplex.py +1 -1
  10. airflow/providers/google/cloud/hooks/dataprep.py +4 -1
  11. airflow/providers/google/cloud/hooks/gcs.py +5 -5
  12. airflow/providers/google/cloud/hooks/looker.py +10 -1
  13. airflow/providers/google/cloud/hooks/mlengine.py +2 -1
  14. airflow/providers/google/cloud/hooks/secret_manager.py +102 -10
  15. airflow/providers/google/cloud/hooks/spanner.py +2 -2
  16. airflow/providers/google/cloud/hooks/translate.py +1 -1
  17. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +0 -36
  18. airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +307 -7
  19. airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +44 -80
  20. airflow/providers/google/cloud/hooks/vertex_ai/ray.py +11 -2
  21. airflow/providers/google/cloud/hooks/vision.py +2 -2
  22. airflow/providers/google/cloud/links/alloy_db.py +0 -46
  23. airflow/providers/google/cloud/links/base.py +75 -11
  24. airflow/providers/google/cloud/links/bigquery.py +0 -47
  25. airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
  26. airflow/providers/google/cloud/links/bigtable.py +0 -48
  27. airflow/providers/google/cloud/links/cloud_build.py +0 -73
  28. airflow/providers/google/cloud/links/cloud_functions.py +0 -33
  29. airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
  30. airflow/providers/google/cloud/links/cloud_run.py +27 -0
  31. airflow/providers/google/cloud/links/cloud_sql.py +0 -33
  32. airflow/providers/google/cloud/links/cloud_storage_transfer.py +16 -43
  33. airflow/providers/google/cloud/links/cloud_tasks.py +6 -25
  34. airflow/providers/google/cloud/links/compute.py +0 -58
  35. airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
  36. airflow/providers/google/cloud/links/datacatalog.py +23 -54
  37. airflow/providers/google/cloud/links/dataflow.py +0 -34
  38. airflow/providers/google/cloud/links/dataform.py +0 -64
  39. airflow/providers/google/cloud/links/datafusion.py +1 -96
  40. airflow/providers/google/cloud/links/dataplex.py +0 -154
  41. airflow/providers/google/cloud/links/dataprep.py +0 -24
  42. airflow/providers/google/cloud/links/dataproc.py +14 -90
  43. airflow/providers/google/cloud/links/datastore.py +0 -31
  44. airflow/providers/google/cloud/links/kubernetes_engine.py +5 -59
  45. airflow/providers/google/cloud/links/life_sciences.py +0 -19
  46. airflow/providers/google/cloud/links/managed_kafka.py +0 -70
  47. airflow/providers/google/cloud/links/mlengine.py +0 -70
  48. airflow/providers/google/cloud/links/pubsub.py +0 -32
  49. airflow/providers/google/cloud/links/spanner.py +0 -33
  50. airflow/providers/google/cloud/links/stackdriver.py +0 -30
  51. airflow/providers/google/cloud/links/translate.py +16 -186
  52. airflow/providers/google/cloud/links/vertex_ai.py +8 -224
  53. airflow/providers/google/cloud/links/workflows.py +0 -52
  54. airflow/providers/google/cloud/log/gcs_task_handler.py +4 -4
  55. airflow/providers/google/cloud/operators/alloy_db.py +69 -54
  56. airflow/providers/google/cloud/operators/automl.py +16 -14
  57. airflow/providers/google/cloud/operators/bigquery.py +49 -25
  58. airflow/providers/google/cloud/operators/bigquery_dts.py +2 -4
  59. airflow/providers/google/cloud/operators/bigtable.py +35 -6
  60. airflow/providers/google/cloud/operators/cloud_base.py +21 -1
  61. airflow/providers/google/cloud/operators/cloud_build.py +74 -31
  62. airflow/providers/google/cloud/operators/cloud_composer.py +34 -35
  63. airflow/providers/google/cloud/operators/cloud_memorystore.py +68 -42
  64. airflow/providers/google/cloud/operators/cloud_run.py +9 -1
  65. airflow/providers/google/cloud/operators/cloud_sql.py +11 -15
  66. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +0 -2
  67. airflow/providers/google/cloud/operators/compute.py +7 -39
  68. airflow/providers/google/cloud/operators/datacatalog.py +156 -20
  69. airflow/providers/google/cloud/operators/dataflow.py +37 -14
  70. airflow/providers/google/cloud/operators/dataform.py +14 -4
  71. airflow/providers/google/cloud/operators/datafusion.py +4 -12
  72. airflow/providers/google/cloud/operators/dataplex.py +180 -96
  73. airflow/providers/google/cloud/operators/dataprep.py +0 -4
  74. airflow/providers/google/cloud/operators/dataproc.py +10 -16
  75. airflow/providers/google/cloud/operators/dataproc_metastore.py +95 -87
  76. airflow/providers/google/cloud/operators/datastore.py +21 -5
  77. airflow/providers/google/cloud/operators/dlp.py +3 -26
  78. airflow/providers/google/cloud/operators/functions.py +15 -6
  79. airflow/providers/google/cloud/operators/gcs.py +1 -7
  80. airflow/providers/google/cloud/operators/kubernetes_engine.py +53 -92
  81. airflow/providers/google/cloud/operators/life_sciences.py +0 -1
  82. airflow/providers/google/cloud/operators/managed_kafka.py +106 -51
  83. airflow/providers/google/cloud/operators/mlengine.py +0 -1
  84. airflow/providers/google/cloud/operators/pubsub.py +4 -5
  85. airflow/providers/google/cloud/operators/spanner.py +0 -4
  86. airflow/providers/google/cloud/operators/speech_to_text.py +0 -1
  87. airflow/providers/google/cloud/operators/stackdriver.py +0 -8
  88. airflow/providers/google/cloud/operators/tasks.py +0 -11
  89. airflow/providers/google/cloud/operators/text_to_speech.py +0 -1
  90. airflow/providers/google/cloud/operators/translate.py +37 -13
  91. airflow/providers/google/cloud/operators/translate_speech.py +0 -1
  92. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +31 -18
  93. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +28 -8
  94. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +38 -25
  95. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +69 -7
  96. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +42 -8
  97. airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +531 -0
  98. airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +93 -117
  99. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +10 -8
  100. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +56 -10
  101. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +29 -6
  102. airflow/providers/google/cloud/operators/vertex_ai/ray.py +9 -6
  103. airflow/providers/google/cloud/operators/workflows.py +1 -9
  104. airflow/providers/google/cloud/sensors/bigquery.py +1 -1
  105. airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -1
  106. airflow/providers/google/cloud/sensors/bigtable.py +15 -3
  107. airflow/providers/google/cloud/sensors/cloud_composer.py +6 -1
  108. airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -1
  109. airflow/providers/google/cloud/sensors/dataflow.py +3 -3
  110. airflow/providers/google/cloud/sensors/dataform.py +6 -1
  111. airflow/providers/google/cloud/sensors/datafusion.py +6 -1
  112. airflow/providers/google/cloud/sensors/dataplex.py +6 -1
  113. airflow/providers/google/cloud/sensors/dataprep.py +6 -1
  114. airflow/providers/google/cloud/sensors/dataproc.py +6 -1
  115. airflow/providers/google/cloud/sensors/dataproc_metastore.py +6 -1
  116. airflow/providers/google/cloud/sensors/gcs.py +9 -3
  117. airflow/providers/google/cloud/sensors/looker.py +6 -1
  118. airflow/providers/google/cloud/sensors/pubsub.py +8 -3
  119. airflow/providers/google/cloud/sensors/tasks.py +6 -1
  120. airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +6 -1
  121. airflow/providers/google/cloud/sensors/workflows.py +6 -1
  122. airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +1 -1
  123. airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +1 -1
  124. airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +10 -7
  125. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +1 -2
  126. airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +0 -1
  127. airflow/providers/google/cloud/transfers/bigquery_to_sql.py +1 -1
  128. airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
  129. airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +1 -1
  130. airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +2 -2
  131. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +1 -2
  132. airflow/providers/google/cloud/transfers/gcs_to_gcs.py +1 -1
  133. airflow/providers/google/cloud/transfers/gcs_to_local.py +1 -1
  134. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
  135. airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +5 -1
  136. airflow/providers/google/cloud/transfers/gdrive_to_local.py +1 -1
  137. airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
  138. airflow/providers/google/cloud/transfers/local_to_gcs.py +1 -1
  139. airflow/providers/google/cloud/transfers/s3_to_gcs.py +11 -5
  140. airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +1 -1
  141. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
  142. airflow/providers/google/cloud/transfers/sheets_to_gcs.py +2 -2
  143. airflow/providers/google/cloud/transfers/sql_to_gcs.py +1 -1
  144. airflow/providers/google/cloud/triggers/bigquery.py +32 -5
  145. airflow/providers/google/cloud/triggers/dataproc.py +62 -10
  146. airflow/providers/google/cloud/utils/field_validator.py +1 -2
  147. airflow/providers/google/common/auth_backend/google_openid.py +2 -1
  148. airflow/providers/google/common/deprecated.py +2 -1
  149. airflow/providers/google/common/hooks/base_google.py +7 -3
  150. airflow/providers/google/common/links/storage.py +0 -22
  151. airflow/providers/google/firebase/operators/firestore.py +1 -1
  152. airflow/providers/google/get_provider_info.py +14 -16
  153. airflow/providers/google/leveldb/hooks/leveldb.py +30 -1
  154. airflow/providers/google/leveldb/operators/leveldb.py +1 -1
  155. airflow/providers/google/marketing_platform/links/analytics_admin.py +3 -6
  156. airflow/providers/google/marketing_platform/operators/analytics_admin.py +0 -1
  157. airflow/providers/google/marketing_platform/operators/campaign_manager.py +4 -4
  158. airflow/providers/google/marketing_platform/operators/display_video.py +6 -6
  159. airflow/providers/google/marketing_platform/operators/search_ads.py +1 -1
  160. airflow/providers/google/marketing_platform/sensors/campaign_manager.py +6 -1
  161. airflow/providers/google/marketing_platform/sensors/display_video.py +6 -1
  162. airflow/providers/google/suite/operators/sheets.py +3 -3
  163. airflow/providers/google/suite/sensors/drive.py +6 -1
  164. airflow/providers/google/suite/transfers/gcs_to_gdrive.py +1 -1
  165. airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
  166. airflow/providers/google/suite/transfers/local_to_drive.py +1 -1
  167. airflow/providers/google/version_compat.py +28 -0
  168. {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/METADATA +35 -35
  169. {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/RECORD +171 -170
  170. airflow/providers/google/cloud/links/automl.py +0 -193
  171. {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/WHEEL +0 -0
  172. {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/entry_points.txt +0 -0
@@ -153,21 +153,24 @@ class AutoMLTrainModelOperator(GoogleCloudBaseOperator):
153
153
  project_id = self.project_id or hook.project_id
154
154
  if project_id:
155
155
  TranslationLegacyModelTrainLink.persist(
156
- context=context, task_instance=self, project_id=project_id
156
+ context=context,
157
+ dataset_id=self.model["dataset_id"],
158
+ project_id=project_id,
159
+ location=self.location,
157
160
  )
158
161
  operation_result = hook.wait_for_operation(timeout=self.timeout, operation=operation)
159
162
  result = Model.to_dict(operation_result)
160
163
  model_id = hook.extract_object_id(result)
161
164
  self.log.info("Model is created, model_id: %s", model_id)
162
165
 
163
- self.xcom_push(context, key="model_id", value=model_id)
166
+ context["task_instance"].xcom_push(key="model_id", value=model_id)
164
167
  if project_id:
165
168
  TranslationLegacyModelLink.persist(
166
169
  context=context,
167
- task_instance=self,
168
170
  dataset_id=self.model["dataset_id"] or "-",
169
171
  model_id=model_id,
170
172
  project_id=project_id,
173
+ location=self.location,
171
174
  )
172
175
  return result
173
176
 
@@ -313,10 +316,10 @@ class AutoMLPredictOperator(GoogleCloudBaseOperator):
313
316
  if project_id and self.model_id and dataset_id:
314
317
  TranslationLegacyModelPredictLink.persist(
315
318
  context=context,
316
- task_instance=self,
317
319
  model_id=self.model_id,
318
320
  dataset_id=dataset_id,
319
321
  project_id=project_id,
322
+ location=self.location,
320
323
  )
321
324
  return PredictResponse.to_dict(result)
322
325
 
@@ -412,14 +415,14 @@ class AutoMLCreateDatasetOperator(GoogleCloudBaseOperator):
412
415
  dataset_id = hook.extract_object_id(result)
413
416
  self.log.info("Creating completed. Dataset id: %s", dataset_id)
414
417
 
415
- self.xcom_push(context, key="dataset_id", value=dataset_id)
418
+ context["task_instance"].xcom_push(key="dataset_id", value=dataset_id)
416
419
  project_id = self.project_id or hook.project_id
417
420
  if project_id:
418
421
  TranslationLegacyDatasetLink.persist(
419
422
  context=context,
420
- task_instance=self,
421
423
  dataset_id=dataset_id,
422
424
  project_id=project_id,
425
+ location=self.location,
423
426
  )
424
427
  return result
425
428
 
@@ -530,9 +533,9 @@ class AutoMLImportDataOperator(GoogleCloudBaseOperator):
530
533
  if project_id:
531
534
  TranslationLegacyDatasetLink.persist(
532
535
  context=context,
533
- task_instance=self,
534
536
  dataset_id=self.dataset_id,
535
537
  project_id=project_id,
538
+ location=self.location,
536
539
  )
537
540
 
538
541
 
@@ -649,9 +652,9 @@ class AutoMLTablesListColumnSpecsOperator(GoogleCloudBaseOperator):
649
652
  if project_id:
650
653
  TranslationLegacyDatasetLink.persist(
651
654
  context=context,
652
- task_instance=self,
653
655
  dataset_id=self.dataset_id,
654
656
  project_id=project_id,
657
+ location=self.location,
655
658
  )
656
659
  return result
657
660
 
@@ -749,9 +752,9 @@ class AutoMLTablesUpdateDatasetOperator(GoogleCloudBaseOperator):
749
752
  if project_id:
750
753
  TranslationLegacyDatasetLink.persist(
751
754
  context=context,
752
- task_instance=self,
753
755
  dataset_id=hook.extract_object_id(self.dataset),
754
756
  project_id=project_id,
757
+ location=self.location,
755
758
  )
756
759
  return Dataset.to_dict(result)
757
760
 
@@ -845,10 +848,10 @@ class AutoMLGetModelOperator(GoogleCloudBaseOperator):
845
848
  if project_id:
846
849
  TranslationLegacyModelLink.persist(
847
850
  context=context,
848
- task_instance=self,
849
851
  dataset_id=model["dataset_id"],
850
852
  model_id=self.model_id,
851
853
  project_id=project_id,
854
+ location=self.location,
852
855
  )
853
856
  return model
854
857
 
@@ -1154,9 +1157,9 @@ class AutoMLTablesListTableSpecsOperator(GoogleCloudBaseOperator):
1154
1157
  if project_id:
1155
1158
  TranslationLegacyDatasetLink.persist(
1156
1159
  context=context,
1157
- task_instance=self,
1158
1160
  dataset_id=self.dataset_id,
1159
1161
  project_id=project_id,
1162
+ location=self.location,
1160
1163
  )
1161
1164
  return result
1162
1165
 
@@ -1245,14 +1248,13 @@ class AutoMLListDatasetOperator(GoogleCloudBaseOperator):
1245
1248
  result.append(Dataset.to_dict(dataset))
1246
1249
  self.log.info("Datasets obtained.")
1247
1250
 
1248
- self.xcom_push(
1249
- context,
1251
+ context["task_instance"].xcom_push(
1250
1252
  key="dataset_id_list",
1251
1253
  value=[hook.extract_object_id(d) for d in result],
1252
1254
  )
1253
1255
  project_id = self.project_id or hook.project_id
1254
1256
  if project_id:
1255
- TranslationDatasetListLink.persist(context=context, task_instance=self, project_id=project_id)
1257
+ TranslationDatasetListLink.persist(context=context, project_id=project_id)
1256
1258
  return result
1257
1259
 
1258
1260
 
@@ -93,16 +93,32 @@ class IfExistAction(enum.Enum):
93
93
  SKIP = "skip"
94
94
 
95
95
 
96
+ class _BigQueryHookWithFlexibleProjectId(BigQueryHook):
97
+ @property
98
+ def project_id(self) -> str:
99
+ _, project_id = self.get_credentials_and_project_id()
100
+ return project_id or PROVIDE_PROJECT_ID
101
+
102
+ @project_id.setter
103
+ def project_id(self, value: str) -> None:
104
+ cached_creds, _ = self.get_credentials_and_project_id()
105
+ self._cached_project_id = value or PROVIDE_PROJECT_ID
106
+ self._cached_credntials = cached_creds
107
+
108
+
96
109
  class _BigQueryDbHookMixin:
97
- def get_db_hook(self: BigQueryCheckOperator) -> BigQueryHook: # type:ignore[misc]
110
+ def get_db_hook(self: BigQueryCheckOperator) -> _BigQueryHookWithFlexibleProjectId: # type:ignore[misc]
98
111
  """Get BigQuery DB Hook."""
99
- return BigQueryHook(
112
+ hook = _BigQueryHookWithFlexibleProjectId(
100
113
  gcp_conn_id=self.gcp_conn_id,
101
114
  use_legacy_sql=self.use_legacy_sql,
102
115
  location=self.location,
103
116
  impersonation_chain=self.impersonation_chain,
104
117
  labels=self.labels,
105
118
  )
119
+ if self.project_id:
120
+ hook.project_id = self.project_id
121
+ return hook
106
122
 
107
123
 
108
124
  class _BigQueryOperatorsEncryptionConfigurationMixin:
@@ -190,6 +206,7 @@ class BigQueryCheckOperator(
190
206
  https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs.
191
207
  For example, [{ 'name': 'corpus', 'parameterType': { 'type': 'STRING' },
192
208
  'parameterValue': { 'value': 'romeoandjuliet' } }]. (templated)
209
+ :param project_id: Google Cloud Project where the job is running
193
210
  """
194
211
 
195
212
  template_fields: Sequence[str] = (
@@ -208,6 +225,7 @@ class BigQueryCheckOperator(
208
225
  *,
209
226
  sql: str,
210
227
  gcp_conn_id: str = "google_cloud_default",
228
+ project_id: str = PROVIDE_PROJECT_ID,
211
229
  use_legacy_sql: bool = True,
212
230
  location: str | None = None,
213
231
  impersonation_chain: str | Sequence[str] | None = None,
@@ -228,6 +246,7 @@ class BigQueryCheckOperator(
228
246
  self.deferrable = deferrable
229
247
  self.poll_interval = poll_interval
230
248
  self.query_params = query_params
249
+ self.project_id = project_id
231
250
 
232
251
  def _submit_job(
233
252
  self,
@@ -243,7 +262,7 @@ class BigQueryCheckOperator(
243
262
 
244
263
  return hook.insert_job(
245
264
  configuration=configuration,
246
- project_id=hook.project_id,
265
+ project_id=self.project_id,
247
266
  location=self.location,
248
267
  job_id=job_id,
249
268
  nowait=True,
@@ -257,6 +276,8 @@ class BigQueryCheckOperator(
257
276
  gcp_conn_id=self.gcp_conn_id,
258
277
  impersonation_chain=self.impersonation_chain,
259
278
  )
279
+ if self.project_id is None:
280
+ self.project_id = hook.project_id
260
281
  job = self._submit_job(hook, job_id="")
261
282
  context["ti"].xcom_push(key="job_id", value=job.job_id)
262
283
  if job.running():
@@ -265,7 +286,7 @@ class BigQueryCheckOperator(
265
286
  trigger=BigQueryCheckTrigger(
266
287
  conn_id=self.gcp_conn_id,
267
288
  job_id=job.job_id,
268
- project_id=hook.project_id,
289
+ project_id=self.project_id,
269
290
  location=self.location or hook.location,
270
291
  poll_interval=self.poll_interval,
271
292
  impersonation_chain=self.impersonation_chain,
@@ -342,6 +363,7 @@ class BigQueryValueCheckOperator(
342
363
  :param deferrable: Run operator in the deferrable mode.
343
364
  :param poll_interval: (Deferrable mode only) polling period in seconds to
344
365
  check for the status of job.
366
+ :param project_id: Google Cloud Project where the job is running
345
367
  """
346
368
 
347
369
  template_fields: Sequence[str] = (
@@ -363,6 +385,7 @@ class BigQueryValueCheckOperator(
363
385
  tolerance: Any = None,
364
386
  encryption_configuration: dict | None = None,
365
387
  gcp_conn_id: str = "google_cloud_default",
388
+ project_id: str = PROVIDE_PROJECT_ID,
366
389
  use_legacy_sql: bool = True,
367
390
  location: str | None = None,
368
391
  impersonation_chain: str | Sequence[str] | None = None,
@@ -380,6 +403,7 @@ class BigQueryValueCheckOperator(
380
403
  self.labels = labels
381
404
  self.deferrable = deferrable
382
405
  self.poll_interval = poll_interval
406
+ self.project_id = project_id
383
407
 
384
408
  def _submit_job(
385
409
  self,
@@ -398,7 +422,7 @@ class BigQueryValueCheckOperator(
398
422
 
399
423
  return hook.insert_job(
400
424
  configuration=configuration,
401
- project_id=hook.project_id,
425
+ project_id=self.project_id,
402
426
  location=self.location,
403
427
  job_id=job_id,
404
428
  nowait=True,
@@ -409,7 +433,8 @@ class BigQueryValueCheckOperator(
409
433
  super().execute(context=context)
410
434
  else:
411
435
  hook = BigQueryHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
412
-
436
+ if self.project_id is None:
437
+ self.project_id = hook.project_id
413
438
  job = self._submit_job(hook, job_id="")
414
439
  context["ti"].xcom_push(key="job_id", value=job.job_id)
415
440
  if job.running():
@@ -418,7 +443,7 @@ class BigQueryValueCheckOperator(
418
443
  trigger=BigQueryValueCheckTrigger(
419
444
  conn_id=self.gcp_conn_id,
420
445
  job_id=job.job_id,
421
- project_id=hook.project_id,
446
+ project_id=self.project_id,
422
447
  location=self.location or hook.location,
423
448
  sql=self.sql,
424
449
  pass_value=self.pass_value,
@@ -575,6 +600,9 @@ class BigQueryIntervalCheckOperator(
575
600
  hook = BigQueryHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
576
601
  self.log.info("Using ratio formula: %s", self.ratio_formula)
577
602
 
603
+ if self.project_id is None:
604
+ self.project_id = hook.project_id
605
+
578
606
  self.log.info("Executing SQL check: %s", self.sql1)
579
607
  job_1 = self._submit_job(hook, sql=self.sql1, job_id="")
580
608
  context["ti"].xcom_push(key="job_id", value=job_1.job_id)
@@ -587,7 +615,7 @@ class BigQueryIntervalCheckOperator(
587
615
  conn_id=self.gcp_conn_id,
588
616
  first_job_id=job_1.job_id,
589
617
  second_job_id=job_2.job_id,
590
- project_id=hook.project_id,
618
+ project_id=self.project_id,
591
619
  table=self.table,
592
620
  location=self.location or hook.location,
593
621
  metrics_thresholds=self.metrics_thresholds,
@@ -654,6 +682,7 @@ class BigQueryColumnCheckOperator(
654
682
  Service Account Token Creator IAM role to the directly preceding identity, with first
655
683
  account from the list granting this role to the originating account (templated).
656
684
  :param labels: a dictionary containing labels for the table, passed to BigQuery
685
+ :param project_id: Google Cloud Project where the job is running
657
686
  """
658
687
 
659
688
  template_fields: Sequence[str] = tuple(set(SQLColumnCheckOperator.template_fields) | {"gcp_conn_id"})
@@ -670,6 +699,7 @@ class BigQueryColumnCheckOperator(
670
699
  accept_none: bool = True,
671
700
  encryption_configuration: dict | None = None,
672
701
  gcp_conn_id: str = "google_cloud_default",
702
+ project_id: str = PROVIDE_PROJECT_ID,
673
703
  use_legacy_sql: bool = True,
674
704
  location: str | None = None,
675
705
  impersonation_chain: str | Sequence[str] | None = None,
@@ -695,6 +725,7 @@ class BigQueryColumnCheckOperator(
695
725
  self.location = location
696
726
  self.impersonation_chain = impersonation_chain
697
727
  self.labels = labels
728
+ self.project_id = project_id
698
729
 
699
730
  def _submit_job(
700
731
  self,
@@ -706,7 +737,7 @@ class BigQueryColumnCheckOperator(
706
737
  self.include_encryption_configuration(configuration, "query")
707
738
  return hook.insert_job(
708
739
  configuration=configuration,
709
- project_id=hook.project_id,
740
+ project_id=self.project_id,
710
741
  location=self.location,
711
742
  job_id=job_id,
712
743
  nowait=False,
@@ -715,6 +746,9 @@ class BigQueryColumnCheckOperator(
715
746
  def execute(self, context=None):
716
747
  """Perform checks on the given columns."""
717
748
  hook = self.get_db_hook()
749
+
750
+ if self.project_id is None:
751
+ self.project_id = hook.project_id
718
752
  failed_tests = []
719
753
 
720
754
  job = self._submit_job(hook, job_id="")
@@ -786,6 +820,7 @@ class BigQueryTableCheckOperator(
786
820
  account from the list granting this role to the originating account (templated).
787
821
  :param labels: a dictionary containing labels for the table, passed to BigQuery
788
822
  :param encryption_configuration: (Optional) Custom encryption configuration (e.g., Cloud KMS keys).
823
+ :param project_id: Google Cloud Project where the job is running
789
824
 
790
825
  .. code-block:: python
791
826
 
@@ -805,6 +840,7 @@ class BigQueryTableCheckOperator(
805
840
  checks: dict,
806
841
  partition_clause: str | None = None,
807
842
  gcp_conn_id: str = "google_cloud_default",
843
+ project_id: str = PROVIDE_PROJECT_ID,
808
844
  use_legacy_sql: bool = True,
809
845
  location: str | None = None,
810
846
  impersonation_chain: str | Sequence[str] | None = None,
@@ -819,6 +855,7 @@ class BigQueryTableCheckOperator(
819
855
  self.impersonation_chain = impersonation_chain
820
856
  self.labels = labels
821
857
  self.encryption_configuration = encryption_configuration
858
+ self.project_id = project_id
822
859
 
823
860
  def _submit_job(
824
861
  self,
@@ -832,7 +869,7 @@ class BigQueryTableCheckOperator(
832
869
 
833
870
  return hook.insert_job(
834
871
  configuration=configuration,
835
- project_id=hook.project_id,
872
+ project_id=self.project_id,
836
873
  location=self.location,
837
874
  job_id=job_id,
838
875
  nowait=False,
@@ -841,6 +878,8 @@ class BigQueryTableCheckOperator(
841
878
  def execute(self, context=None):
842
879
  """Execute the given checks on the table."""
843
880
  hook = self.get_db_hook()
881
+ if self.project_id is None:
882
+ self.project_id = hook.project_id
844
883
  job = self._submit_job(hook, job_id="")
845
884
  context["ti"].xcom_push(key="job_id", value=job.job_id)
846
885
  records = job.result().to_dataframe()
@@ -1285,7 +1324,6 @@ class BigQueryCreateTableOperator(GoogleCloudBaseOperator):
1285
1324
  if self._table:
1286
1325
  persist_kwargs = {
1287
1326
  "context": context,
1288
- "task_instance": self,
1289
1327
  "project_id": self._table.to_api_repr()["tableReference"]["projectId"],
1290
1328
  "dataset_id": self._table.to_api_repr()["tableReference"]["datasetId"],
1291
1329
  "table_id": self._table.to_api_repr()["tableReference"]["tableId"],
@@ -1304,7 +1342,6 @@ class BigQueryCreateTableOperator(GoogleCloudBaseOperator):
1304
1342
  self.log.info(error_msg)
1305
1343
  persist_kwargs = {
1306
1344
  "context": context,
1307
- "task_instance": self,
1308
1345
  "project_id": self.project_id or bq_hook.project_id,
1309
1346
  "dataset_id": self.dataset_id,
1310
1347
  "table_id": self.table_id,
@@ -1569,7 +1606,6 @@ class BigQueryCreateEmptyTableOperator(GoogleCloudBaseOperator):
1569
1606
  if self._table:
1570
1607
  persist_kwargs = {
1571
1608
  "context": context,
1572
- "task_instance": self,
1573
1609
  "project_id": self._table.to_api_repr()["tableReference"]["projectId"],
1574
1610
  "dataset_id": self._table.to_api_repr()["tableReference"]["datasetId"],
1575
1611
  "table_id": self._table.to_api_repr()["tableReference"]["tableId"],
@@ -1588,7 +1624,6 @@ class BigQueryCreateEmptyTableOperator(GoogleCloudBaseOperator):
1588
1624
  self.log.info(error_msg)
1589
1625
  persist_kwargs = {
1590
1626
  "context": context,
1591
- "task_instance": self,
1592
1627
  "project_id": self.project_id or bq_hook.project_id,
1593
1628
  "dataset_id": self.dataset_id,
1594
1629
  "table_id": self.table_id,
@@ -1859,7 +1894,6 @@ class BigQueryCreateExternalTableOperator(GoogleCloudBaseOperator):
1859
1894
  if self._table:
1860
1895
  BigQueryTableLink.persist(
1861
1896
  context=context,
1862
- task_instance=self,
1863
1897
  dataset_id=self._table.dataset_id,
1864
1898
  project_id=self._table.project,
1865
1899
  table_id=self._table.table_id,
@@ -1918,7 +1952,6 @@ class BigQueryCreateExternalTableOperator(GoogleCloudBaseOperator):
1918
1952
  if self._table:
1919
1953
  BigQueryTableLink.persist(
1920
1954
  context=context,
1921
- task_instance=self,
1922
1955
  dataset_id=self._table.dataset_id,
1923
1956
  project_id=self._table.project,
1924
1957
  table_id=self._table.table_id,
@@ -2116,7 +2149,6 @@ class BigQueryCreateEmptyDatasetOperator(GoogleCloudBaseOperator):
2116
2149
  )
2117
2150
  persist_kwargs = {
2118
2151
  "context": context,
2119
- "task_instance": self,
2120
2152
  "project_id": dataset["datasetReference"]["projectId"],
2121
2153
  "dataset_id": dataset["datasetReference"]["datasetId"],
2122
2154
  }
@@ -2128,7 +2160,6 @@ class BigQueryCreateEmptyDatasetOperator(GoogleCloudBaseOperator):
2128
2160
  )
2129
2161
  persist_kwargs = {
2130
2162
  "context": context,
2131
- "task_instance": self,
2132
2163
  "project_id": project_id,
2133
2164
  "dataset_id": dataset_id,
2134
2165
  }
@@ -2200,7 +2231,6 @@ class BigQueryGetDatasetOperator(GoogleCloudBaseOperator):
2200
2231
  dataset_api_repr = dataset.to_api_repr()
2201
2232
  BigQueryDatasetLink.persist(
2202
2233
  context=context,
2203
- task_instance=self,
2204
2234
  dataset_id=dataset_api_repr["datasetReference"]["datasetId"],
2205
2235
  project_id=dataset_api_repr["datasetReference"]["projectId"],
2206
2236
  )
@@ -2349,7 +2379,6 @@ class BigQueryUpdateTableOperator(GoogleCloudBaseOperator):
2349
2379
  if self._table:
2350
2380
  BigQueryTableLink.persist(
2351
2381
  context=context,
2352
- task_instance=self,
2353
2382
  dataset_id=self._table["tableReference"]["datasetId"],
2354
2383
  project_id=self._table["tableReference"]["projectId"],
2355
2384
  table_id=self._table["tableReference"]["tableId"],
@@ -2452,7 +2481,6 @@ class BigQueryUpdateDatasetOperator(GoogleCloudBaseOperator):
2452
2481
  dataset_api_repr = dataset.to_api_repr()
2453
2482
  BigQueryDatasetLink.persist(
2454
2483
  context=context,
2455
- task_instance=self,
2456
2484
  dataset_id=dataset_api_repr["datasetReference"]["datasetId"],
2457
2485
  project_id=dataset_api_repr["datasetReference"]["projectId"],
2458
2486
  )
@@ -2624,7 +2652,6 @@ class BigQueryUpsertTableOperator(GoogleCloudBaseOperator):
2624
2652
  if self._table:
2625
2653
  BigQueryTableLink.persist(
2626
2654
  context=context,
2627
- task_instance=self,
2628
2655
  dataset_id=self._table["tableReference"]["datasetId"],
2629
2656
  project_id=self._table["tableReference"]["projectId"],
2630
2657
  table_id=self._table["tableReference"]["tableId"],
@@ -2754,7 +2781,6 @@ class BigQueryUpdateTableSchemaOperator(GoogleCloudBaseOperator):
2754
2781
  if self._table:
2755
2782
  BigQueryTableLink.persist(
2756
2783
  context=context,
2757
- task_instance=self,
2758
2784
  dataset_id=self._table["tableReference"]["datasetId"],
2759
2785
  project_id=self._table["tableReference"]["projectId"],
2760
2786
  table_id=self._table["tableReference"]["tableId"],
@@ -3000,7 +3026,6 @@ class BigQueryInsertJobOperator(GoogleCloudBaseOperator, _BigQueryInsertJobOpera
3000
3026
  table = job_configuration[job_type][table_prop]
3001
3027
  persist_kwargs = {
3002
3028
  "context": context,
3003
- "task_instance": self,
3004
3029
  "project_id": self.project_id,
3005
3030
  "table_id": table,
3006
3031
  }
@@ -3022,7 +3047,6 @@ class BigQueryInsertJobOperator(GoogleCloudBaseOperator, _BigQueryInsertJobOpera
3022
3047
 
3023
3048
  persist_kwargs = {
3024
3049
  "context": context,
3025
- "task_instance": self,
3026
3050
  "project_id": self.project_id,
3027
3051
  "location": self.location,
3028
3052
  "job_id": self.job_id,
@@ -134,7 +134,6 @@ class BigQueryCreateDataTransferOperator(GoogleCloudBaseOperator):
134
134
  transfer_config = _get_transfer_config_details(response.name)
135
135
  BigQueryDataTransferConfigLink.persist(
136
136
  context=context,
137
- task_instance=self,
138
137
  region=transfer_config["region"],
139
138
  config_id=transfer_config["config_id"],
140
139
  project_id=transfer_config["project_id"],
@@ -142,7 +141,7 @@ class BigQueryCreateDataTransferOperator(GoogleCloudBaseOperator):
142
141
 
143
142
  result = TransferConfig.to_dict(response)
144
143
  self.log.info("Created DTS transfer config %s", get_object_id(result))
145
- self.xcom_push(context, key="transfer_config_id", value=get_object_id(result))
144
+ context["ti"].xcom_push(key="transfer_config_id", value=get_object_id(result))
146
145
  # don't push AWS secret in XCOM
147
146
  result.get("params", {}).pop("secret_access_key", None)
148
147
  result.get("params", {}).pop("access_key_id", None)
@@ -329,7 +328,6 @@ class BigQueryDataTransferServiceStartTransferRunsOperator(GoogleCloudBaseOperat
329
328
  transfer_config = _get_transfer_config_details(response.runs[0].name)
330
329
  BigQueryDataTransferConfigLink.persist(
331
330
  context=context,
332
- task_instance=self,
333
331
  region=transfer_config["region"],
334
332
  config_id=transfer_config["config_id"],
335
333
  project_id=transfer_config["project_id"],
@@ -337,7 +335,7 @@ class BigQueryDataTransferServiceStartTransferRunsOperator(GoogleCloudBaseOperat
337
335
 
338
336
  result = StartManualTransferRunsResponse.to_dict(response)
339
337
  run_id = get_object_id(result["runs"][0])
340
- self.xcom_push(context, key="run_id", value=run_id)
338
+ context["ti"].xcom_push(key="run_id", value=run_id)
341
339
 
342
340
  if not self.deferrable:
343
341
  # Save as attribute for further use by OpenLineage
@@ -20,7 +20,7 @@
20
20
  from __future__ import annotations
21
21
 
22
22
  from collections.abc import Iterable, Sequence
23
- from typing import TYPE_CHECKING
23
+ from typing import TYPE_CHECKING, Any
24
24
 
25
25
  import google.api_core.exceptions
26
26
 
@@ -142,6 +142,13 @@ class BigtableCreateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
142
142
  self.impersonation_chain = impersonation_chain
143
143
  super().__init__(**kwargs)
144
144
 
145
+ @property
146
+ def extra_links_params(self) -> dict[str, Any]:
147
+ return {
148
+ "instance_id": self.instance_id,
149
+ "project_id": self.project_id,
150
+ }
151
+
145
152
  def execute(self, context: Context) -> None:
146
153
  hook = BigtableHook(
147
154
  gcp_conn_id=self.gcp_conn_id,
@@ -155,7 +162,7 @@ class BigtableCreateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
155
162
  "The instance '%s' already exists in this project. Consider it as created",
156
163
  self.instance_id,
157
164
  )
158
- BigtableInstanceLink.persist(context=context, task_instance=self)
165
+ BigtableInstanceLink.persist(context=context)
159
166
  return
160
167
  try:
161
168
  hook.create_instance(
@@ -171,7 +178,7 @@ class BigtableCreateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
171
178
  cluster_storage_type=self.cluster_storage_type,
172
179
  timeout=self.timeout,
173
180
  )
174
- BigtableInstanceLink.persist(context=context, task_instance=self)
181
+ BigtableInstanceLink.persist(context=context)
175
182
  except google.api_core.exceptions.GoogleAPICallError as e:
176
183
  self.log.error("An error occurred. Exiting.")
177
184
  raise e
@@ -240,6 +247,13 @@ class BigtableUpdateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
240
247
  self.impersonation_chain = impersonation_chain
241
248
  super().__init__(**kwargs)
242
249
 
250
+ @property
251
+ def extra_links_params(self) -> dict[str, Any]:
252
+ return {
253
+ "instance_id": self.instance_id,
254
+ "project_id": self.project_id,
255
+ }
256
+
243
257
  def execute(self, context: Context) -> None:
244
258
  hook = BigtableHook(
245
259
  gcp_conn_id=self.gcp_conn_id,
@@ -258,7 +272,7 @@ class BigtableUpdateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
258
272
  instance_labels=self.instance_labels,
259
273
  timeout=self.timeout,
260
274
  )
261
- BigtableInstanceLink.persist(context=context, task_instance=self)
275
+ BigtableInstanceLink.persist(context=context)
262
276
  except google.api_core.exceptions.GoogleAPICallError as e:
263
277
  self.log.error("An error occurred. Exiting.")
264
278
  raise e
@@ -414,6 +428,13 @@ class BigtableCreateTableOperator(GoogleCloudBaseOperator, BigtableValidationMix
414
428
  return False
415
429
  return True
416
430
 
431
+ @property
432
+ def extra_links_params(self) -> dict[str, Any]:
433
+ return {
434
+ "instance_id": self.instance_id,
435
+ "project_id": self.project_id,
436
+ }
437
+
417
438
  def execute(self, context: Context) -> None:
418
439
  hook = BigtableHook(
419
440
  gcp_conn_id=self.gcp_conn_id,
@@ -431,7 +452,7 @@ class BigtableCreateTableOperator(GoogleCloudBaseOperator, BigtableValidationMix
431
452
  initial_split_keys=self.initial_split_keys,
432
453
  column_families=self.column_families,
433
454
  )
434
- BigtableTablesLink.persist(context=context, task_instance=self)
455
+ BigtableTablesLink.persist(context=context)
435
456
  except google.api_core.exceptions.AlreadyExists:
436
457
  if not self._compare_column_families(hook, instance):
437
458
  raise AirflowException(
@@ -575,6 +596,14 @@ class BigtableUpdateClusterOperator(GoogleCloudBaseOperator, BigtableValidationM
575
596
  self.impersonation_chain = impersonation_chain
576
597
  super().__init__(**kwargs)
577
598
 
599
+ @property
600
+ def extra_links_params(self) -> dict[str, Any]:
601
+ return {
602
+ "instance_id": self.instance_id,
603
+ "cluster_id": self.cluster_id,
604
+ "project_id": self.project_id,
605
+ }
606
+
578
607
  def execute(self, context: Context) -> None:
579
608
  hook = BigtableHook(
580
609
  gcp_conn_id=self.gcp_conn_id,
@@ -586,7 +615,7 @@ class BigtableUpdateClusterOperator(GoogleCloudBaseOperator, BigtableValidationM
586
615
 
587
616
  try:
588
617
  hook.update_cluster(instance=instance, cluster_id=self.cluster_id, nodes=self.nodes)
589
- BigtableClusterLink.persist(context=context, task_instance=self)
618
+ BigtableClusterLink.persist(context=context)
590
619
  except google.api_core.exceptions.NotFound:
591
620
  raise AirflowException(
592
621
  f"Dependency: cluster '{self.cluster_id}' does not exist for instance '{self.instance_id}'."
@@ -19,9 +19,11 @@
19
19
 
20
20
  from __future__ import annotations
21
21
 
22
+ from typing import Any
23
+
22
24
  from google.api_core.gapic_v1.method import DEFAULT
23
25
 
24
- from airflow.models import BaseOperator
26
+ from airflow.providers.google.version_compat import BaseOperator
25
27
 
26
28
 
27
29
  class GoogleCloudBaseOperator(BaseOperator):
@@ -36,3 +38,21 @@ class GoogleCloudBaseOperator(BaseOperator):
36
38
  """
37
39
  memo[id(DEFAULT)] = DEFAULT
38
40
  return super().__deepcopy__(memo)
41
+
42
+ @property
43
+ def extra_links_params(self) -> dict[str, Any]:
44
+ """
45
+ Override this method to include parameters for link formatting in extra links.
46
+
47
+ For example; most of the links on the Google provider require `project_id` and `location` in the Link.
48
+ To be not repeat; you can override this function and return something like the following:
49
+
50
+ .. code-block:: python
51
+
52
+ {
53
+ "project_id": self.project_id,
54
+ "location": self.location,
55
+ }
56
+
57
+ """
58
+ return {}