apache-airflow-providers-google 16.0.0a1__py3-none-any.whl → 16.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/ads/hooks/ads.py +43 -5
- airflow/providers/google/ads/operators/ads.py +1 -1
- airflow/providers/google/ads/transfers/ads_to_gcs.py +1 -1
- airflow/providers/google/cloud/hooks/bigquery.py +63 -77
- airflow/providers/google/cloud/hooks/cloud_sql.py +8 -4
- airflow/providers/google/cloud/hooks/datacatalog.py +9 -1
- airflow/providers/google/cloud/hooks/dataflow.py +2 -2
- airflow/providers/google/cloud/hooks/dataplex.py +1 -1
- airflow/providers/google/cloud/hooks/dataprep.py +4 -1
- airflow/providers/google/cloud/hooks/gcs.py +5 -5
- airflow/providers/google/cloud/hooks/looker.py +10 -1
- airflow/providers/google/cloud/hooks/mlengine.py +2 -1
- airflow/providers/google/cloud/hooks/secret_manager.py +102 -10
- airflow/providers/google/cloud/hooks/spanner.py +2 -2
- airflow/providers/google/cloud/hooks/translate.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +0 -36
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +307 -7
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +44 -80
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +11 -2
- airflow/providers/google/cloud/hooks/vision.py +2 -2
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +75 -11
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/cloud_run.py +27 -0
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +16 -43
- airflow/providers/google/cloud/links/cloud_tasks.py +6 -25
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -96
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +14 -90
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +5 -59
- airflow/providers/google/cloud/links/life_sciences.py +0 -19
- airflow/providers/google/cloud/links/managed_kafka.py +0 -70
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +16 -186
- airflow/providers/google/cloud/links/vertex_ai.py +8 -224
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/log/gcs_task_handler.py +4 -4
- airflow/providers/google/cloud/operators/alloy_db.py +69 -54
- airflow/providers/google/cloud/operators/automl.py +16 -14
- airflow/providers/google/cloud/operators/bigquery.py +49 -25
- airflow/providers/google/cloud/operators/bigquery_dts.py +2 -4
- airflow/providers/google/cloud/operators/bigtable.py +35 -6
- airflow/providers/google/cloud/operators/cloud_base.py +21 -1
- airflow/providers/google/cloud/operators/cloud_build.py +74 -31
- airflow/providers/google/cloud/operators/cloud_composer.py +34 -35
- airflow/providers/google/cloud/operators/cloud_memorystore.py +68 -42
- airflow/providers/google/cloud/operators/cloud_run.py +9 -1
- airflow/providers/google/cloud/operators/cloud_sql.py +11 -15
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +0 -2
- airflow/providers/google/cloud/operators/compute.py +7 -39
- airflow/providers/google/cloud/operators/datacatalog.py +156 -20
- airflow/providers/google/cloud/operators/dataflow.py +37 -14
- airflow/providers/google/cloud/operators/dataform.py +14 -4
- airflow/providers/google/cloud/operators/datafusion.py +4 -12
- airflow/providers/google/cloud/operators/dataplex.py +180 -96
- airflow/providers/google/cloud/operators/dataprep.py +0 -4
- airflow/providers/google/cloud/operators/dataproc.py +10 -16
- airflow/providers/google/cloud/operators/dataproc_metastore.py +95 -87
- airflow/providers/google/cloud/operators/datastore.py +21 -5
- airflow/providers/google/cloud/operators/dlp.py +3 -26
- airflow/providers/google/cloud/operators/functions.py +15 -6
- airflow/providers/google/cloud/operators/gcs.py +1 -7
- airflow/providers/google/cloud/operators/kubernetes_engine.py +53 -92
- airflow/providers/google/cloud/operators/life_sciences.py +0 -1
- airflow/providers/google/cloud/operators/managed_kafka.py +106 -51
- airflow/providers/google/cloud/operators/mlengine.py +0 -1
- airflow/providers/google/cloud/operators/pubsub.py +4 -5
- airflow/providers/google/cloud/operators/spanner.py +0 -4
- airflow/providers/google/cloud/operators/speech_to_text.py +0 -1
- airflow/providers/google/cloud/operators/stackdriver.py +0 -8
- airflow/providers/google/cloud/operators/tasks.py +0 -11
- airflow/providers/google/cloud/operators/text_to_speech.py +0 -1
- airflow/providers/google/cloud/operators/translate.py +37 -13
- airflow/providers/google/cloud/operators/translate_speech.py +0 -1
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +31 -18
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +28 -8
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +38 -25
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +69 -7
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +42 -8
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +531 -0
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +93 -117
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +10 -8
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +56 -10
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +29 -6
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +9 -6
- airflow/providers/google/cloud/operators/workflows.py +1 -9
- airflow/providers/google/cloud/sensors/bigquery.py +1 -1
- airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -1
- airflow/providers/google/cloud/sensors/bigtable.py +15 -3
- airflow/providers/google/cloud/sensors/cloud_composer.py +6 -1
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -1
- airflow/providers/google/cloud/sensors/dataflow.py +3 -3
- airflow/providers/google/cloud/sensors/dataform.py +6 -1
- airflow/providers/google/cloud/sensors/datafusion.py +6 -1
- airflow/providers/google/cloud/sensors/dataplex.py +6 -1
- airflow/providers/google/cloud/sensors/dataprep.py +6 -1
- airflow/providers/google/cloud/sensors/dataproc.py +6 -1
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +6 -1
- airflow/providers/google/cloud/sensors/gcs.py +9 -3
- airflow/providers/google/cloud/sensors/looker.py +6 -1
- airflow/providers/google/cloud/sensors/pubsub.py +8 -3
- airflow/providers/google/cloud/sensors/tasks.py +6 -1
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +6 -1
- airflow/providers/google/cloud/sensors/workflows.py +6 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +10 -7
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +1 -2
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +0 -1
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +1 -1
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +1 -2
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/gcs_to_local.py +1 -1
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +5 -1
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +1 -1
- airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
- airflow/providers/google/cloud/transfers/local_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +11 -5
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +1 -1
- airflow/providers/google/cloud/triggers/bigquery.py +32 -5
- airflow/providers/google/cloud/triggers/dataproc.py +62 -10
- airflow/providers/google/cloud/utils/field_validator.py +1 -2
- airflow/providers/google/common/auth_backend/google_openid.py +2 -1
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +7 -3
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/firebase/operators/firestore.py +1 -1
- airflow/providers/google/get_provider_info.py +14 -16
- airflow/providers/google/leveldb/hooks/leveldb.py +30 -1
- airflow/providers/google/leveldb/operators/leveldb.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +3 -6
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +0 -1
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +4 -4
- airflow/providers/google/marketing_platform/operators/display_video.py +6 -6
- airflow/providers/google/marketing_platform/operators/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +6 -1
- airflow/providers/google/marketing_platform/sensors/display_video.py +6 -1
- airflow/providers/google/suite/operators/sheets.py +3 -3
- airflow/providers/google/suite/sensors/drive.py +6 -1
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +1 -1
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +1 -1
- airflow/providers/google/version_compat.py +28 -0
- {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/METADATA +35 -35
- {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/RECORD +171 -170
- airflow/providers/google/cloud/links/automl.py +0 -193
- {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/entry_points.txt +0 -0
@@ -144,7 +144,6 @@ class CloudDLPCancelDLPJobOperator(GoogleCloudBaseOperator):
|
|
144
144
|
if project_id:
|
145
145
|
CloudDLPJobDetailsLink.persist(
|
146
146
|
context=context,
|
147
|
-
task_instance=self,
|
148
147
|
project_id=project_id,
|
149
148
|
job_name=self.dlp_job_id,
|
150
149
|
)
|
@@ -251,7 +250,6 @@ class CloudDLPCreateDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
251
250
|
if project_id and template_id:
|
252
251
|
CloudDLPDeidentifyTemplateDetailsLink.persist(
|
253
252
|
context=context,
|
254
|
-
task_instance=self,
|
255
253
|
project_id=project_id,
|
256
254
|
template_name=template_id,
|
257
255
|
)
|
@@ -363,7 +361,6 @@ class CloudDLPCreateDLPJobOperator(GoogleCloudBaseOperator):
|
|
363
361
|
if project_id:
|
364
362
|
CloudDLPJobDetailsLink.persist(
|
365
363
|
context=context,
|
366
|
-
task_instance=self,
|
367
364
|
project_id=project_id,
|
368
365
|
job_name=result["name"].split("/")[-1] if result["name"] else None,
|
369
366
|
)
|
@@ -473,7 +470,6 @@ class CloudDLPCreateInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
473
470
|
if project_id and template_id:
|
474
471
|
CloudDLPInspectTemplateDetailsLink.persist(
|
475
472
|
context=context,
|
476
|
-
task_instance=self,
|
477
473
|
project_id=project_id,
|
478
474
|
template_name=template_id,
|
479
475
|
)
|
@@ -578,7 +574,6 @@ class CloudDLPCreateJobTriggerOperator(GoogleCloudBaseOperator):
|
|
578
574
|
if project_id:
|
579
575
|
CloudDLPJobTriggerDetailsLink.persist(
|
580
576
|
context=context,
|
581
|
-
task_instance=self,
|
582
577
|
project_id=project_id,
|
583
578
|
trigger_name=trigger_name,
|
584
579
|
)
|
@@ -692,7 +687,6 @@ class CloudDLPCreateStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
692
687
|
if project_id and stored_info_type_id:
|
693
688
|
CloudDLPInfoTypeDetailsLink.persist(
|
694
689
|
context=context,
|
695
|
-
task_instance=self,
|
696
690
|
project_id=project_id,
|
697
691
|
info_type_name=stored_info_type_id,
|
698
692
|
)
|
@@ -880,7 +874,6 @@ class CloudDLPDeleteDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
880
874
|
if project_id:
|
881
875
|
CloudDLPDeidentifyTemplatesListLink.persist(
|
882
876
|
context=context,
|
883
|
-
task_instance=self,
|
884
877
|
project_id=project_id,
|
885
878
|
)
|
886
879
|
except NotFound:
|
@@ -966,7 +959,6 @@ class CloudDLPDeleteDLPJobOperator(GoogleCloudBaseOperator):
|
|
966
959
|
if project_id:
|
967
960
|
CloudDLPJobsListLink.persist(
|
968
961
|
context=context,
|
969
|
-
task_instance=self,
|
970
962
|
project_id=project_id,
|
971
963
|
)
|
972
964
|
|
@@ -1056,7 +1048,6 @@ class CloudDLPDeleteInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
1056
1048
|
if project_id:
|
1057
1049
|
CloudDLPInspectTemplatesListLink.persist(
|
1058
1050
|
context=context,
|
1059
|
-
task_instance=self,
|
1060
1051
|
project_id=project_id,
|
1061
1052
|
)
|
1062
1053
|
|
@@ -1140,7 +1131,6 @@ class CloudDLPDeleteJobTriggerOperator(GoogleCloudBaseOperator):
|
|
1140
1131
|
if project_id:
|
1141
1132
|
CloudDLPJobTriggersListLink.persist(
|
1142
1133
|
context=context,
|
1143
|
-
task_instance=self,
|
1144
1134
|
project_id=project_id,
|
1145
1135
|
)
|
1146
1136
|
|
@@ -1232,7 +1222,6 @@ class CloudDLPDeleteStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
1232
1222
|
if project_id:
|
1233
1223
|
CloudDLPInfoTypesListLink.persist(
|
1234
1224
|
context=context,
|
1235
|
-
task_instance=self,
|
1236
1225
|
project_id=project_id,
|
1237
1226
|
)
|
1238
1227
|
|
@@ -1318,7 +1307,9 @@ class CloudDLPGetDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
1318
1307
|
project_id = self.project_id or hook.project_id
|
1319
1308
|
if project_id:
|
1320
1309
|
CloudDLPDeidentifyTemplateDetailsLink.persist(
|
1321
|
-
context=context,
|
1310
|
+
context=context,
|
1311
|
+
project_id=project_id,
|
1312
|
+
template_name=self.template_id,
|
1322
1313
|
)
|
1323
1314
|
|
1324
1315
|
return DeidentifyTemplate.to_dict(template)
|
@@ -1400,7 +1391,6 @@ class CloudDLPGetDLPJobOperator(GoogleCloudBaseOperator):
|
|
1400
1391
|
if project_id:
|
1401
1392
|
CloudDLPJobDetailsLink.persist(
|
1402
1393
|
context=context,
|
1403
|
-
task_instance=self,
|
1404
1394
|
project_id=project_id,
|
1405
1395
|
job_name=self.dlp_job_id,
|
1406
1396
|
)
|
@@ -1490,7 +1480,6 @@ class CloudDLPGetInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
1490
1480
|
if project_id:
|
1491
1481
|
CloudDLPInspectTemplateDetailsLink.persist(
|
1492
1482
|
context=context,
|
1493
|
-
task_instance=self,
|
1494
1483
|
project_id=project_id,
|
1495
1484
|
template_name=self.template_id,
|
1496
1485
|
)
|
@@ -1574,7 +1563,6 @@ class CloudDLPGetDLPJobTriggerOperator(GoogleCloudBaseOperator):
|
|
1574
1563
|
if project_id:
|
1575
1564
|
CloudDLPJobTriggerDetailsLink.persist(
|
1576
1565
|
context=context,
|
1577
|
-
task_instance=self,
|
1578
1566
|
project_id=project_id,
|
1579
1567
|
trigger_name=self.job_trigger_id,
|
1580
1568
|
)
|
@@ -1664,7 +1652,6 @@ class CloudDLPGetStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
1664
1652
|
if project_id:
|
1665
1653
|
CloudDLPInfoTypeDetailsLink.persist(
|
1666
1654
|
context=context,
|
1667
|
-
task_instance=self,
|
1668
1655
|
project_id=project_id,
|
1669
1656
|
info_type_name=self.stored_info_type_id,
|
1670
1657
|
)
|
@@ -1844,7 +1831,6 @@ class CloudDLPListDeidentifyTemplatesOperator(GoogleCloudBaseOperator):
|
|
1844
1831
|
if project_id:
|
1845
1832
|
CloudDLPDeidentifyTemplatesListLink.persist(
|
1846
1833
|
context=context,
|
1847
|
-
task_instance=self,
|
1848
1834
|
project_id=project_id,
|
1849
1835
|
)
|
1850
1836
|
|
@@ -1940,7 +1926,6 @@ class CloudDLPListDLPJobsOperator(GoogleCloudBaseOperator):
|
|
1940
1926
|
if project_id:
|
1941
1927
|
CloudDLPJobsListLink.persist(
|
1942
1928
|
context=context,
|
1943
|
-
task_instance=self,
|
1944
1929
|
project_id=project_id,
|
1945
1930
|
)
|
1946
1931
|
|
@@ -2025,7 +2010,6 @@ class CloudDLPListInfoTypesOperator(GoogleCloudBaseOperator):
|
|
2025
2010
|
if project_id:
|
2026
2011
|
CloudDLPPossibleInfoTypesListLink.persist(
|
2027
2012
|
context=context,
|
2028
|
-
task_instance=self,
|
2029
2013
|
project_id=project_id,
|
2030
2014
|
)
|
2031
2015
|
|
@@ -2119,7 +2103,6 @@ class CloudDLPListInspectTemplatesOperator(GoogleCloudBaseOperator):
|
|
2119
2103
|
if project_id:
|
2120
2104
|
CloudDLPInspectTemplatesListLink.persist(
|
2121
2105
|
context=context,
|
2122
|
-
task_instance=self,
|
2123
2106
|
project_id=project_id,
|
2124
2107
|
)
|
2125
2108
|
|
@@ -2211,7 +2194,6 @@ class CloudDLPListJobTriggersOperator(GoogleCloudBaseOperator):
|
|
2211
2194
|
if project_id:
|
2212
2195
|
CloudDLPJobTriggersListLink.persist(
|
2213
2196
|
context=context,
|
2214
|
-
task_instance=self,
|
2215
2197
|
project_id=project_id,
|
2216
2198
|
)
|
2217
2199
|
|
@@ -2305,7 +2287,6 @@ class CloudDLPListStoredInfoTypesOperator(GoogleCloudBaseOperator):
|
|
2305
2287
|
if project_id:
|
2306
2288
|
CloudDLPInfoTypesListLink.persist(
|
2307
2289
|
context=context,
|
2308
|
-
task_instance=self,
|
2309
2290
|
project_id=project_id,
|
2310
2291
|
)
|
2311
2292
|
|
@@ -2592,7 +2573,6 @@ class CloudDLPUpdateDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
2592
2573
|
if project_id:
|
2593
2574
|
CloudDLPDeidentifyTemplateDetailsLink.persist(
|
2594
2575
|
context=context,
|
2595
|
-
task_instance=self,
|
2596
2576
|
project_id=project_id,
|
2597
2577
|
template_name=self.template_id,
|
2598
2578
|
)
|
@@ -2692,7 +2672,6 @@ class CloudDLPUpdateInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
2692
2672
|
if project_id:
|
2693
2673
|
CloudDLPInspectTemplateDetailsLink.persist(
|
2694
2674
|
context=context,
|
2695
|
-
task_instance=self,
|
2696
2675
|
project_id=project_id,
|
2697
2676
|
template_name=self.template_id,
|
2698
2677
|
)
|
@@ -2786,7 +2765,6 @@ class CloudDLPUpdateJobTriggerOperator(GoogleCloudBaseOperator):
|
|
2786
2765
|
if project_id:
|
2787
2766
|
CloudDLPJobTriggerDetailsLink.persist(
|
2788
2767
|
context=context,
|
2789
|
-
task_instance=self,
|
2790
2768
|
project_id=project_id,
|
2791
2769
|
trigger_name=self.job_trigger_id,
|
2792
2770
|
)
|
@@ -2887,7 +2865,6 @@ class CloudDLPUpdateStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
2887
2865
|
if project_id:
|
2888
2866
|
CloudDLPInfoTypeDetailsLink.persist(
|
2889
2867
|
context=context,
|
2890
|
-
task_instance=self,
|
2891
2868
|
project_id=project_id,
|
2892
2869
|
info_type_name=self.stored_info_type_id,
|
2893
2870
|
)
|
@@ -219,6 +219,13 @@ class CloudFunctionDeployFunctionOperator(GoogleCloudBaseOperator):
|
|
219
219
|
self.body["labels"] = {}
|
220
220
|
self.body["labels"].update({"airflow-version": "v" + version.replace(".", "-").replace("+", "-")})
|
221
221
|
|
222
|
+
@property
|
223
|
+
def extra_links_params(self) -> dict[str, Any]:
|
224
|
+
return {
|
225
|
+
"location": self.location,
|
226
|
+
"function_name": self.body["name"].split("/")[-1],
|
227
|
+
}
|
228
|
+
|
222
229
|
def execute(self, context: Context):
|
223
230
|
hook = CloudFunctionsHook(
|
224
231
|
gcp_conn_id=self.gcp_conn_id,
|
@@ -237,7 +244,6 @@ class CloudFunctionDeployFunctionOperator(GoogleCloudBaseOperator):
|
|
237
244
|
if project_id:
|
238
245
|
CloudFunctionsDetailsLink.persist(
|
239
246
|
context=context,
|
240
|
-
task_instance=self,
|
241
247
|
location=self.location,
|
242
248
|
project_id=project_id,
|
243
249
|
function_name=self.body["name"].split("/")[-1],
|
@@ -394,7 +400,6 @@ class CloudFunctionDeleteFunctionOperator(GoogleCloudBaseOperator):
|
|
394
400
|
if project_id:
|
395
401
|
CloudFunctionsListLink.persist(
|
396
402
|
context=context,
|
397
|
-
task_instance=self,
|
398
403
|
project_id=project_id,
|
399
404
|
)
|
400
405
|
return hook.delete_function(self.name)
|
@@ -462,6 +467,13 @@ class CloudFunctionInvokeFunctionOperator(GoogleCloudBaseOperator):
|
|
462
467
|
self.api_version = api_version
|
463
468
|
self.impersonation_chain = impersonation_chain
|
464
469
|
|
470
|
+
@property
|
471
|
+
def extra_links_params(self) -> dict[str, Any]:
|
472
|
+
return {
|
473
|
+
"location": self.location,
|
474
|
+
"function_name": self.function_id,
|
475
|
+
}
|
476
|
+
|
465
477
|
def execute(self, context: Context):
|
466
478
|
hook = CloudFunctionsHook(
|
467
479
|
api_version=self.api_version,
|
@@ -476,16 +488,13 @@ class CloudFunctionInvokeFunctionOperator(GoogleCloudBaseOperator):
|
|
476
488
|
project_id=self.project_id,
|
477
489
|
)
|
478
490
|
self.log.info("Function called successfully. Execution id %s", result.get("executionId"))
|
479
|
-
|
491
|
+
context["ti"].xcom_push(key="execution_id", value=result.get("executionId"))
|
480
492
|
|
481
493
|
project_id = self.project_id or hook.project_id
|
482
494
|
if project_id:
|
483
495
|
CloudFunctionsDetailsLink.persist(
|
484
496
|
context=context,
|
485
|
-
task_instance=self,
|
486
|
-
location=self.location,
|
487
497
|
project_id=project_id,
|
488
|
-
function_name=self.function_id,
|
489
498
|
)
|
490
499
|
|
491
500
|
return result
|
@@ -145,7 +145,6 @@ class GCSCreateBucketOperator(GoogleCloudBaseOperator):
|
|
145
145
|
)
|
146
146
|
StorageLink.persist(
|
147
147
|
context=context,
|
148
|
-
task_instance=self,
|
149
148
|
uri=self.bucket_name,
|
150
149
|
project_id=self.project_id or hook.project_id,
|
151
150
|
)
|
@@ -204,6 +203,7 @@ class GCSListObjectsOperator(GoogleCloudBaseOperator):
|
|
204
203
|
"bucket",
|
205
204
|
"prefix",
|
206
205
|
"delimiter",
|
206
|
+
"match_glob",
|
207
207
|
"impersonation_chain",
|
208
208
|
)
|
209
209
|
|
@@ -259,7 +259,6 @@ class GCSListObjectsOperator(GoogleCloudBaseOperator):
|
|
259
259
|
|
260
260
|
StorageLink.persist(
|
261
261
|
context=context,
|
262
|
-
task_instance=self,
|
263
262
|
uri=self.bucket,
|
264
263
|
project_id=hook.project_id,
|
265
264
|
)
|
@@ -438,7 +437,6 @@ class GCSBucketCreateAclEntryOperator(GoogleCloudBaseOperator):
|
|
438
437
|
)
|
439
438
|
StorageLink.persist(
|
440
439
|
context=context,
|
441
|
-
task_instance=self,
|
442
440
|
uri=self.bucket,
|
443
441
|
project_id=hook.project_id,
|
444
442
|
)
|
@@ -521,7 +519,6 @@ class GCSObjectCreateAclEntryOperator(GoogleCloudBaseOperator):
|
|
521
519
|
)
|
522
520
|
FileDetailsLink.persist(
|
523
521
|
context=context,
|
524
|
-
task_instance=self,
|
525
522
|
uri=f"{self.bucket}/{self.object_name}",
|
526
523
|
project_id=hook.project_id,
|
527
524
|
)
|
@@ -630,7 +627,6 @@ class GCSFileTransformOperator(GoogleCloudBaseOperator):
|
|
630
627
|
self.log.info("Uploading file to %s as %s", self.destination_bucket, self.destination_object)
|
631
628
|
FileDetailsLink.persist(
|
632
629
|
context=context,
|
633
|
-
task_instance=self,
|
634
630
|
uri=f"{self.destination_bucket}/{self.destination_object}",
|
635
631
|
project_id=hook.project_id,
|
636
632
|
)
|
@@ -828,7 +824,6 @@ class GCSTimeSpanFileTransformOperator(GoogleCloudBaseOperator):
|
|
828
824
|
)
|
829
825
|
StorageLink.persist(
|
830
826
|
context=context,
|
831
|
-
task_instance=self,
|
832
827
|
uri=self.destination_bucket,
|
833
828
|
project_id=destination_hook.project_id,
|
834
829
|
)
|
@@ -1079,7 +1074,6 @@ class GCSSynchronizeBucketsOperator(GoogleCloudBaseOperator):
|
|
1079
1074
|
)
|
1080
1075
|
StorageLink.persist(
|
1081
1076
|
context=context,
|
1082
|
-
task_instance=self,
|
1083
1077
|
uri=self._get_uri(self.destination_bucket, self.destination_object),
|
1084
1078
|
project_id=hook.project_id,
|
1085
1079
|
)
|
@@ -57,7 +57,6 @@ from airflow.providers.google.cloud.triggers.kubernetes_engine import (
|
|
57
57
|
GKEOperationTrigger,
|
58
58
|
GKEStartPodTrigger,
|
59
59
|
)
|
60
|
-
from airflow.providers.google.common.deprecated import deprecated
|
61
60
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
62
61
|
from airflow.providers_manager import ProvidersManager
|
63
62
|
from airflow.utils.timezone import utcnow
|
@@ -222,7 +221,6 @@ class GKEDeleteClusterOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
222
221
|
If set as a sequence, the identities from the list must grant
|
223
222
|
Service Account Token Creator IAM role to the directly preceding identity, with first
|
224
223
|
account from the list granting this role to the originating account (templated).
|
225
|
-
:param name: (Deprecated) The name of the resource to delete, in this case cluster name
|
226
224
|
:param api_version: The api version to use
|
227
225
|
:param deferrable: Run operator in the deferrable mode.
|
228
226
|
:param poll_interval: Interval size which defines how often operation status is checked.
|
@@ -241,7 +239,6 @@ class GKEDeleteClusterOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
241
239
|
gcp_conn_id: str = "google_cloud_default",
|
242
240
|
impersonation_chain: str | Sequence[str] | None = None,
|
243
241
|
cluster_name: str | None = None,
|
244
|
-
name: str | None = None,
|
245
242
|
api_version: str = "v2",
|
246
243
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
247
244
|
poll_interval: int = 10,
|
@@ -251,36 +248,17 @@ class GKEDeleteClusterOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
251
248
|
super().__init__(*args, **kwargs)
|
252
249
|
|
253
250
|
self.location = location
|
254
|
-
self.cluster_name = cluster_name
|
251
|
+
self.cluster_name = cluster_name
|
255
252
|
self.use_internal_ip = use_internal_ip
|
256
253
|
self.use_dns_endpoint = use_dns_endpoint
|
257
254
|
self.project_id = project_id
|
258
255
|
self.gcp_conn_id = gcp_conn_id
|
259
256
|
self.impersonation_chain = impersonation_chain
|
260
|
-
self._name = name
|
261
257
|
self.api_version = api_version
|
262
258
|
self.deferrable = deferrable
|
263
259
|
self.poll_interval = poll_interval
|
264
260
|
self._check_input()
|
265
261
|
|
266
|
-
@property
|
267
|
-
@deprecated(
|
268
|
-
planned_removal_date="May 01, 2025",
|
269
|
-
use_instead="cluster_name",
|
270
|
-
category=AirflowProviderDeprecationWarning,
|
271
|
-
)
|
272
|
-
def name(self) -> str | None:
|
273
|
-
return self._name
|
274
|
-
|
275
|
-
@name.setter
|
276
|
-
@deprecated(
|
277
|
-
planned_removal_date="May 01, 2025",
|
278
|
-
use_instead="cluster_name",
|
279
|
-
category=AirflowProviderDeprecationWarning,
|
280
|
-
)
|
281
|
-
def name(self, name: str) -> None:
|
282
|
-
self._name = name
|
283
|
-
|
284
262
|
def _check_input(self) -> None:
|
285
263
|
if not all([self.project_id, self.cluster_name, self.location]):
|
286
264
|
self.log.error("One of (project_id, cluster_name, location) is missing or incorrect")
|
@@ -475,8 +453,15 @@ class GKECreateClusterOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
475
453
|
stacklevel=2,
|
476
454
|
)
|
477
455
|
|
456
|
+
@property
|
457
|
+
def extra_links_params(self) -> dict[str, Any]:
|
458
|
+
return {
|
459
|
+
"project_id": self.project_id,
|
460
|
+
"location": self.location,
|
461
|
+
}
|
462
|
+
|
478
463
|
def execute(self, context: Context) -> str:
|
479
|
-
KubernetesEngineClusterLink.persist(context=context,
|
464
|
+
KubernetesEngineClusterLink.persist(context=context, cluster=self.body)
|
480
465
|
|
481
466
|
try:
|
482
467
|
operation = self.cluster_hook.create_cluster(
|
@@ -575,9 +560,16 @@ class GKEStartKueueInsideClusterOperator(GKEOperatorMixin, KubernetesInstallKueu
|
|
575
560
|
self.use_dns_endpoint = use_dns_endpoint
|
576
561
|
self.impersonation_chain = impersonation_chain
|
577
562
|
|
563
|
+
@property
|
564
|
+
def extra_links_params(self) -> dict[str, Any]:
|
565
|
+
return {
|
566
|
+
"project_id": self.project_id,
|
567
|
+
"location": self.location,
|
568
|
+
}
|
569
|
+
|
578
570
|
def execute(self, context: Context):
|
579
571
|
cluster = self.cluster_hook.get_cluster(name=self.cluster_name, project_id=self.project_id)
|
580
|
-
KubernetesEngineClusterLink.persist(context=context,
|
572
|
+
KubernetesEngineClusterLink.persist(context=context, cluster=cluster)
|
581
573
|
|
582
574
|
if self.cluster_hook.check_cluster_autoscaling_ability(cluster=cluster):
|
583
575
|
super().execute(context)
|
@@ -622,16 +614,10 @@ class GKEStartPodOperator(GKEOperatorMixin, KubernetesPodOperator):
|
|
622
614
|
If set as a sequence, the identities from the list must grant
|
623
615
|
Service Account Token Creator IAM role to the directly preceding identity, with first
|
624
616
|
account from the list granting this role to the originating account (templated).
|
625
|
-
:param regional: (Deprecated) The location param is region name.
|
626
617
|
:param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted.
|
627
618
|
If "delete_pod", the pod will be deleted regardless its state; if "delete_succeeded_pod",
|
628
619
|
only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod.
|
629
620
|
Current default is `keep_pod`, but this will be changed in the next major release of this provider.
|
630
|
-
:param is_delete_operator_pod: (Deprecated) What to do when the pod reaches its final
|
631
|
-
state, or the execution is interrupted. If True, delete the
|
632
|
-
pod; if False, leave the pod. Current default is False, but this will be
|
633
|
-
changed in the next major release of this provider.
|
634
|
-
Deprecated - use `on_finish_action` instead.
|
635
621
|
:param deferrable: Run operator in the deferrable mode.
|
636
622
|
"""
|
637
623
|
|
@@ -651,30 +637,15 @@ class GKEStartPodOperator(GKEOperatorMixin, KubernetesPodOperator):
|
|
651
637
|
project_id: str = PROVIDE_PROJECT_ID,
|
652
638
|
gcp_conn_id: str = "google_cloud_default",
|
653
639
|
impersonation_chain: str | Sequence[str] | None = None,
|
654
|
-
regional: bool | None = None,
|
655
640
|
on_finish_action: str | None = None,
|
656
|
-
is_delete_operator_pod: bool | None = None,
|
657
641
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
658
642
|
*args,
|
659
643
|
**kwargs,
|
660
644
|
) -> None:
|
661
|
-
if
|
662
|
-
kwargs["on_finish_action"] = (
|
663
|
-
OnFinishAction.DELETE_POD if is_delete_operator_pod else OnFinishAction.KEEP_POD
|
664
|
-
)
|
665
|
-
elif on_finish_action is not None:
|
645
|
+
if on_finish_action is not None:
|
666
646
|
kwargs["on_finish_action"] = OnFinishAction(on_finish_action)
|
667
647
|
else:
|
668
|
-
|
669
|
-
f"You have not set parameter `on_finish_action` in class {self.__class__.__name__}. "
|
670
|
-
"Currently the default for this parameter is `keep_pod` but in a future release"
|
671
|
-
" the default will be changed to `delete_pod`. To ensure pods are not deleted in"
|
672
|
-
" the future you will need to set `on_finish_action=keep_pod` explicitly.",
|
673
|
-
AirflowProviderDeprecationWarning,
|
674
|
-
stacklevel=2,
|
675
|
-
)
|
676
|
-
kwargs["on_finish_action"] = OnFinishAction.KEEP_POD
|
677
|
-
|
648
|
+
kwargs["on_finish_action"] = OnFinishAction.DELETE_POD
|
678
649
|
super().__init__(*args, **kwargs)
|
679
650
|
self.project_id = project_id
|
680
651
|
self.location = location
|
@@ -683,9 +654,6 @@ class GKEStartPodOperator(GKEOperatorMixin, KubernetesPodOperator):
|
|
683
654
|
self.use_internal_ip = use_internal_ip
|
684
655
|
self.use_dns_endpoint = use_dns_endpoint
|
685
656
|
self.impersonation_chain = impersonation_chain
|
686
|
-
self._regional = regional
|
687
|
-
if is_delete_operator_pod is not None:
|
688
|
-
self.is_delete_operator_pod = is_delete_operator_pod
|
689
657
|
self.deferrable = deferrable
|
690
658
|
|
691
659
|
# There is no need to manage the kube_config file, as it will be generated automatically.
|
@@ -693,42 +661,6 @@ class GKEStartPodOperator(GKEOperatorMixin, KubernetesPodOperator):
|
|
693
661
|
if self.config_file:
|
694
662
|
raise AirflowException("config_file is not an allowed parameter for the GKEStartPodOperator.")
|
695
663
|
|
696
|
-
@property
|
697
|
-
@deprecated(
|
698
|
-
planned_removal_date="May 01, 2025",
|
699
|
-
use_instead="on_finish_action",
|
700
|
-
category=AirflowProviderDeprecationWarning,
|
701
|
-
)
|
702
|
-
def is_delete_operator_pod(self) -> bool | None:
|
703
|
-
return self._is_delete_operator_pod
|
704
|
-
|
705
|
-
@is_delete_operator_pod.setter
|
706
|
-
@deprecated(
|
707
|
-
planned_removal_date="May 01, 2025",
|
708
|
-
use_instead="on_finish_action",
|
709
|
-
category=AirflowProviderDeprecationWarning,
|
710
|
-
)
|
711
|
-
def is_delete_operator_pod(self, is_delete_operator_pod) -> None:
|
712
|
-
self._is_delete_operator_pod = is_delete_operator_pod
|
713
|
-
|
714
|
-
@property
|
715
|
-
@deprecated(
|
716
|
-
planned_removal_date="May 01, 2025",
|
717
|
-
reason="The parameter is not in actual use.",
|
718
|
-
category=AirflowProviderDeprecationWarning,
|
719
|
-
)
|
720
|
-
def regional(self) -> bool | None:
|
721
|
-
return self._regional
|
722
|
-
|
723
|
-
@regional.setter
|
724
|
-
@deprecated(
|
725
|
-
planned_removal_date="May 01, 2025",
|
726
|
-
reason="The parameter is not in actual use.",
|
727
|
-
category=AirflowProviderDeprecationWarning,
|
728
|
-
)
|
729
|
-
def regional(self, regional) -> None:
|
730
|
-
self._regional = regional
|
731
|
-
|
732
664
|
def invoke_defer_method(self, last_log_time: DateTime | None = None):
|
733
665
|
"""Redefine triggers which are being used in child classes."""
|
734
666
|
trigger_start_time = utcnow()
|
@@ -936,7 +868,14 @@ class GKEDescribeJobOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
936
868
|
self.cluster_name,
|
937
869
|
self.job,
|
938
870
|
)
|
939
|
-
KubernetesEngineJobLink.persist(
|
871
|
+
KubernetesEngineJobLink.persist(
|
872
|
+
context=context,
|
873
|
+
location=self.location,
|
874
|
+
cluster_name=self.cluster_name,
|
875
|
+
namespace=self.job.metadata.namespace,
|
876
|
+
job_name=self.job.metadata.name,
|
877
|
+
project_id=self.project_id,
|
878
|
+
)
|
940
879
|
return None
|
941
880
|
|
942
881
|
|
@@ -1000,6 +939,15 @@ class GKEListJobsOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
1000
939
|
self.namespace = namespace
|
1001
940
|
self.do_xcom_push = do_xcom_push
|
1002
941
|
|
942
|
+
@property
|
943
|
+
def extra_links_params(self) -> dict[str, Any]:
|
944
|
+
return {
|
945
|
+
"location": self.location,
|
946
|
+
"cluster_name": self.cluster_name,
|
947
|
+
"namespace": self.namespace,
|
948
|
+
"project_id": self.project_id,
|
949
|
+
}
|
950
|
+
|
1003
951
|
def execute(self, context: Context) -> dict:
|
1004
952
|
if self.namespace:
|
1005
953
|
jobs = self.hook.list_jobs_from_namespace(namespace=self.namespace)
|
@@ -1010,7 +958,7 @@ class GKEListJobsOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
1010
958
|
if self.do_xcom_push:
|
1011
959
|
ti = context["ti"]
|
1012
960
|
ti.xcom_push(key="jobs_list", value=V1JobList.to_dict(jobs))
|
1013
|
-
KubernetesEngineWorkloadsLink.persist(context=context
|
961
|
+
KubernetesEngineWorkloadsLink.persist(context=context)
|
1014
962
|
return V1JobList.to_dict(jobs)
|
1015
963
|
|
1016
964
|
|
@@ -1352,8 +1300,14 @@ class GKESuspendJobOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
1352
1300
|
self.name,
|
1353
1301
|
self.cluster_name,
|
1354
1302
|
)
|
1355
|
-
KubernetesEngineJobLink.persist(
|
1356
|
-
|
1303
|
+
KubernetesEngineJobLink.persist(
|
1304
|
+
context=context,
|
1305
|
+
location=self.location,
|
1306
|
+
cluster_name=self.cluster_name,
|
1307
|
+
namespace=self.job.metadata.namespace,
|
1308
|
+
job_name=self.job.metadata.name,
|
1309
|
+
project_id=self.project_id,
|
1310
|
+
)
|
1357
1311
|
return k8s.V1Job.to_dict(self.job)
|
1358
1312
|
|
1359
1313
|
|
@@ -1426,6 +1380,13 @@ class GKEResumeJobOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
1426
1380
|
self.name,
|
1427
1381
|
self.cluster_name,
|
1428
1382
|
)
|
1429
|
-
KubernetesEngineJobLink.persist(
|
1383
|
+
KubernetesEngineJobLink.persist(
|
1384
|
+
context=context,
|
1385
|
+
location=self.location,
|
1386
|
+
cluster_name=self.cluster_name,
|
1387
|
+
namespace=self.job.metadata.namespace,
|
1388
|
+
job_name=self.job.metadata.name,
|
1389
|
+
project_id=self.project_id,
|
1390
|
+
)
|
1430
1391
|
|
1431
1392
|
return k8s.V1Job.to_dict(self.job)
|
@@ -113,7 +113,6 @@ class LifeSciencesRunPipelineOperator(GoogleCloudBaseOperator):
|
|
113
113
|
if project_id:
|
114
114
|
LifeSciencesLink.persist(
|
115
115
|
context=context,
|
116
|
-
task_instance=self,
|
117
116
|
project_id=project_id,
|
118
117
|
)
|
119
118
|
return hook.run_pipeline(body=self.body, location=self.location, project_id=self.project_id)
|