apache-airflow-providers-google 14.0.0__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
- airflow/providers/google/3rd-party-licenses/NOTICE +5 -0
- airflow/providers/google/__init__.py +3 -3
- airflow/providers/google/_vendor/__init__.py +0 -0
- airflow/providers/google/_vendor/json_merge_patch.py +91 -0
- airflow/providers/google/ads/hooks/ads.py +52 -43
- airflow/providers/google/ads/operators/ads.py +2 -2
- airflow/providers/google/ads/transfers/ads_to_gcs.py +3 -19
- airflow/providers/google/assets/gcs.py +1 -11
- airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
- airflow/providers/google/cloud/bundles/gcs.py +161 -0
- airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
- airflow/providers/google/cloud/hooks/bigquery.py +195 -318
- airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
- airflow/providers/google/cloud/hooks/bigtable.py +3 -2
- airflow/providers/google/cloud/hooks/cloud_batch.py +8 -9
- airflow/providers/google/cloud/hooks/cloud_build.py +6 -65
- airflow/providers/google/cloud/hooks/cloud_composer.py +292 -24
- airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
- airflow/providers/google/cloud/hooks/cloud_run.py +20 -11
- airflow/providers/google/cloud/hooks/cloud_sql.py +136 -64
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +35 -15
- airflow/providers/google/cloud/hooks/compute.py +7 -6
- airflow/providers/google/cloud/hooks/compute_ssh.py +7 -4
- airflow/providers/google/cloud/hooks/datacatalog.py +12 -3
- airflow/providers/google/cloud/hooks/dataflow.py +87 -242
- airflow/providers/google/cloud/hooks/dataform.py +9 -14
- airflow/providers/google/cloud/hooks/datafusion.py +7 -9
- airflow/providers/google/cloud/hooks/dataplex.py +13 -12
- airflow/providers/google/cloud/hooks/dataprep.py +2 -2
- airflow/providers/google/cloud/hooks/dataproc.py +76 -74
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/hooks/dlp.py +5 -4
- airflow/providers/google/cloud/hooks/gcs.py +144 -33
- airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
- airflow/providers/google/cloud/hooks/kms.py +3 -2
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +22 -17
- airflow/providers/google/cloud/hooks/looker.py +6 -1
- airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
- airflow/providers/google/cloud/hooks/mlengine.py +7 -8
- airflow/providers/google/cloud/hooks/natural_language.py +3 -2
- airflow/providers/google/cloud/hooks/os_login.py +3 -2
- airflow/providers/google/cloud/hooks/pubsub.py +6 -6
- airflow/providers/google/cloud/hooks/secret_manager.py +105 -12
- airflow/providers/google/cloud/hooks/spanner.py +75 -10
- airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
- airflow/providers/google/cloud/hooks/stackdriver.py +18 -18
- airflow/providers/google/cloud/hooks/tasks.py +4 -3
- airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
- airflow/providers/google/cloud/hooks/translate.py +8 -17
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +8 -222
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +9 -15
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +33 -283
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +311 -10
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +7 -13
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +8 -12
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
- airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
- airflow/providers/google/cloud/hooks/vision.py +7 -7
- airflow/providers/google/cloud/hooks/workflows.py +4 -3
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +77 -7
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -46
- airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -90
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +11 -89
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +11 -61
- airflow/providers/google/cloud/links/managed_kafka.py +11 -51
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +17 -187
- airflow/providers/google/cloud/links/vertex_ai.py +28 -195
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/log/gcs_task_handler.py +166 -118
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +14 -9
- airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
- airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
- airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
- airflow/providers/google/cloud/openlineage/facets.py +141 -40
- airflow/providers/google/cloud/openlineage/mixins.py +14 -13
- airflow/providers/google/cloud/openlineage/utils.py +19 -3
- airflow/providers/google/cloud/operators/alloy_db.py +76 -61
- airflow/providers/google/cloud/operators/bigquery.py +104 -667
- airflow/providers/google/cloud/operators/bigquery_dts.py +12 -12
- airflow/providers/google/cloud/operators/bigtable.py +38 -7
- airflow/providers/google/cloud/operators/cloud_base.py +22 -1
- airflow/providers/google/cloud/operators/cloud_batch.py +18 -18
- airflow/providers/google/cloud/operators/cloud_build.py +80 -36
- airflow/providers/google/cloud/operators/cloud_composer.py +157 -71
- airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
- airflow/providers/google/cloud/operators/cloud_memorystore.py +74 -46
- airflow/providers/google/cloud/operators/cloud_run.py +39 -20
- airflow/providers/google/cloud/operators/cloud_sql.py +46 -61
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +92 -14
- airflow/providers/google/cloud/operators/compute.py +18 -50
- airflow/providers/google/cloud/operators/datacatalog.py +167 -29
- airflow/providers/google/cloud/operators/dataflow.py +38 -15
- airflow/providers/google/cloud/operators/dataform.py +19 -7
- airflow/providers/google/cloud/operators/datafusion.py +43 -43
- airflow/providers/google/cloud/operators/dataplex.py +212 -126
- airflow/providers/google/cloud/operators/dataprep.py +1 -5
- airflow/providers/google/cloud/operators/dataproc.py +134 -207
- airflow/providers/google/cloud/operators/dataproc_metastore.py +102 -84
- airflow/providers/google/cloud/operators/datastore.py +22 -6
- airflow/providers/google/cloud/operators/dlp.py +24 -45
- airflow/providers/google/cloud/operators/functions.py +21 -14
- airflow/providers/google/cloud/operators/gcs.py +15 -12
- airflow/providers/google/cloud/operators/gen_ai.py +389 -0
- airflow/providers/google/cloud/operators/kubernetes_engine.py +115 -106
- airflow/providers/google/cloud/operators/looker.py +1 -1
- airflow/providers/google/cloud/operators/managed_kafka.py +362 -40
- airflow/providers/google/cloud/operators/natural_language.py +5 -3
- airflow/providers/google/cloud/operators/pubsub.py +69 -21
- airflow/providers/google/cloud/operators/spanner.py +53 -45
- airflow/providers/google/cloud/operators/speech_to_text.py +5 -4
- airflow/providers/google/cloud/operators/stackdriver.py +5 -11
- airflow/providers/google/cloud/operators/tasks.py +6 -15
- airflow/providers/google/cloud/operators/text_to_speech.py +4 -3
- airflow/providers/google/cloud/operators/translate.py +46 -20
- airflow/providers/google/cloud/operators/translate_speech.py +4 -3
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +44 -34
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +34 -12
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +62 -53
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +75 -11
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +48 -12
- airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -116
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +16 -12
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +62 -14
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +35 -10
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
- airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
- airflow/providers/google/cloud/operators/vision.py +7 -5
- airflow/providers/google/cloud/operators/workflows.py +24 -19
- airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
- airflow/providers/google/cloud/sensors/bigquery.py +2 -2
- airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -4
- airflow/providers/google/cloud/sensors/bigtable.py +14 -6
- airflow/providers/google/cloud/sensors/cloud_composer.py +535 -33
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -5
- airflow/providers/google/cloud/sensors/dataflow.py +27 -10
- airflow/providers/google/cloud/sensors/dataform.py +2 -2
- airflow/providers/google/cloud/sensors/datafusion.py +4 -4
- airflow/providers/google/cloud/sensors/dataplex.py +7 -5
- airflow/providers/google/cloud/sensors/dataprep.py +2 -2
- airflow/providers/google/cloud/sensors/dataproc.py +10 -9
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/sensors/gcs.py +22 -21
- airflow/providers/google/cloud/sensors/looker.py +5 -5
- airflow/providers/google/cloud/sensors/pubsub.py +20 -20
- airflow/providers/google/cloud/sensors/tasks.py +2 -2
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -2
- airflow/providers/google/cloud/sensors/workflows.py +6 -4
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +14 -13
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
- airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +18 -22
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -5
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +45 -38
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/gcs_to_local.py +5 -3
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +10 -4
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
- airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
- airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +44 -12
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +12 -6
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +36 -14
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
- airflow/providers/google/cloud/triggers/bigquery.py +75 -34
- airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +3 -2
- airflow/providers/google/cloud/triggers/cloud_composer.py +303 -47
- airflow/providers/google/cloud/triggers/cloud_run.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +96 -5
- airflow/providers/google/cloud/triggers/dataflow.py +125 -2
- airflow/providers/google/cloud/triggers/datafusion.py +1 -1
- airflow/providers/google/cloud/triggers/dataplex.py +16 -3
- airflow/providers/google/cloud/triggers/dataproc.py +124 -53
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +46 -28
- airflow/providers/google/cloud/triggers/mlengine.py +1 -1
- airflow/providers/google/cloud/triggers/pubsub.py +17 -20
- airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
- airflow/providers/google/cloud/utils/bigquery.py +5 -7
- airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
- airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
- airflow/providers/google/cloud/utils/field_validator.py +1 -2
- airflow/providers/google/cloud/utils/validators.py +43 -0
- airflow/providers/google/common/auth_backend/google_openid.py +26 -9
- airflow/providers/google/common/consts.py +2 -1
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +40 -43
- airflow/providers/google/common/hooks/operation_helpers.py +78 -0
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/common/utils/get_secret.py +31 -0
- airflow/providers/google/common/utils/id_token_credentials.py +4 -5
- airflow/providers/google/firebase/operators/firestore.py +2 -2
- airflow/providers/google/get_provider_info.py +61 -216
- airflow/providers/google/go_module_utils.py +35 -3
- airflow/providers/google/leveldb/hooks/leveldb.py +30 -6
- airflow/providers/google/leveldb/operators/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
- airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
- airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +4 -5
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -6
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +5 -5
- airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
- airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
- airflow/providers/google/marketing_platform/sensors/display_video.py +4 -64
- airflow/providers/google/suite/hooks/calendar.py +1 -1
- airflow/providers/google/suite/hooks/drive.py +2 -2
- airflow/providers/google/suite/hooks/sheets.py +15 -1
- airflow/providers/google/suite/operators/sheets.py +8 -3
- airflow/providers/google/suite/sensors/drive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
- airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
- airflow/providers/google/version_compat.py +15 -1
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/METADATA +117 -72
- apache_airflow_providers_google-19.1.0rc1.dist-info/RECORD +331 -0
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/WHEEL +1 -1
- apache_airflow_providers_google-19.1.0rc1.dist-info/licenses/NOTICE +5 -0
- airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
- airflow/providers/google/cloud/hooks/automl.py +0 -679
- airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
- airflow/providers/google/cloud/links/automl.py +0 -193
- airflow/providers/google/cloud/operators/automl.py +0 -1360
- airflow/providers/google/cloud/operators/life_sciences.py +0 -119
- airflow/providers/google/cloud/operators/mlengine.py +0 -1515
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +0 -273
- apache_airflow_providers_google-14.0.0.dist-info/RECORD +0 -318
- /airflow/providers/google/cloud/{example_dags → bundles}/__init__.py +0 -0
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/entry_points.txt +0 -0
- {airflow/providers/google → apache_airflow_providers_google-19.1.0rc1.dist-info/licenses}/LICENSE +0 -0
|
@@ -24,13 +24,6 @@ from collections.abc import Sequence
|
|
|
24
24
|
from functools import cached_property
|
|
25
25
|
from typing import TYPE_CHECKING
|
|
26
26
|
|
|
27
|
-
from airflow.configuration import conf
|
|
28
|
-
from airflow.exceptions import AirflowException
|
|
29
|
-
from airflow.providers.google.cloud.hooks.bigquery_dts import BiqQueryDataTransferServiceHook, get_object_id
|
|
30
|
-
from airflow.providers.google.cloud.links.bigquery_dts import BigQueryDataTransferConfigLink
|
|
31
|
-
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
32
|
-
from airflow.providers.google.cloud.triggers.bigquery_dts import BigQueryDataTransferRunTrigger
|
|
33
|
-
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
34
27
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
35
28
|
from google.cloud.bigquery_datatransfer_v1 import (
|
|
36
29
|
StartManualTransferRunsResponse,
|
|
@@ -39,10 +32,19 @@ from google.cloud.bigquery_datatransfer_v1 import (
|
|
|
39
32
|
TransferState,
|
|
40
33
|
)
|
|
41
34
|
|
|
35
|
+
from airflow.configuration import conf
|
|
36
|
+
from airflow.exceptions import AirflowException
|
|
37
|
+
from airflow.providers.google.cloud.hooks.bigquery_dts import BiqQueryDataTransferServiceHook, get_object_id
|
|
38
|
+
from airflow.providers.google.cloud.links.bigquery_dts import BigQueryDataTransferConfigLink
|
|
39
|
+
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
40
|
+
from airflow.providers.google.cloud.triggers.bigquery_dts import BigQueryDataTransferRunTrigger
|
|
41
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
42
|
+
|
|
42
43
|
if TYPE_CHECKING:
|
|
43
|
-
from airflow.utils.context import Context
|
|
44
44
|
from google.api_core.retry import Retry
|
|
45
45
|
|
|
46
|
+
from airflow.providers.common.compat.sdk import Context
|
|
47
|
+
|
|
46
48
|
|
|
47
49
|
def _get_transfer_config_details(config_transfer_name: str):
|
|
48
50
|
config_details = config_transfer_name.split("/")
|
|
@@ -132,7 +134,6 @@ class BigQueryCreateDataTransferOperator(GoogleCloudBaseOperator):
|
|
|
132
134
|
transfer_config = _get_transfer_config_details(response.name)
|
|
133
135
|
BigQueryDataTransferConfigLink.persist(
|
|
134
136
|
context=context,
|
|
135
|
-
task_instance=self,
|
|
136
137
|
region=transfer_config["region"],
|
|
137
138
|
config_id=transfer_config["config_id"],
|
|
138
139
|
project_id=transfer_config["project_id"],
|
|
@@ -140,7 +141,7 @@ class BigQueryCreateDataTransferOperator(GoogleCloudBaseOperator):
|
|
|
140
141
|
|
|
141
142
|
result = TransferConfig.to_dict(response)
|
|
142
143
|
self.log.info("Created DTS transfer config %s", get_object_id(result))
|
|
143
|
-
|
|
144
|
+
context["ti"].xcom_push(key="transfer_config_id", value=get_object_id(result))
|
|
144
145
|
# don't push AWS secret in XCOM
|
|
145
146
|
result.get("params", {}).pop("secret_access_key", None)
|
|
146
147
|
result.get("params", {}).pop("access_key_id", None)
|
|
@@ -327,7 +328,6 @@ class BigQueryDataTransferServiceStartTransferRunsOperator(GoogleCloudBaseOperat
|
|
|
327
328
|
transfer_config = _get_transfer_config_details(response.runs[0].name)
|
|
328
329
|
BigQueryDataTransferConfigLink.persist(
|
|
329
330
|
context=context,
|
|
330
|
-
task_instance=self,
|
|
331
331
|
region=transfer_config["region"],
|
|
332
332
|
config_id=transfer_config["config_id"],
|
|
333
333
|
project_id=transfer_config["project_id"],
|
|
@@ -335,7 +335,7 @@ class BigQueryDataTransferServiceStartTransferRunsOperator(GoogleCloudBaseOperat
|
|
|
335
335
|
|
|
336
336
|
result = StartManualTransferRunsResponse.to_dict(response)
|
|
337
337
|
run_id = get_object_id(result["runs"][0])
|
|
338
|
-
|
|
338
|
+
context["ti"].xcom_push(key="run_id", value=run_id)
|
|
339
339
|
|
|
340
340
|
if not self.deferrable:
|
|
341
341
|
# Save as attribute for further use by OpenLineage
|
|
@@ -20,9 +20,10 @@
|
|
|
20
20
|
from __future__ import annotations
|
|
21
21
|
|
|
22
22
|
from collections.abc import Iterable, Sequence
|
|
23
|
-
from typing import TYPE_CHECKING
|
|
23
|
+
from typing import TYPE_CHECKING, Any
|
|
24
24
|
|
|
25
25
|
import google.api_core.exceptions
|
|
26
|
+
|
|
26
27
|
from airflow.exceptions import AirflowException
|
|
27
28
|
from airflow.providers.google.cloud.hooks.bigtable import BigtableHook
|
|
28
29
|
from airflow.providers.google.cloud.links.bigtable import (
|
|
@@ -36,10 +37,11 @@ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
|
36
37
|
if TYPE_CHECKING:
|
|
37
38
|
import enum
|
|
38
39
|
|
|
39
|
-
from airflow.utils.context import Context
|
|
40
40
|
from google.cloud.bigtable import enums
|
|
41
41
|
from google.cloud.bigtable.column_family import GarbageCollectionRule
|
|
42
42
|
|
|
43
|
+
from airflow.providers.common.compat.sdk import Context
|
|
44
|
+
|
|
43
45
|
|
|
44
46
|
class BigtableValidationMixin:
|
|
45
47
|
"""Common class for Cloud Bigtable operators for validating required fields."""
|
|
@@ -140,6 +142,13 @@ class BigtableCreateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
|
|
|
140
142
|
self.impersonation_chain = impersonation_chain
|
|
141
143
|
super().__init__(**kwargs)
|
|
142
144
|
|
|
145
|
+
@property
|
|
146
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
147
|
+
return {
|
|
148
|
+
"instance_id": self.instance_id,
|
|
149
|
+
"project_id": self.project_id,
|
|
150
|
+
}
|
|
151
|
+
|
|
143
152
|
def execute(self, context: Context) -> None:
|
|
144
153
|
hook = BigtableHook(
|
|
145
154
|
gcp_conn_id=self.gcp_conn_id,
|
|
@@ -153,7 +162,7 @@ class BigtableCreateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
|
|
|
153
162
|
"The instance '%s' already exists in this project. Consider it as created",
|
|
154
163
|
self.instance_id,
|
|
155
164
|
)
|
|
156
|
-
BigtableInstanceLink.persist(context=context
|
|
165
|
+
BigtableInstanceLink.persist(context=context)
|
|
157
166
|
return
|
|
158
167
|
try:
|
|
159
168
|
hook.create_instance(
|
|
@@ -169,7 +178,7 @@ class BigtableCreateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
|
|
|
169
178
|
cluster_storage_type=self.cluster_storage_type,
|
|
170
179
|
timeout=self.timeout,
|
|
171
180
|
)
|
|
172
|
-
BigtableInstanceLink.persist(context=context
|
|
181
|
+
BigtableInstanceLink.persist(context=context)
|
|
173
182
|
except google.api_core.exceptions.GoogleAPICallError as e:
|
|
174
183
|
self.log.error("An error occurred. Exiting.")
|
|
175
184
|
raise e
|
|
@@ -238,6 +247,13 @@ class BigtableUpdateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
|
|
|
238
247
|
self.impersonation_chain = impersonation_chain
|
|
239
248
|
super().__init__(**kwargs)
|
|
240
249
|
|
|
250
|
+
@property
|
|
251
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
252
|
+
return {
|
|
253
|
+
"instance_id": self.instance_id,
|
|
254
|
+
"project_id": self.project_id,
|
|
255
|
+
}
|
|
256
|
+
|
|
241
257
|
def execute(self, context: Context) -> None:
|
|
242
258
|
hook = BigtableHook(
|
|
243
259
|
gcp_conn_id=self.gcp_conn_id,
|
|
@@ -256,7 +272,7 @@ class BigtableUpdateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
|
|
|
256
272
|
instance_labels=self.instance_labels,
|
|
257
273
|
timeout=self.timeout,
|
|
258
274
|
)
|
|
259
|
-
BigtableInstanceLink.persist(context=context
|
|
275
|
+
BigtableInstanceLink.persist(context=context)
|
|
260
276
|
except google.api_core.exceptions.GoogleAPICallError as e:
|
|
261
277
|
self.log.error("An error occurred. Exiting.")
|
|
262
278
|
raise e
|
|
@@ -412,6 +428,13 @@ class BigtableCreateTableOperator(GoogleCloudBaseOperator, BigtableValidationMix
|
|
|
412
428
|
return False
|
|
413
429
|
return True
|
|
414
430
|
|
|
431
|
+
@property
|
|
432
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
433
|
+
return {
|
|
434
|
+
"instance_id": self.instance_id,
|
|
435
|
+
"project_id": self.project_id,
|
|
436
|
+
}
|
|
437
|
+
|
|
415
438
|
def execute(self, context: Context) -> None:
|
|
416
439
|
hook = BigtableHook(
|
|
417
440
|
gcp_conn_id=self.gcp_conn_id,
|
|
@@ -429,7 +452,7 @@ class BigtableCreateTableOperator(GoogleCloudBaseOperator, BigtableValidationMix
|
|
|
429
452
|
initial_split_keys=self.initial_split_keys,
|
|
430
453
|
column_families=self.column_families,
|
|
431
454
|
)
|
|
432
|
-
BigtableTablesLink.persist(context=context
|
|
455
|
+
BigtableTablesLink.persist(context=context)
|
|
433
456
|
except google.api_core.exceptions.AlreadyExists:
|
|
434
457
|
if not self._compare_column_families(hook, instance):
|
|
435
458
|
raise AirflowException(
|
|
@@ -573,6 +596,14 @@ class BigtableUpdateClusterOperator(GoogleCloudBaseOperator, BigtableValidationM
|
|
|
573
596
|
self.impersonation_chain = impersonation_chain
|
|
574
597
|
super().__init__(**kwargs)
|
|
575
598
|
|
|
599
|
+
@property
|
|
600
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
601
|
+
return {
|
|
602
|
+
"instance_id": self.instance_id,
|
|
603
|
+
"cluster_id": self.cluster_id,
|
|
604
|
+
"project_id": self.project_id,
|
|
605
|
+
}
|
|
606
|
+
|
|
576
607
|
def execute(self, context: Context) -> None:
|
|
577
608
|
hook = BigtableHook(
|
|
578
609
|
gcp_conn_id=self.gcp_conn_id,
|
|
@@ -584,7 +615,7 @@ class BigtableUpdateClusterOperator(GoogleCloudBaseOperator, BigtableValidationM
|
|
|
584
615
|
|
|
585
616
|
try:
|
|
586
617
|
hook.update_cluster(instance=instance, cluster_id=self.cluster_id, nodes=self.nodes)
|
|
587
|
-
BigtableClusterLink.persist(context=context
|
|
618
|
+
BigtableClusterLink.persist(context=context)
|
|
588
619
|
except google.api_core.exceptions.NotFound:
|
|
589
620
|
raise AirflowException(
|
|
590
621
|
f"Dependency: cluster '{self.cluster_id}' does not exist for instance '{self.instance_id}'."
|
|
@@ -19,9 +19,12 @@
|
|
|
19
19
|
|
|
20
20
|
from __future__ import annotations
|
|
21
21
|
|
|
22
|
-
from
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
23
24
|
from google.api_core.gapic_v1.method import DEFAULT
|
|
24
25
|
|
|
26
|
+
from airflow.providers.google.version_compat import BaseOperator
|
|
27
|
+
|
|
25
28
|
|
|
26
29
|
class GoogleCloudBaseOperator(BaseOperator):
|
|
27
30
|
"""Abstract base class for operators using Google API client libraries."""
|
|
@@ -35,3 +38,21 @@ class GoogleCloudBaseOperator(BaseOperator):
|
|
|
35
38
|
"""
|
|
36
39
|
memo[id(DEFAULT)] = DEFAULT
|
|
37
40
|
return super().__deepcopy__(memo)
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
44
|
+
"""
|
|
45
|
+
Override this method to include parameters for link formatting in extra links.
|
|
46
|
+
|
|
47
|
+
For example; most of the links on the Google provider require `project_id` and `location` in the Link.
|
|
48
|
+
To be not repeat; you can override this function and return something like the following:
|
|
49
|
+
|
|
50
|
+
.. code-block:: python
|
|
51
|
+
|
|
52
|
+
{
|
|
53
|
+
"project_id": self.project_id,
|
|
54
|
+
"location": self.location,
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
"""
|
|
58
|
+
return {}
|
|
@@ -20,17 +20,19 @@ from __future__ import annotations
|
|
|
20
20
|
from collections.abc import Sequence
|
|
21
21
|
from typing import TYPE_CHECKING
|
|
22
22
|
|
|
23
|
+
from google.cloud.batch_v1 import Job, Task
|
|
24
|
+
|
|
23
25
|
from airflow.configuration import conf
|
|
24
26
|
from airflow.exceptions import AirflowException
|
|
25
27
|
from airflow.providers.google.cloud.hooks.cloud_batch import CloudBatchHook
|
|
26
28
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
27
29
|
from airflow.providers.google.cloud.triggers.cloud_batch import CloudBatchJobFinishedTrigger
|
|
28
|
-
from google.cloud.batch_v1 import Job, Task
|
|
29
30
|
|
|
30
31
|
if TYPE_CHECKING:
|
|
31
|
-
from airflow.utils.context import Context
|
|
32
32
|
from google.api_core import operation
|
|
33
33
|
|
|
34
|
+
from airflow.providers.common.compat.sdk import Context
|
|
35
|
+
|
|
34
36
|
|
|
35
37
|
class CloudBatchSubmitJobOperator(GoogleCloudBaseOperator):
|
|
36
38
|
"""
|
|
@@ -98,19 +100,18 @@ class CloudBatchSubmitJobOperator(GoogleCloudBaseOperator):
|
|
|
98
100
|
|
|
99
101
|
return Job.to_dict(completed_job)
|
|
100
102
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
)
|
|
103
|
+
self.defer(
|
|
104
|
+
trigger=CloudBatchJobFinishedTrigger(
|
|
105
|
+
job_name=job.name,
|
|
106
|
+
project_id=self.project_id,
|
|
107
|
+
gcp_conn_id=self.gcp_conn_id,
|
|
108
|
+
impersonation_chain=self.impersonation_chain,
|
|
109
|
+
location=self.region,
|
|
110
|
+
polling_period_seconds=self.polling_period_seconds,
|
|
111
|
+
timeout=self.timeout_seconds,
|
|
112
|
+
),
|
|
113
|
+
method_name="execute_complete",
|
|
114
|
+
)
|
|
114
115
|
|
|
115
116
|
def execute_complete(self, context: Context, event: dict):
|
|
116
117
|
job_status = event["status"]
|
|
@@ -118,8 +119,7 @@ class CloudBatchSubmitJobOperator(GoogleCloudBaseOperator):
|
|
|
118
119
|
hook: CloudBatchHook = CloudBatchHook(self.gcp_conn_id, self.impersonation_chain)
|
|
119
120
|
job = hook.get_job(job_name=event["job_name"])
|
|
120
121
|
return Job.to_dict(job)
|
|
121
|
-
|
|
122
|
-
raise AirflowException(f"Unexpected error in the operation: {event['message']}")
|
|
122
|
+
raise AirflowException(f"Unexpected error in the operation: {event['message']}")
|
|
123
123
|
|
|
124
124
|
|
|
125
125
|
class CloudBatchDeleteJobOperator(GoogleCloudBaseOperator):
|
|
@@ -244,7 +244,7 @@ class CloudBatchListTasksOperator(GoogleCloudBaseOperator):
|
|
|
244
244
|
:param job_name: Required. The name of the job for which to list tasks.
|
|
245
245
|
:param gcp_conn_id: The connection ID used to connect to Google Cloud.
|
|
246
246
|
:param filter: The filter based on which to list the jobs. If left empty, all the jobs are listed.
|
|
247
|
-
:param group_name: The name of the group that owns the task. By default it's `group0`.
|
|
247
|
+
:param group_name: The name of the group that owns the task. By default, it's `group0`.
|
|
248
248
|
:param limit: The number of tasks to list.
|
|
249
249
|
If left empty, all the tasks matching the filter will be returned.
|
|
250
250
|
:param impersonation_chain: Optional service account to impersonate using short-term
|
|
@@ -26,6 +26,9 @@ from copy import deepcopy
|
|
|
26
26
|
from typing import TYPE_CHECKING, Any
|
|
27
27
|
from urllib.parse import unquote, urlsplit
|
|
28
28
|
|
|
29
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
30
|
+
from google.cloud.devtools.cloudbuild_v1.types import Build, BuildTrigger, RepoSource
|
|
31
|
+
|
|
29
32
|
from airflow.configuration import conf
|
|
30
33
|
from airflow.exceptions import AirflowException
|
|
31
34
|
from airflow.providers.google.cloud.hooks.cloud_build import CloudBuildHook
|
|
@@ -41,13 +44,12 @@ from airflow.providers.google.common.consts import GOOGLE_DEFAULT_DEFERRABLE_MET
|
|
|
41
44
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
42
45
|
from airflow.utils import yaml
|
|
43
46
|
from airflow.utils.helpers import exactly_one
|
|
44
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
45
|
-
from google.cloud.devtools.cloudbuild_v1.types import Build, BuildTrigger, RepoSource
|
|
46
47
|
|
|
47
48
|
if TYPE_CHECKING:
|
|
48
|
-
from airflow.utils.context import Context
|
|
49
49
|
from google.api_core.retry import Retry
|
|
50
50
|
|
|
51
|
+
from airflow.providers.common.compat.sdk import Context
|
|
52
|
+
|
|
51
53
|
|
|
52
54
|
REGEX_REPO_PATH = re.compile(r"^/(?P<project_id>[^/]+)/(?P<repo_name>[^/]+)[\+/]*(?P<branch_name>[^:]+)?")
|
|
53
55
|
|
|
@@ -106,6 +108,12 @@ class CloudBuildCancelBuildOperator(GoogleCloudBaseOperator):
|
|
|
106
108
|
self.impersonation_chain = impersonation_chain
|
|
107
109
|
self.location = location
|
|
108
110
|
|
|
111
|
+
@property
|
|
112
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
113
|
+
return {
|
|
114
|
+
"region": self.location,
|
|
115
|
+
}
|
|
116
|
+
|
|
109
117
|
def execute(self, context: Context):
|
|
110
118
|
hook = CloudBuildHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
|
|
111
119
|
result = hook.cancel_build(
|
|
@@ -117,14 +125,12 @@ class CloudBuildCancelBuildOperator(GoogleCloudBaseOperator):
|
|
|
117
125
|
location=self.location,
|
|
118
126
|
)
|
|
119
127
|
|
|
120
|
-
|
|
128
|
+
context["task_instance"].xcom_push(key="id", value=result.id)
|
|
121
129
|
project_id = self.project_id or hook.project_id
|
|
122
130
|
if project_id:
|
|
123
131
|
CloudBuildLink.persist(
|
|
124
132
|
context=context,
|
|
125
|
-
task_instance=self,
|
|
126
133
|
project_id=project_id,
|
|
127
|
-
region=self.location,
|
|
128
134
|
build_id=result.id,
|
|
129
135
|
)
|
|
130
136
|
return Build.to_dict(result)
|
|
@@ -208,6 +214,12 @@ class CloudBuildCreateBuildOperator(GoogleCloudBaseOperator):
|
|
|
208
214
|
if self.build_raw.endswith(".json"):
|
|
209
215
|
self.build = json.loads(file.read())
|
|
210
216
|
|
|
217
|
+
@property
|
|
218
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
219
|
+
return {
|
|
220
|
+
"region": self.location,
|
|
221
|
+
}
|
|
222
|
+
|
|
211
223
|
def execute(self, context: Context):
|
|
212
224
|
hook = CloudBuildHook(
|
|
213
225
|
gcp_conn_id=self.gcp_conn_id,
|
|
@@ -223,7 +235,7 @@ class CloudBuildCreateBuildOperator(GoogleCloudBaseOperator):
|
|
|
223
235
|
metadata=self.metadata,
|
|
224
236
|
location=self.location,
|
|
225
237
|
)
|
|
226
|
-
|
|
238
|
+
context["task_instance"].xcom_push(key="id", value=self.id_)
|
|
227
239
|
if not self.wait:
|
|
228
240
|
return Build.to_dict(
|
|
229
241
|
hook.get_build(id_=self.id_, project_id=self.project_id, location=self.location)
|
|
@@ -249,9 +261,7 @@ class CloudBuildCreateBuildOperator(GoogleCloudBaseOperator):
|
|
|
249
261
|
if project_id:
|
|
250
262
|
CloudBuildLink.persist(
|
|
251
263
|
context=context,
|
|
252
|
-
task_instance=self,
|
|
253
264
|
project_id=project_id,
|
|
254
|
-
region=self.location,
|
|
255
265
|
build_id=cloud_build_instance_result.id,
|
|
256
266
|
)
|
|
257
267
|
return Build.to_dict(cloud_build_instance_result)
|
|
@@ -267,14 +277,11 @@ class CloudBuildCreateBuildOperator(GoogleCloudBaseOperator):
|
|
|
267
277
|
if project_id:
|
|
268
278
|
CloudBuildLink.persist(
|
|
269
279
|
context=context,
|
|
270
|
-
task_instance=self,
|
|
271
280
|
project_id=project_id,
|
|
272
|
-
region=self.location,
|
|
273
281
|
build_id=event["id_"],
|
|
274
282
|
)
|
|
275
283
|
return event["instance"]
|
|
276
|
-
|
|
277
|
-
raise AirflowException(f"Unexpected error in the operation: {event['message']}")
|
|
284
|
+
raise AirflowException(f"Unexpected error in the operation: {event['message']}")
|
|
278
285
|
|
|
279
286
|
|
|
280
287
|
class CloudBuildCreateBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
@@ -335,6 +342,12 @@ class CloudBuildCreateBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
|
335
342
|
self.impersonation_chain = impersonation_chain
|
|
336
343
|
self.location = location
|
|
337
344
|
|
|
345
|
+
@property
|
|
346
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
347
|
+
return {
|
|
348
|
+
"region": self.location,
|
|
349
|
+
}
|
|
350
|
+
|
|
338
351
|
def execute(self, context: Context):
|
|
339
352
|
hook = CloudBuildHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
|
|
340
353
|
result = hook.create_build_trigger(
|
|
@@ -345,21 +358,17 @@ class CloudBuildCreateBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
|
345
358
|
metadata=self.metadata,
|
|
346
359
|
location=self.location,
|
|
347
360
|
)
|
|
348
|
-
|
|
361
|
+
context["task_instance"].xcom_push(key="id", value=result.id)
|
|
349
362
|
project_id = self.project_id or hook.project_id
|
|
350
363
|
if project_id:
|
|
351
364
|
CloudBuildTriggerDetailsLink.persist(
|
|
352
365
|
context=context,
|
|
353
|
-
task_instance=self,
|
|
354
366
|
project_id=project_id,
|
|
355
|
-
region=self.location,
|
|
356
367
|
trigger_id=result.id,
|
|
357
368
|
)
|
|
358
369
|
CloudBuildTriggersListLink.persist(
|
|
359
370
|
context=context,
|
|
360
|
-
task_instance=self,
|
|
361
371
|
project_id=project_id,
|
|
362
|
-
region=self.location,
|
|
363
372
|
)
|
|
364
373
|
return BuildTrigger.to_dict(result)
|
|
365
374
|
|
|
@@ -418,6 +427,12 @@ class CloudBuildDeleteBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
|
418
427
|
self.impersonation_chain = impersonation_chain
|
|
419
428
|
self.location = location
|
|
420
429
|
|
|
430
|
+
@property
|
|
431
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
432
|
+
return {
|
|
433
|
+
"region": self.location,
|
|
434
|
+
}
|
|
435
|
+
|
|
421
436
|
def execute(self, context: Context):
|
|
422
437
|
hook = CloudBuildHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
|
|
423
438
|
hook.delete_build_trigger(
|
|
@@ -432,9 +447,7 @@ class CloudBuildDeleteBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
|
432
447
|
if project_id:
|
|
433
448
|
CloudBuildTriggersListLink.persist(
|
|
434
449
|
context=context,
|
|
435
|
-
task_instance=self,
|
|
436
450
|
project_id=project_id,
|
|
437
|
-
region=self.location,
|
|
438
451
|
)
|
|
439
452
|
|
|
440
453
|
|
|
@@ -492,6 +505,12 @@ class CloudBuildGetBuildOperator(GoogleCloudBaseOperator):
|
|
|
492
505
|
self.impersonation_chain = impersonation_chain
|
|
493
506
|
self.location = location
|
|
494
507
|
|
|
508
|
+
@property
|
|
509
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
510
|
+
return {
|
|
511
|
+
"region": self.location,
|
|
512
|
+
}
|
|
513
|
+
|
|
495
514
|
def execute(self, context: Context):
|
|
496
515
|
hook = CloudBuildHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
|
|
497
516
|
result = hook.get_build(
|
|
@@ -506,9 +525,7 @@ class CloudBuildGetBuildOperator(GoogleCloudBaseOperator):
|
|
|
506
525
|
if project_id:
|
|
507
526
|
CloudBuildLink.persist(
|
|
508
527
|
context=context,
|
|
509
|
-
task_instance=self,
|
|
510
528
|
project_id=project_id,
|
|
511
|
-
region=self.location,
|
|
512
529
|
build_id=result.id,
|
|
513
530
|
)
|
|
514
531
|
return Build.to_dict(result)
|
|
@@ -568,6 +585,12 @@ class CloudBuildGetBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
|
568
585
|
self.impersonation_chain = impersonation_chain
|
|
569
586
|
self.location = location
|
|
570
587
|
|
|
588
|
+
@property
|
|
589
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
590
|
+
return {
|
|
591
|
+
"region": self.location,
|
|
592
|
+
}
|
|
593
|
+
|
|
571
594
|
def execute(self, context: Context):
|
|
572
595
|
hook = CloudBuildHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
|
|
573
596
|
result = hook.get_build_trigger(
|
|
@@ -582,9 +605,7 @@ class CloudBuildGetBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
|
582
605
|
if project_id:
|
|
583
606
|
CloudBuildTriggerDetailsLink.persist(
|
|
584
607
|
context=context,
|
|
585
|
-
task_instance=self,
|
|
586
608
|
project_id=project_id,
|
|
587
|
-
region=self.location,
|
|
588
609
|
trigger_id=result.id,
|
|
589
610
|
)
|
|
590
611
|
return BuildTrigger.to_dict(result)
|
|
@@ -648,6 +669,12 @@ class CloudBuildListBuildTriggersOperator(GoogleCloudBaseOperator):
|
|
|
648
669
|
self.gcp_conn_id = gcp_conn_id
|
|
649
670
|
self.impersonation_chain = impersonation_chain
|
|
650
671
|
|
|
672
|
+
@property
|
|
673
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
674
|
+
return {
|
|
675
|
+
"region": self.location,
|
|
676
|
+
}
|
|
677
|
+
|
|
651
678
|
def execute(self, context: Context):
|
|
652
679
|
hook = CloudBuildHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
|
|
653
680
|
results = hook.list_build_triggers(
|
|
@@ -663,9 +690,7 @@ class CloudBuildListBuildTriggersOperator(GoogleCloudBaseOperator):
|
|
|
663
690
|
if project_id:
|
|
664
691
|
CloudBuildTriggersListLink.persist(
|
|
665
692
|
context=context,
|
|
666
|
-
task_instance=self,
|
|
667
693
|
project_id=project_id,
|
|
668
|
-
region=self.location,
|
|
669
694
|
)
|
|
670
695
|
return [BuildTrigger.to_dict(result) for result in results]
|
|
671
696
|
|
|
@@ -728,6 +753,12 @@ class CloudBuildListBuildsOperator(GoogleCloudBaseOperator):
|
|
|
728
753
|
self.gcp_conn_id = gcp_conn_id
|
|
729
754
|
self.impersonation_chain = impersonation_chain
|
|
730
755
|
|
|
756
|
+
@property
|
|
757
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
758
|
+
return {
|
|
759
|
+
"region": self.location,
|
|
760
|
+
}
|
|
761
|
+
|
|
731
762
|
def execute(self, context: Context):
|
|
732
763
|
hook = CloudBuildHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
|
|
733
764
|
results = hook.list_builds(
|
|
@@ -742,7 +773,8 @@ class CloudBuildListBuildsOperator(GoogleCloudBaseOperator):
|
|
|
742
773
|
project_id = self.project_id or hook.project_id
|
|
743
774
|
if project_id:
|
|
744
775
|
CloudBuildListLink.persist(
|
|
745
|
-
context=context,
|
|
776
|
+
context=context,
|
|
777
|
+
project_id=project_id,
|
|
746
778
|
)
|
|
747
779
|
return [Build.to_dict(result) for result in results]
|
|
748
780
|
|
|
@@ -804,6 +836,12 @@ class CloudBuildRetryBuildOperator(GoogleCloudBaseOperator):
|
|
|
804
836
|
self.impersonation_chain = impersonation_chain
|
|
805
837
|
self.location = location
|
|
806
838
|
|
|
839
|
+
@property
|
|
840
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
841
|
+
return {
|
|
842
|
+
"region": self.location,
|
|
843
|
+
}
|
|
844
|
+
|
|
807
845
|
def execute(self, context: Context):
|
|
808
846
|
hook = CloudBuildHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
|
|
809
847
|
result = hook.retry_build(
|
|
@@ -816,14 +854,12 @@ class CloudBuildRetryBuildOperator(GoogleCloudBaseOperator):
|
|
|
816
854
|
location=self.location,
|
|
817
855
|
)
|
|
818
856
|
|
|
819
|
-
|
|
857
|
+
context["task_instance"].xcom_push(key="id", value=result.id)
|
|
820
858
|
project_id = self.project_id or hook.project_id
|
|
821
859
|
if project_id:
|
|
822
860
|
CloudBuildLink.persist(
|
|
823
861
|
context=context,
|
|
824
|
-
task_instance=self,
|
|
825
862
|
project_id=project_id,
|
|
826
|
-
region=self.location,
|
|
827
863
|
build_id=result.id,
|
|
828
864
|
)
|
|
829
865
|
return Build.to_dict(result)
|
|
@@ -890,6 +926,12 @@ class CloudBuildRunBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
|
890
926
|
self.impersonation_chain = impersonation_chain
|
|
891
927
|
self.location = location
|
|
892
928
|
|
|
929
|
+
@property
|
|
930
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
931
|
+
return {
|
|
932
|
+
"region": self.location,
|
|
933
|
+
}
|
|
934
|
+
|
|
893
935
|
def execute(self, context: Context):
|
|
894
936
|
hook = CloudBuildHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
|
|
895
937
|
result = hook.run_build_trigger(
|
|
@@ -902,14 +944,12 @@ class CloudBuildRunBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
|
902
944
|
metadata=self.metadata,
|
|
903
945
|
location=self.location,
|
|
904
946
|
)
|
|
905
|
-
|
|
947
|
+
context["task_instance"].xcom_push(key="id", value=result.id)
|
|
906
948
|
project_id = self.project_id or hook.project_id
|
|
907
949
|
if project_id:
|
|
908
950
|
CloudBuildLink.persist(
|
|
909
951
|
context=context,
|
|
910
|
-
task_instance=self,
|
|
911
952
|
project_id=project_id,
|
|
912
|
-
region=self.location,
|
|
913
953
|
build_id=result.id,
|
|
914
954
|
)
|
|
915
955
|
return Build.to_dict(result)
|
|
@@ -973,6 +1013,12 @@ class CloudBuildUpdateBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
|
973
1013
|
self.impersonation_chain = impersonation_chain
|
|
974
1014
|
self.location = location
|
|
975
1015
|
|
|
1016
|
+
@property
|
|
1017
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
1018
|
+
return {
|
|
1019
|
+
"region": self.location,
|
|
1020
|
+
}
|
|
1021
|
+
|
|
976
1022
|
def execute(self, context: Context):
|
|
977
1023
|
hook = CloudBuildHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
|
|
978
1024
|
result = hook.update_build_trigger(
|
|
@@ -984,14 +1030,12 @@ class CloudBuildUpdateBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
|
984
1030
|
metadata=self.metadata,
|
|
985
1031
|
location=self.location,
|
|
986
1032
|
)
|
|
987
|
-
|
|
1033
|
+
context["task_instance"].xcom_push(key="id", value=result.id)
|
|
988
1034
|
project_id = self.project_id or hook.project_id
|
|
989
1035
|
if project_id:
|
|
990
1036
|
CloudBuildTriggerDetailsLink.persist(
|
|
991
1037
|
context=context,
|
|
992
|
-
task_instance=self,
|
|
993
1038
|
project_id=project_id,
|
|
994
|
-
region=self.location,
|
|
995
1039
|
trigger_id=result.id,
|
|
996
1040
|
)
|
|
997
1041
|
return BuildTrigger.to_dict(result)
|