apache-airflow-providers-google 14.0.0__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
- airflow/providers/google/3rd-party-licenses/NOTICE +5 -0
- airflow/providers/google/__init__.py +3 -3
- airflow/providers/google/_vendor/__init__.py +0 -0
- airflow/providers/google/_vendor/json_merge_patch.py +91 -0
- airflow/providers/google/ads/hooks/ads.py +52 -43
- airflow/providers/google/ads/operators/ads.py +2 -2
- airflow/providers/google/ads/transfers/ads_to_gcs.py +3 -19
- airflow/providers/google/assets/gcs.py +1 -11
- airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
- airflow/providers/google/cloud/bundles/gcs.py +161 -0
- airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
- airflow/providers/google/cloud/hooks/bigquery.py +195 -318
- airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
- airflow/providers/google/cloud/hooks/bigtable.py +3 -2
- airflow/providers/google/cloud/hooks/cloud_batch.py +8 -9
- airflow/providers/google/cloud/hooks/cloud_build.py +6 -65
- airflow/providers/google/cloud/hooks/cloud_composer.py +292 -24
- airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
- airflow/providers/google/cloud/hooks/cloud_run.py +20 -11
- airflow/providers/google/cloud/hooks/cloud_sql.py +136 -64
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +35 -15
- airflow/providers/google/cloud/hooks/compute.py +7 -6
- airflow/providers/google/cloud/hooks/compute_ssh.py +7 -4
- airflow/providers/google/cloud/hooks/datacatalog.py +12 -3
- airflow/providers/google/cloud/hooks/dataflow.py +87 -242
- airflow/providers/google/cloud/hooks/dataform.py +9 -14
- airflow/providers/google/cloud/hooks/datafusion.py +7 -9
- airflow/providers/google/cloud/hooks/dataplex.py +13 -12
- airflow/providers/google/cloud/hooks/dataprep.py +2 -2
- airflow/providers/google/cloud/hooks/dataproc.py +76 -74
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/hooks/dlp.py +5 -4
- airflow/providers/google/cloud/hooks/gcs.py +144 -33
- airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
- airflow/providers/google/cloud/hooks/kms.py +3 -2
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +22 -17
- airflow/providers/google/cloud/hooks/looker.py +6 -1
- airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
- airflow/providers/google/cloud/hooks/mlengine.py +7 -8
- airflow/providers/google/cloud/hooks/natural_language.py +3 -2
- airflow/providers/google/cloud/hooks/os_login.py +3 -2
- airflow/providers/google/cloud/hooks/pubsub.py +6 -6
- airflow/providers/google/cloud/hooks/secret_manager.py +105 -12
- airflow/providers/google/cloud/hooks/spanner.py +75 -10
- airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
- airflow/providers/google/cloud/hooks/stackdriver.py +18 -18
- airflow/providers/google/cloud/hooks/tasks.py +4 -3
- airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
- airflow/providers/google/cloud/hooks/translate.py +8 -17
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +8 -222
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +9 -15
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +33 -283
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +311 -10
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +7 -13
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +8 -12
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
- airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
- airflow/providers/google/cloud/hooks/vision.py +7 -7
- airflow/providers/google/cloud/hooks/workflows.py +4 -3
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +77 -7
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -46
- airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -90
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +11 -89
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +11 -61
- airflow/providers/google/cloud/links/managed_kafka.py +11 -51
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +17 -187
- airflow/providers/google/cloud/links/vertex_ai.py +28 -195
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/log/gcs_task_handler.py +166 -118
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +14 -9
- airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
- airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
- airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
- airflow/providers/google/cloud/openlineage/facets.py +141 -40
- airflow/providers/google/cloud/openlineage/mixins.py +14 -13
- airflow/providers/google/cloud/openlineage/utils.py +19 -3
- airflow/providers/google/cloud/operators/alloy_db.py +76 -61
- airflow/providers/google/cloud/operators/bigquery.py +104 -667
- airflow/providers/google/cloud/operators/bigquery_dts.py +12 -12
- airflow/providers/google/cloud/operators/bigtable.py +38 -7
- airflow/providers/google/cloud/operators/cloud_base.py +22 -1
- airflow/providers/google/cloud/operators/cloud_batch.py +18 -18
- airflow/providers/google/cloud/operators/cloud_build.py +80 -36
- airflow/providers/google/cloud/operators/cloud_composer.py +157 -71
- airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
- airflow/providers/google/cloud/operators/cloud_memorystore.py +74 -46
- airflow/providers/google/cloud/operators/cloud_run.py +39 -20
- airflow/providers/google/cloud/operators/cloud_sql.py +46 -61
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +92 -14
- airflow/providers/google/cloud/operators/compute.py +18 -50
- airflow/providers/google/cloud/operators/datacatalog.py +167 -29
- airflow/providers/google/cloud/operators/dataflow.py +38 -15
- airflow/providers/google/cloud/operators/dataform.py +19 -7
- airflow/providers/google/cloud/operators/datafusion.py +43 -43
- airflow/providers/google/cloud/operators/dataplex.py +212 -126
- airflow/providers/google/cloud/operators/dataprep.py +1 -5
- airflow/providers/google/cloud/operators/dataproc.py +134 -207
- airflow/providers/google/cloud/operators/dataproc_metastore.py +102 -84
- airflow/providers/google/cloud/operators/datastore.py +22 -6
- airflow/providers/google/cloud/operators/dlp.py +24 -45
- airflow/providers/google/cloud/operators/functions.py +21 -14
- airflow/providers/google/cloud/operators/gcs.py +15 -12
- airflow/providers/google/cloud/operators/gen_ai.py +389 -0
- airflow/providers/google/cloud/operators/kubernetes_engine.py +115 -106
- airflow/providers/google/cloud/operators/looker.py +1 -1
- airflow/providers/google/cloud/operators/managed_kafka.py +362 -40
- airflow/providers/google/cloud/operators/natural_language.py +5 -3
- airflow/providers/google/cloud/operators/pubsub.py +69 -21
- airflow/providers/google/cloud/operators/spanner.py +53 -45
- airflow/providers/google/cloud/operators/speech_to_text.py +5 -4
- airflow/providers/google/cloud/operators/stackdriver.py +5 -11
- airflow/providers/google/cloud/operators/tasks.py +6 -15
- airflow/providers/google/cloud/operators/text_to_speech.py +4 -3
- airflow/providers/google/cloud/operators/translate.py +46 -20
- airflow/providers/google/cloud/operators/translate_speech.py +4 -3
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +44 -34
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +34 -12
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +62 -53
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +75 -11
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +48 -12
- airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -116
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +16 -12
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +62 -14
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +35 -10
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
- airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
- airflow/providers/google/cloud/operators/vision.py +7 -5
- airflow/providers/google/cloud/operators/workflows.py +24 -19
- airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
- airflow/providers/google/cloud/sensors/bigquery.py +2 -2
- airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -4
- airflow/providers/google/cloud/sensors/bigtable.py +14 -6
- airflow/providers/google/cloud/sensors/cloud_composer.py +535 -33
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -5
- airflow/providers/google/cloud/sensors/dataflow.py +27 -10
- airflow/providers/google/cloud/sensors/dataform.py +2 -2
- airflow/providers/google/cloud/sensors/datafusion.py +4 -4
- airflow/providers/google/cloud/sensors/dataplex.py +7 -5
- airflow/providers/google/cloud/sensors/dataprep.py +2 -2
- airflow/providers/google/cloud/sensors/dataproc.py +10 -9
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/sensors/gcs.py +22 -21
- airflow/providers/google/cloud/sensors/looker.py +5 -5
- airflow/providers/google/cloud/sensors/pubsub.py +20 -20
- airflow/providers/google/cloud/sensors/tasks.py +2 -2
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -2
- airflow/providers/google/cloud/sensors/workflows.py +6 -4
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +14 -13
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
- airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +18 -22
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -5
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +45 -38
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/gcs_to_local.py +5 -3
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +10 -4
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
- airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
- airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +44 -12
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +12 -6
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +36 -14
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
- airflow/providers/google/cloud/triggers/bigquery.py +75 -34
- airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +3 -2
- airflow/providers/google/cloud/triggers/cloud_composer.py +303 -47
- airflow/providers/google/cloud/triggers/cloud_run.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +96 -5
- airflow/providers/google/cloud/triggers/dataflow.py +125 -2
- airflow/providers/google/cloud/triggers/datafusion.py +1 -1
- airflow/providers/google/cloud/triggers/dataplex.py +16 -3
- airflow/providers/google/cloud/triggers/dataproc.py +124 -53
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +46 -28
- airflow/providers/google/cloud/triggers/mlengine.py +1 -1
- airflow/providers/google/cloud/triggers/pubsub.py +17 -20
- airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
- airflow/providers/google/cloud/utils/bigquery.py +5 -7
- airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
- airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
- airflow/providers/google/cloud/utils/field_validator.py +1 -2
- airflow/providers/google/cloud/utils/validators.py +43 -0
- airflow/providers/google/common/auth_backend/google_openid.py +26 -9
- airflow/providers/google/common/consts.py +2 -1
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +40 -43
- airflow/providers/google/common/hooks/operation_helpers.py +78 -0
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/common/utils/get_secret.py +31 -0
- airflow/providers/google/common/utils/id_token_credentials.py +4 -5
- airflow/providers/google/firebase/operators/firestore.py +2 -2
- airflow/providers/google/get_provider_info.py +61 -216
- airflow/providers/google/go_module_utils.py +35 -3
- airflow/providers/google/leveldb/hooks/leveldb.py +30 -6
- airflow/providers/google/leveldb/operators/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
- airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
- airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +4 -5
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -6
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +5 -5
- airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
- airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
- airflow/providers/google/marketing_platform/sensors/display_video.py +4 -64
- airflow/providers/google/suite/hooks/calendar.py +1 -1
- airflow/providers/google/suite/hooks/drive.py +2 -2
- airflow/providers/google/suite/hooks/sheets.py +15 -1
- airflow/providers/google/suite/operators/sheets.py +8 -3
- airflow/providers/google/suite/sensors/drive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
- airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
- airflow/providers/google/version_compat.py +15 -1
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/METADATA +117 -72
- apache_airflow_providers_google-19.1.0rc1.dist-info/RECORD +331 -0
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/WHEEL +1 -1
- apache_airflow_providers_google-19.1.0rc1.dist-info/licenses/NOTICE +5 -0
- airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
- airflow/providers/google/cloud/hooks/automl.py +0 -679
- airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
- airflow/providers/google/cloud/links/automl.py +0 -193
- airflow/providers/google/cloud/operators/automl.py +0 -1360
- airflow/providers/google/cloud/operators/life_sciences.py +0 -119
- airflow/providers/google/cloud/operators/mlengine.py +0 -1515
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +0 -273
- apache_airflow_providers_google-14.0.0.dist-info/RECORD +0 -318
- /airflow/providers/google/cloud/{example_dags → bundles}/__init__.py +0 -0
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/entry_points.txt +0 -0
- {airflow/providers/google → apache_airflow_providers_google-19.1.0rc1.dist-info/licenses}/LICENSE +0 -0
|
@@ -27,7 +27,11 @@ Operators for Google Cloud Memorystore service.
|
|
|
27
27
|
from __future__ import annotations
|
|
28
28
|
|
|
29
29
|
from collections.abc import Sequence
|
|
30
|
-
from typing import TYPE_CHECKING
|
|
30
|
+
from typing import TYPE_CHECKING, Any
|
|
31
|
+
|
|
32
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
33
|
+
from google.cloud.memcache_v1beta2.types import cloud_memcache
|
|
34
|
+
from google.cloud.redis_v1 import FailoverInstanceRequest, InputConfig, Instance, OutputConfig
|
|
31
35
|
|
|
32
36
|
from airflow.providers.google.cloud.hooks.cloud_memorystore import (
|
|
33
37
|
CloudMemorystoreHook,
|
|
@@ -41,15 +45,13 @@ from airflow.providers.google.cloud.links.cloud_memorystore import (
|
|
|
41
45
|
)
|
|
42
46
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
43
47
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
44
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
45
|
-
from google.cloud.memcache_v1beta2.types import cloud_memcache
|
|
46
|
-
from google.cloud.redis_v1 import FailoverInstanceRequest, InputConfig, Instance, OutputConfig
|
|
47
48
|
|
|
48
49
|
if TYPE_CHECKING:
|
|
49
|
-
from airflow.utils.context import Context
|
|
50
50
|
from google.api_core.retry import Retry
|
|
51
51
|
from google.protobuf.field_mask_pb2 import FieldMask
|
|
52
52
|
|
|
53
|
+
from airflow.providers.common.compat.sdk import Context
|
|
54
|
+
|
|
53
55
|
|
|
54
56
|
class CloudMemorystoreCreateInstanceOperator(GoogleCloudBaseOperator):
|
|
55
57
|
"""
|
|
@@ -131,6 +133,13 @@ class CloudMemorystoreCreateInstanceOperator(GoogleCloudBaseOperator):
|
|
|
131
133
|
self.gcp_conn_id = gcp_conn_id
|
|
132
134
|
self.impersonation_chain = impersonation_chain
|
|
133
135
|
|
|
136
|
+
@property
|
|
137
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
138
|
+
return {
|
|
139
|
+
"instance_id": self.instance_id,
|
|
140
|
+
"location_id": self.location,
|
|
141
|
+
}
|
|
142
|
+
|
|
134
143
|
def execute(self, context: Context):
|
|
135
144
|
hook = CloudMemorystoreHook(
|
|
136
145
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -146,9 +155,6 @@ class CloudMemorystoreCreateInstanceOperator(GoogleCloudBaseOperator):
|
|
|
146
155
|
)
|
|
147
156
|
RedisInstanceDetailsLink.persist(
|
|
148
157
|
context=context,
|
|
149
|
-
task_instance=self,
|
|
150
|
-
instance_id=self.instance_id,
|
|
151
|
-
location_id=self.location,
|
|
152
158
|
project_id=self.project_id or hook.project_id,
|
|
153
159
|
)
|
|
154
160
|
return Instance.to_dict(result)
|
|
@@ -302,6 +308,13 @@ class CloudMemorystoreExportInstanceOperator(GoogleCloudBaseOperator):
|
|
|
302
308
|
self.gcp_conn_id = gcp_conn_id
|
|
303
309
|
self.impersonation_chain = impersonation_chain
|
|
304
310
|
|
|
311
|
+
@property
|
|
312
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
313
|
+
return {
|
|
314
|
+
"instance_id": self.instance,
|
|
315
|
+
"location_id": self.location,
|
|
316
|
+
}
|
|
317
|
+
|
|
305
318
|
def execute(self, context: Context) -> None:
|
|
306
319
|
hook = CloudMemorystoreHook(
|
|
307
320
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -318,9 +331,6 @@ class CloudMemorystoreExportInstanceOperator(GoogleCloudBaseOperator):
|
|
|
318
331
|
)
|
|
319
332
|
RedisInstanceDetailsLink.persist(
|
|
320
333
|
context=context,
|
|
321
|
-
task_instance=self,
|
|
322
|
-
instance_id=self.instance,
|
|
323
|
-
location_id=self.location,
|
|
324
334
|
project_id=self.project_id or hook.project_id,
|
|
325
335
|
)
|
|
326
336
|
|
|
@@ -395,6 +405,13 @@ class CloudMemorystoreFailoverInstanceOperator(GoogleCloudBaseOperator):
|
|
|
395
405
|
self.gcp_conn_id = gcp_conn_id
|
|
396
406
|
self.impersonation_chain = impersonation_chain
|
|
397
407
|
|
|
408
|
+
@property
|
|
409
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
410
|
+
return {
|
|
411
|
+
"instance_id": self.instance,
|
|
412
|
+
"location_id": self.location,
|
|
413
|
+
}
|
|
414
|
+
|
|
398
415
|
def execute(self, context: Context) -> None:
|
|
399
416
|
hook = CloudMemorystoreHook(
|
|
400
417
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -410,9 +427,6 @@ class CloudMemorystoreFailoverInstanceOperator(GoogleCloudBaseOperator):
|
|
|
410
427
|
)
|
|
411
428
|
RedisInstanceDetailsLink.persist(
|
|
412
429
|
context=context,
|
|
413
|
-
task_instance=self,
|
|
414
|
-
instance_id=self.instance,
|
|
415
|
-
location_id=self.location,
|
|
416
430
|
project_id=self.project_id or hook.project_id,
|
|
417
431
|
)
|
|
418
432
|
|
|
@@ -480,6 +494,13 @@ class CloudMemorystoreGetInstanceOperator(GoogleCloudBaseOperator):
|
|
|
480
494
|
self.gcp_conn_id = gcp_conn_id
|
|
481
495
|
self.impersonation_chain = impersonation_chain
|
|
482
496
|
|
|
497
|
+
@property
|
|
498
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
499
|
+
return {
|
|
500
|
+
"instance_id": self.instance,
|
|
501
|
+
"location_id": self.location,
|
|
502
|
+
}
|
|
503
|
+
|
|
483
504
|
def execute(self, context: Context):
|
|
484
505
|
hook = CloudMemorystoreHook(
|
|
485
506
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -494,9 +515,6 @@ class CloudMemorystoreGetInstanceOperator(GoogleCloudBaseOperator):
|
|
|
494
515
|
)
|
|
495
516
|
RedisInstanceDetailsLink.persist(
|
|
496
517
|
context=context,
|
|
497
|
-
task_instance=self,
|
|
498
|
-
instance_id=self.instance,
|
|
499
|
-
location_id=self.location,
|
|
500
518
|
project_id=self.project_id or hook.project_id,
|
|
501
519
|
)
|
|
502
520
|
return Instance.to_dict(result)
|
|
@@ -575,6 +593,13 @@ class CloudMemorystoreImportOperator(GoogleCloudBaseOperator):
|
|
|
575
593
|
self.gcp_conn_id = gcp_conn_id
|
|
576
594
|
self.impersonation_chain = impersonation_chain
|
|
577
595
|
|
|
596
|
+
@property
|
|
597
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
598
|
+
return {
|
|
599
|
+
"instance_id": self.instance,
|
|
600
|
+
"location_id": self.location,
|
|
601
|
+
}
|
|
602
|
+
|
|
578
603
|
def execute(self, context: Context) -> None:
|
|
579
604
|
hook = CloudMemorystoreHook(
|
|
580
605
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -590,9 +615,6 @@ class CloudMemorystoreImportOperator(GoogleCloudBaseOperator):
|
|
|
590
615
|
)
|
|
591
616
|
RedisInstanceDetailsLink.persist(
|
|
592
617
|
context=context,
|
|
593
|
-
task_instance=self,
|
|
594
|
-
instance_id=self.instance,
|
|
595
|
-
location_id=self.location,
|
|
596
618
|
project_id=self.project_id or hook.project_id,
|
|
597
619
|
)
|
|
598
620
|
|
|
@@ -678,7 +700,6 @@ class CloudMemorystoreListInstancesOperator(GoogleCloudBaseOperator):
|
|
|
678
700
|
)
|
|
679
701
|
RedisInstanceListLink.persist(
|
|
680
702
|
context=context,
|
|
681
|
-
task_instance=self,
|
|
682
703
|
project_id=self.project_id or hook.project_id,
|
|
683
704
|
)
|
|
684
705
|
instances = [Instance.to_dict(a) for a in result]
|
|
@@ -787,7 +808,6 @@ class CloudMemorystoreUpdateInstanceOperator(GoogleCloudBaseOperator):
|
|
|
787
808
|
location_id, instance_id = res.name.split("/")[-3::2]
|
|
788
809
|
RedisInstanceDetailsLink.persist(
|
|
789
810
|
context=context,
|
|
790
|
-
task_instance=self,
|
|
791
811
|
instance_id=self.instance_id or instance_id,
|
|
792
812
|
location_id=self.location or location_id,
|
|
793
813
|
project_id=self.project_id or hook.project_id,
|
|
@@ -880,7 +900,6 @@ class CloudMemorystoreScaleInstanceOperator(GoogleCloudBaseOperator):
|
|
|
880
900
|
location_id, instance_id = res.name.split("/")[-3::2]
|
|
881
901
|
RedisInstanceDetailsLink.persist(
|
|
882
902
|
context=context,
|
|
883
|
-
task_instance=self,
|
|
884
903
|
instance_id=self.instance_id or instance_id,
|
|
885
904
|
location_id=self.location or location_id,
|
|
886
905
|
project_id=self.project_id or hook.project_id,
|
|
@@ -1000,7 +1019,6 @@ class CloudMemorystoreCreateInstanceAndImportOperator(GoogleCloudBaseOperator):
|
|
|
1000
1019
|
)
|
|
1001
1020
|
RedisInstanceDetailsLink.persist(
|
|
1002
1021
|
context=context,
|
|
1003
|
-
task_instance=self,
|
|
1004
1022
|
instance_id=self.instance_id,
|
|
1005
1023
|
location_id=self.location,
|
|
1006
1024
|
project_id=self.project_id or hook.project_id,
|
|
@@ -1169,6 +1187,14 @@ class CloudMemorystoreMemcachedApplyParametersOperator(GoogleCloudBaseOperator):
|
|
|
1169
1187
|
self.gcp_conn_id = gcp_conn_id
|
|
1170
1188
|
self.impersonation_chain = impersonation_chain
|
|
1171
1189
|
|
|
1190
|
+
@property
|
|
1191
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
1192
|
+
return {
|
|
1193
|
+
"instance_id": self.instance_id,
|
|
1194
|
+
"location_id": self.location,
|
|
1195
|
+
"project_id": self.project_id,
|
|
1196
|
+
}
|
|
1197
|
+
|
|
1172
1198
|
def execute(self, context: Context):
|
|
1173
1199
|
hook = CloudMemorystoreMemcachedHook(
|
|
1174
1200
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -1183,13 +1209,7 @@ class CloudMemorystoreMemcachedApplyParametersOperator(GoogleCloudBaseOperator):
|
|
|
1183
1209
|
timeout=self.timeout,
|
|
1184
1210
|
metadata=self.metadata,
|
|
1185
1211
|
)
|
|
1186
|
-
MemcachedInstanceDetailsLink.persist(
|
|
1187
|
-
context=context,
|
|
1188
|
-
task_instance=self,
|
|
1189
|
-
instance_id=self.instance_id,
|
|
1190
|
-
location_id=self.location,
|
|
1191
|
-
project_id=self.project_id,
|
|
1192
|
-
)
|
|
1212
|
+
MemcachedInstanceDetailsLink.persist(context=context)
|
|
1193
1213
|
|
|
1194
1214
|
|
|
1195
1215
|
class CloudMemorystoreMemcachedCreateInstanceOperator(GoogleCloudBaseOperator):
|
|
@@ -1261,6 +1281,13 @@ class CloudMemorystoreMemcachedCreateInstanceOperator(GoogleCloudBaseOperator):
|
|
|
1261
1281
|
self.metadata = metadata
|
|
1262
1282
|
self.gcp_conn_id = gcp_conn_id
|
|
1263
1283
|
|
|
1284
|
+
@property
|
|
1285
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
1286
|
+
return {
|
|
1287
|
+
"instance_id": self.instance_id,
|
|
1288
|
+
"location_id": self.location,
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1264
1291
|
def execute(self, context: Context):
|
|
1265
1292
|
hook = CloudMemorystoreMemcachedHook(gcp_conn_id=self.gcp_conn_id)
|
|
1266
1293
|
result = hook.create_instance(
|
|
@@ -1274,9 +1301,6 @@ class CloudMemorystoreMemcachedCreateInstanceOperator(GoogleCloudBaseOperator):
|
|
|
1274
1301
|
)
|
|
1275
1302
|
MemcachedInstanceDetailsLink.persist(
|
|
1276
1303
|
context=context,
|
|
1277
|
-
task_instance=self,
|
|
1278
|
-
instance_id=self.instance_id,
|
|
1279
|
-
location_id=self.location,
|
|
1280
1304
|
project_id=self.project_id or hook.project_id,
|
|
1281
1305
|
)
|
|
1282
1306
|
return cloud_memcache.Instance.to_dict(result)
|
|
@@ -1408,6 +1432,13 @@ class CloudMemorystoreMemcachedGetInstanceOperator(GoogleCloudBaseOperator):
|
|
|
1408
1432
|
self.gcp_conn_id = gcp_conn_id
|
|
1409
1433
|
self.impersonation_chain = impersonation_chain
|
|
1410
1434
|
|
|
1435
|
+
@property
|
|
1436
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
1437
|
+
return {
|
|
1438
|
+
"instance_id": self.instance,
|
|
1439
|
+
"location_id": self.location,
|
|
1440
|
+
}
|
|
1441
|
+
|
|
1411
1442
|
def execute(self, context: Context):
|
|
1412
1443
|
hook = CloudMemorystoreMemcachedHook(
|
|
1413
1444
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -1422,9 +1453,6 @@ class CloudMemorystoreMemcachedGetInstanceOperator(GoogleCloudBaseOperator):
|
|
|
1422
1453
|
)
|
|
1423
1454
|
MemcachedInstanceDetailsLink.persist(
|
|
1424
1455
|
context=context,
|
|
1425
|
-
task_instance=self,
|
|
1426
|
-
instance_id=self.instance,
|
|
1427
|
-
location_id=self.location,
|
|
1428
1456
|
project_id=self.project_id or hook.project_id,
|
|
1429
1457
|
)
|
|
1430
1458
|
return cloud_memcache.Instance.to_dict(result)
|
|
@@ -1504,7 +1532,6 @@ class CloudMemorystoreMemcachedListInstancesOperator(GoogleCloudBaseOperator):
|
|
|
1504
1532
|
)
|
|
1505
1533
|
MemcachedInstanceListLink.persist(
|
|
1506
1534
|
context=context,
|
|
1507
|
-
task_instance=self,
|
|
1508
1535
|
project_id=self.project_id or hook.project_id,
|
|
1509
1536
|
)
|
|
1510
1537
|
instances = [cloud_memcache.Instance.to_dict(a) for a in result]
|
|
@@ -1610,7 +1637,6 @@ class CloudMemorystoreMemcachedUpdateInstanceOperator(GoogleCloudBaseOperator):
|
|
|
1610
1637
|
location_id, instance_id = res.name.split("/")[-3::2]
|
|
1611
1638
|
MemcachedInstanceDetailsLink.persist(
|
|
1612
1639
|
context=context,
|
|
1613
|
-
task_instance=self,
|
|
1614
1640
|
instance_id=self.instance_id or instance_id,
|
|
1615
1641
|
location_id=self.location or location_id,
|
|
1616
1642
|
project_id=self.project_id or hook.project_id,
|
|
@@ -1686,6 +1712,14 @@ class CloudMemorystoreMemcachedUpdateParametersOperator(GoogleCloudBaseOperator)
|
|
|
1686
1712
|
self.gcp_conn_id = gcp_conn_id
|
|
1687
1713
|
self.impersonation_chain = impersonation_chain
|
|
1688
1714
|
|
|
1715
|
+
@property
|
|
1716
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
1717
|
+
return {
|
|
1718
|
+
"instance_id": self.instance_id,
|
|
1719
|
+
"location_id": self.location,
|
|
1720
|
+
"project_id": self.project_id,
|
|
1721
|
+
}
|
|
1722
|
+
|
|
1689
1723
|
def execute(self, context: Context):
|
|
1690
1724
|
hook = CloudMemorystoreMemcachedHook(
|
|
1691
1725
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -1700,10 +1734,4 @@ class CloudMemorystoreMemcachedUpdateParametersOperator(GoogleCloudBaseOperator)
|
|
|
1700
1734
|
timeout=self.timeout,
|
|
1701
1735
|
metadata=self.metadata,
|
|
1702
1736
|
)
|
|
1703
|
-
MemcachedInstanceDetailsLink.persist(
|
|
1704
|
-
context=context,
|
|
1705
|
-
task_instance=self,
|
|
1706
|
-
instance_id=self.instance_id,
|
|
1707
|
-
location_id=self.location,
|
|
1708
|
-
project_id=self.project_id,
|
|
1709
|
-
)
|
|
1737
|
+
MemcachedInstanceDetailsLink.persist(context=context)
|
|
@@ -21,19 +21,22 @@ from collections.abc import Sequence
|
|
|
21
21
|
from typing import TYPE_CHECKING, Any
|
|
22
22
|
|
|
23
23
|
import google.cloud.exceptions
|
|
24
|
+
from google.api_core.exceptions import AlreadyExists
|
|
25
|
+
from google.cloud.run_v2 import Job, Service
|
|
26
|
+
|
|
24
27
|
from airflow.configuration import conf
|
|
25
28
|
from airflow.exceptions import AirflowException
|
|
26
29
|
from airflow.providers.google.cloud.hooks.cloud_run import CloudRunHook, CloudRunServiceHook
|
|
30
|
+
from airflow.providers.google.cloud.links.cloud_run import CloudRunJobLoggingLink
|
|
27
31
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
28
32
|
from airflow.providers.google.cloud.triggers.cloud_run import CloudRunJobFinishedTrigger, RunJobStatus
|
|
29
|
-
from google.api_core.exceptions import AlreadyExists
|
|
30
|
-
from google.cloud.run_v2 import Job, Service
|
|
31
33
|
|
|
32
34
|
if TYPE_CHECKING:
|
|
33
|
-
from airflow.utils.context import Context
|
|
34
35
|
from google.api_core import operation
|
|
35
36
|
from google.cloud.run_v2.types import Execution
|
|
36
37
|
|
|
38
|
+
from airflow.providers.common.compat.sdk import Context
|
|
39
|
+
|
|
37
40
|
|
|
38
41
|
class CloudRunCreateJobOperator(GoogleCloudBaseOperator):
|
|
39
42
|
"""
|
|
@@ -138,7 +141,7 @@ class CloudRunUpdateJobOperator(GoogleCloudBaseOperator):
|
|
|
138
141
|
|
|
139
142
|
class CloudRunDeleteJobOperator(GoogleCloudBaseOperator):
|
|
140
143
|
"""
|
|
141
|
-
Deletes a job and wait for the
|
|
144
|
+
Deletes a job and wait for the operation to be completed. Pushes the deleted job to xcom.
|
|
142
145
|
|
|
143
146
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
|
144
147
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
|
@@ -246,7 +249,7 @@ class CloudRunExecuteJobOperator(GoogleCloudBaseOperator):
|
|
|
246
249
|
|
|
247
250
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
|
248
251
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
|
249
|
-
:param job_name: Required. The name of the job to
|
|
252
|
+
:param job_name: Required. The name of the job to execute.
|
|
250
253
|
:param overrides: Optional map of override values.
|
|
251
254
|
:param gcp_conn_id: The connection ID used to connect to Google Cloud.
|
|
252
255
|
:param polling_period_seconds: Optional. Control the rate of the poll for the result of deferrable run.
|
|
@@ -263,7 +266,17 @@ class CloudRunExecuteJobOperator(GoogleCloudBaseOperator):
|
|
|
263
266
|
:param deferrable: Run the operator in deferrable mode.
|
|
264
267
|
"""
|
|
265
268
|
|
|
266
|
-
|
|
269
|
+
operator_extra_links = (CloudRunJobLoggingLink(),)
|
|
270
|
+
template_fields = (
|
|
271
|
+
"project_id",
|
|
272
|
+
"region",
|
|
273
|
+
"gcp_conn_id",
|
|
274
|
+
"impersonation_chain",
|
|
275
|
+
"job_name",
|
|
276
|
+
"overrides",
|
|
277
|
+
"polling_period_seconds",
|
|
278
|
+
"timeout_seconds",
|
|
279
|
+
)
|
|
267
280
|
|
|
268
281
|
def __init__(
|
|
269
282
|
self,
|
|
@@ -301,24 +314,29 @@ class CloudRunExecuteJobOperator(GoogleCloudBaseOperator):
|
|
|
301
314
|
if self.operation is None:
|
|
302
315
|
raise AirflowException("Operation is None")
|
|
303
316
|
|
|
317
|
+
if self.operation.metadata.log_uri:
|
|
318
|
+
CloudRunJobLoggingLink.persist(
|
|
319
|
+
context=context,
|
|
320
|
+
log_uri=self.operation.metadata.log_uri,
|
|
321
|
+
)
|
|
322
|
+
|
|
304
323
|
if not self.deferrable:
|
|
305
324
|
result: Execution = self._wait_for_operation(self.operation)
|
|
306
325
|
self._fail_if_execution_failed(result)
|
|
307
326
|
job = hook.get_job(job_name=result.job, region=self.region, project_id=self.project_id)
|
|
308
327
|
return Job.to_dict(job)
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
)
|
|
328
|
+
self.defer(
|
|
329
|
+
trigger=CloudRunJobFinishedTrigger(
|
|
330
|
+
operation_name=self.operation.operation.name,
|
|
331
|
+
job_name=self.job_name,
|
|
332
|
+
project_id=self.project_id,
|
|
333
|
+
location=self.region,
|
|
334
|
+
gcp_conn_id=self.gcp_conn_id,
|
|
335
|
+
impersonation_chain=self.impersonation_chain,
|
|
336
|
+
polling_period_seconds=self.polling_period_seconds,
|
|
337
|
+
),
|
|
338
|
+
method_name="execute_complete",
|
|
339
|
+
)
|
|
322
340
|
|
|
323
341
|
def execute_complete(self, context: Context, event: dict):
|
|
324
342
|
status = event["status"]
|
|
@@ -423,9 +441,10 @@ class CloudRunCreateServiceOperator(GoogleCloudBaseOperator):
|
|
|
423
441
|
self.service_name,
|
|
424
442
|
self.region,
|
|
425
443
|
)
|
|
426
|
-
|
|
444
|
+
service = hook.get_service(
|
|
427
445
|
service_name=self.service_name, region=self.region, project_id=self.project_id
|
|
428
446
|
)
|
|
447
|
+
return Service.to_dict(service)
|
|
429
448
|
except google.cloud.exceptions.GoogleCloudError as e:
|
|
430
449
|
self.log.error("An error occurred. Exiting.")
|
|
431
450
|
raise e
|
|
@@ -28,7 +28,7 @@ from googleapiclient.errors import HttpError
|
|
|
28
28
|
|
|
29
29
|
from airflow.configuration import conf
|
|
30
30
|
from airflow.exceptions import AirflowException
|
|
31
|
-
from airflow.
|
|
31
|
+
from airflow.providers.common.compat.sdk import BaseHook
|
|
32
32
|
from airflow.providers.google.cloud.hooks.cloud_sql import CloudSQLDatabaseHook, CloudSQLHook
|
|
33
33
|
from airflow.providers.google.cloud.links.cloud_sql import CloudSQLInstanceDatabaseLink, CloudSQLInstanceLink
|
|
34
34
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
@@ -39,8 +39,8 @@ from airflow.providers.google.common.links.storage import FileDetailsLink
|
|
|
39
39
|
|
|
40
40
|
if TYPE_CHECKING:
|
|
41
41
|
from airflow.models import Connection
|
|
42
|
+
from airflow.providers.common.compat.sdk import Context
|
|
42
43
|
from airflow.providers.openlineage.extractors import OperatorLineage
|
|
43
|
-
from airflow.utils.context import Context
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
SETTINGS = "settings"
|
|
@@ -285,6 +285,12 @@ class CloudSQLBaseOperator(GoogleCloudBaseOperator):
|
|
|
285
285
|
return False
|
|
286
286
|
raise e
|
|
287
287
|
|
|
288
|
+
@property
|
|
289
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
290
|
+
return {
|
|
291
|
+
"instance": self.instance,
|
|
292
|
+
}
|
|
293
|
+
|
|
288
294
|
def execute(self, context: Context):
|
|
289
295
|
pass
|
|
290
296
|
|
|
@@ -384,8 +390,6 @@ class CloudSQLCreateInstanceOperator(CloudSQLBaseOperator):
|
|
|
384
390
|
|
|
385
391
|
CloudSQLInstanceLink.persist(
|
|
386
392
|
context=context,
|
|
387
|
-
task_instance=self,
|
|
388
|
-
cloud_sql_instance=self.instance,
|
|
389
393
|
project_id=self.project_id or hook.project_id,
|
|
390
394
|
)
|
|
391
395
|
|
|
@@ -477,15 +481,12 @@ class CloudSQLInstancePatchOperator(CloudSQLBaseOperator):
|
|
|
477
481
|
f"Cloud SQL instance with ID {self.instance} does not exist. "
|
|
478
482
|
"Please specify another instance to patch."
|
|
479
483
|
)
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
cloud_sql_instance=self.instance,
|
|
485
|
-
project_id=self.project_id or hook.project_id,
|
|
486
|
-
)
|
|
484
|
+
CloudSQLInstanceLink.persist(
|
|
485
|
+
context=context,
|
|
486
|
+
project_id=self.project_id or hook.project_id,
|
|
487
|
+
)
|
|
487
488
|
|
|
488
|
-
|
|
489
|
+
return hook.patch_instance(project_id=self.project_id, body=self.body, instance=self.instance)
|
|
489
490
|
|
|
490
491
|
|
|
491
492
|
class CloudSQLDeleteInstanceOperator(CloudSQLBaseOperator):
|
|
@@ -531,8 +532,7 @@ class CloudSQLDeleteInstanceOperator(CloudSQLBaseOperator):
|
|
|
531
532
|
if not self._check_if_instance_exists(self.instance, hook):
|
|
532
533
|
print(f"Cloud SQL instance with ID {self.instance} does not exist. Aborting delete.")
|
|
533
534
|
return True
|
|
534
|
-
|
|
535
|
-
return hook.delete_instance(project_id=self.project_id, instance=self.instance)
|
|
535
|
+
return hook.delete_instance(project_id=self.project_id, instance=self.instance)
|
|
536
536
|
|
|
537
537
|
|
|
538
538
|
class CloudSQLCloneInstanceOperator(CloudSQLBaseOperator):
|
|
@@ -612,19 +612,18 @@ class CloudSQLCloneInstanceOperator(CloudSQLBaseOperator):
|
|
|
612
612
|
f"Cloud SQL instance with ID {self.instance} does not exist. "
|
|
613
613
|
"Please specify another instance to patch."
|
|
614
614
|
)
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
"
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
**self.clone_context,
|
|
621
|
-
}
|
|
615
|
+
body = {
|
|
616
|
+
"cloneContext": {
|
|
617
|
+
"kind": "sql#cloneContext",
|
|
618
|
+
"destinationInstanceName": self.destination_instance_name,
|
|
619
|
+
**self.clone_context,
|
|
622
620
|
}
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
621
|
+
}
|
|
622
|
+
return hook.clone_instance(
|
|
623
|
+
project_id=self.project_id,
|
|
624
|
+
body=body,
|
|
625
|
+
instance=self.instance,
|
|
626
|
+
)
|
|
628
627
|
|
|
629
628
|
|
|
630
629
|
class CloudSQLCreateInstanceDatabaseOperator(CloudSQLBaseOperator):
|
|
@@ -717,8 +716,6 @@ class CloudSQLCreateInstanceDatabaseOperator(CloudSQLBaseOperator):
|
|
|
717
716
|
)
|
|
718
717
|
CloudSQLInstanceDatabaseLink.persist(
|
|
719
718
|
context=context,
|
|
720
|
-
task_instance=self,
|
|
721
|
-
cloud_sql_instance=self.instance,
|
|
722
719
|
project_id=self.project_id or hook.project_id,
|
|
723
720
|
)
|
|
724
721
|
if self._check_if_db_exists(database, hook):
|
|
@@ -728,8 +725,7 @@ class CloudSQLCreateInstanceDatabaseOperator(CloudSQLBaseOperator):
|
|
|
728
725
|
database,
|
|
729
726
|
)
|
|
730
727
|
return True
|
|
731
|
-
|
|
732
|
-
return hook.create_database(project_id=self.project_id, instance=self.instance, body=self.body)
|
|
728
|
+
return hook.create_database(project_id=self.project_id, instance=self.instance, body=self.body)
|
|
733
729
|
|
|
734
730
|
|
|
735
731
|
class CloudSQLPatchInstanceDatabaseOperator(CloudSQLBaseOperator):
|
|
@@ -824,16 +820,13 @@ class CloudSQLPatchInstanceDatabaseOperator(CloudSQLBaseOperator):
|
|
|
824
820
|
f"Cloud SQL instance with ID {self.instance} does not contain database '{self.database}'. "
|
|
825
821
|
"Please specify another database to patch."
|
|
826
822
|
)
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
return hook.patch_database(
|
|
835
|
-
project_id=self.project_id, instance=self.instance, database=self.database, body=self.body
|
|
836
|
-
)
|
|
823
|
+
CloudSQLInstanceDatabaseLink.persist(
|
|
824
|
+
context=context,
|
|
825
|
+
project_id=self.project_id or hook.project_id,
|
|
826
|
+
)
|
|
827
|
+
return hook.patch_database(
|
|
828
|
+
project_id=self.project_id, instance=self.instance, database=self.database, body=self.body
|
|
829
|
+
)
|
|
837
830
|
|
|
838
831
|
|
|
839
832
|
class CloudSQLDeleteInstanceDatabaseOperator(CloudSQLBaseOperator):
|
|
@@ -910,10 +903,9 @@ class CloudSQLDeleteInstanceDatabaseOperator(CloudSQLBaseOperator):
|
|
|
910
903
|
f"Aborting database delete."
|
|
911
904
|
)
|
|
912
905
|
return True
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
)
|
|
906
|
+
return hook.delete_database(
|
|
907
|
+
project_id=self.project_id, instance=self.instance, database=self.database
|
|
908
|
+
)
|
|
917
909
|
|
|
918
910
|
|
|
919
911
|
class CloudSQLExportInstanceOperator(CloudSQLBaseOperator):
|
|
@@ -1010,13 +1002,10 @@ class CloudSQLExportInstanceOperator(CloudSQLBaseOperator):
|
|
|
1010
1002
|
)
|
|
1011
1003
|
CloudSQLInstanceLink.persist(
|
|
1012
1004
|
context=context,
|
|
1013
|
-
task_instance=self,
|
|
1014
|
-
cloud_sql_instance=self.instance,
|
|
1015
1005
|
project_id=self.project_id or hook.project_id,
|
|
1016
1006
|
)
|
|
1017
1007
|
FileDetailsLink.persist(
|
|
1018
1008
|
context=context,
|
|
1019
|
-
task_instance=self,
|
|
1020
1009
|
uri=self.body["exportContext"]["uri"][5:],
|
|
1021
1010
|
project_id=self.project_id or hook.project_id,
|
|
1022
1011
|
)
|
|
@@ -1029,17 +1018,16 @@ class CloudSQLExportInstanceOperator(CloudSQLBaseOperator):
|
|
|
1029
1018
|
return hook._wait_for_operation_to_complete(
|
|
1030
1019
|
project_id=self.project_id, operation_name=operation_name
|
|
1031
1020
|
)
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
)
|
|
1021
|
+
self.defer(
|
|
1022
|
+
trigger=CloudSQLExportTrigger(
|
|
1023
|
+
operation_name=operation_name,
|
|
1024
|
+
project_id=self.project_id or hook.project_id,
|
|
1025
|
+
gcp_conn_id=self.gcp_conn_id,
|
|
1026
|
+
impersonation_chain=self.impersonation_chain,
|
|
1027
|
+
poke_interval=self.poke_interval,
|
|
1028
|
+
),
|
|
1029
|
+
method_name="execute_complete",
|
|
1030
|
+
)
|
|
1043
1031
|
|
|
1044
1032
|
def execute_complete(self, context, event=None) -> None:
|
|
1045
1033
|
"""
|
|
@@ -1154,13 +1142,10 @@ class CloudSQLImportInstanceOperator(CloudSQLBaseOperator):
|
|
|
1154
1142
|
)
|
|
1155
1143
|
CloudSQLInstanceLink.persist(
|
|
1156
1144
|
context=context,
|
|
1157
|
-
task_instance=self,
|
|
1158
|
-
cloud_sql_instance=self.instance,
|
|
1159
1145
|
project_id=self.project_id or hook.project_id,
|
|
1160
1146
|
)
|
|
1161
1147
|
FileDetailsLink.persist(
|
|
1162
1148
|
context=context,
|
|
1163
|
-
task_instance=self,
|
|
1164
1149
|
uri=self.body["importContext"]["uri"][5:],
|
|
1165
1150
|
project_id=self.project_id or hook.project_id,
|
|
1166
1151
|
)
|