apache-airflow-providers-google 15.1.0rc1__py3-none-any.whl → 19.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/3rd-party-licenses/NOTICE +2 -12
- airflow/providers/google/__init__.py +3 -3
- airflow/providers/google/ads/hooks/ads.py +39 -6
- airflow/providers/google/ads/operators/ads.py +2 -2
- airflow/providers/google/ads/transfers/ads_to_gcs.py +2 -2
- airflow/providers/google/assets/gcs.py +1 -11
- airflow/providers/google/cloud/bundles/__init__.py +16 -0
- airflow/providers/google/cloud/bundles/gcs.py +161 -0
- airflow/providers/google/cloud/hooks/alloy_db.py +1 -1
- airflow/providers/google/cloud/hooks/bigquery.py +176 -293
- airflow/providers/google/cloud/hooks/cloud_batch.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_build.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_composer.py +288 -15
- airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_run.py +18 -10
- airflow/providers/google/cloud/hooks/cloud_sql.py +102 -23
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +29 -7
- airflow/providers/google/cloud/hooks/compute.py +1 -1
- airflow/providers/google/cloud/hooks/compute_ssh.py +6 -2
- airflow/providers/google/cloud/hooks/datacatalog.py +10 -1
- airflow/providers/google/cloud/hooks/dataflow.py +72 -95
- airflow/providers/google/cloud/hooks/dataform.py +1 -1
- airflow/providers/google/cloud/hooks/datafusion.py +21 -19
- airflow/providers/google/cloud/hooks/dataplex.py +2 -2
- airflow/providers/google/cloud/hooks/dataprep.py +1 -1
- airflow/providers/google/cloud/hooks/dataproc.py +73 -72
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +1 -1
- airflow/providers/google/cloud/hooks/dlp.py +1 -1
- airflow/providers/google/cloud/hooks/functions.py +1 -1
- airflow/providers/google/cloud/hooks/gcs.py +112 -15
- airflow/providers/google/cloud/hooks/gdm.py +1 -1
- airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +3 -3
- airflow/providers/google/cloud/hooks/looker.py +6 -2
- airflow/providers/google/cloud/hooks/managed_kafka.py +1 -1
- airflow/providers/google/cloud/hooks/mlengine.py +4 -3
- airflow/providers/google/cloud/hooks/pubsub.py +3 -0
- airflow/providers/google/cloud/hooks/secret_manager.py +102 -10
- airflow/providers/google/cloud/hooks/spanner.py +74 -9
- airflow/providers/google/cloud/hooks/stackdriver.py +11 -9
- airflow/providers/google/cloud/hooks/tasks.py +1 -1
- airflow/providers/google/cloud/hooks/translate.py +2 -2
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +2 -210
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +3 -3
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +28 -2
- airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +308 -8
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
- airflow/providers/google/cloud/hooks/vision.py +3 -3
- airflow/providers/google/cloud/hooks/workflows.py +1 -1
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +77 -13
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -44
- airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -96
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +11 -95
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +9 -60
- airflow/providers/google/cloud/links/managed_kafka.py +0 -70
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +17 -187
- airflow/providers/google/cloud/links/vertex_ai.py +28 -195
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/log/gcs_task_handler.py +58 -22
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +9 -6
- airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
- airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
- airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
- airflow/providers/google/cloud/openlineage/facets.py +102 -1
- airflow/providers/google/cloud/openlineage/mixins.py +10 -8
- airflow/providers/google/cloud/openlineage/utils.py +15 -1
- airflow/providers/google/cloud/operators/alloy_db.py +71 -56
- airflow/providers/google/cloud/operators/bigquery.py +73 -636
- airflow/providers/google/cloud/operators/bigquery_dts.py +4 -6
- airflow/providers/google/cloud/operators/bigtable.py +37 -8
- airflow/providers/google/cloud/operators/cloud_base.py +21 -1
- airflow/providers/google/cloud/operators/cloud_batch.py +3 -3
- airflow/providers/google/cloud/operators/cloud_build.py +76 -33
- airflow/providers/google/cloud/operators/cloud_composer.py +129 -41
- airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
- airflow/providers/google/cloud/operators/cloud_memorystore.py +69 -43
- airflow/providers/google/cloud/operators/cloud_run.py +24 -6
- airflow/providers/google/cloud/operators/cloud_sql.py +8 -17
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +93 -12
- airflow/providers/google/cloud/operators/compute.py +9 -41
- airflow/providers/google/cloud/operators/datacatalog.py +157 -21
- airflow/providers/google/cloud/operators/dataflow.py +40 -16
- airflow/providers/google/cloud/operators/dataform.py +15 -5
- airflow/providers/google/cloud/operators/datafusion.py +42 -21
- airflow/providers/google/cloud/operators/dataplex.py +194 -110
- airflow/providers/google/cloud/operators/dataprep.py +1 -5
- airflow/providers/google/cloud/operators/dataproc.py +80 -36
- airflow/providers/google/cloud/operators/dataproc_metastore.py +97 -89
- airflow/providers/google/cloud/operators/datastore.py +23 -7
- airflow/providers/google/cloud/operators/dlp.py +6 -29
- airflow/providers/google/cloud/operators/functions.py +17 -8
- airflow/providers/google/cloud/operators/gcs.py +12 -9
- airflow/providers/google/cloud/operators/gen_ai.py +389 -0
- airflow/providers/google/cloud/operators/kubernetes_engine.py +62 -100
- airflow/providers/google/cloud/operators/looker.py +2 -2
- airflow/providers/google/cloud/operators/managed_kafka.py +108 -53
- airflow/providers/google/cloud/operators/natural_language.py +1 -1
- airflow/providers/google/cloud/operators/pubsub.py +68 -15
- airflow/providers/google/cloud/operators/spanner.py +26 -13
- airflow/providers/google/cloud/operators/speech_to_text.py +2 -3
- airflow/providers/google/cloud/operators/stackdriver.py +1 -9
- airflow/providers/google/cloud/operators/tasks.py +1 -12
- airflow/providers/google/cloud/operators/text_to_speech.py +2 -3
- airflow/providers/google/cloud/operators/translate.py +41 -17
- airflow/providers/google/cloud/operators/translate_speech.py +2 -3
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +39 -19
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +30 -10
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +55 -27
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +70 -8
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +43 -9
- airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -115
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +12 -10
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +57 -11
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +31 -8
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
- airflow/providers/google/cloud/operators/video_intelligence.py +1 -1
- airflow/providers/google/cloud/operators/vision.py +2 -2
- airflow/providers/google/cloud/operators/workflows.py +18 -15
- airflow/providers/google/cloud/secrets/secret_manager.py +3 -2
- airflow/providers/google/cloud/sensors/bigquery.py +3 -3
- airflow/providers/google/cloud/sensors/bigquery_dts.py +2 -3
- airflow/providers/google/cloud/sensors/bigtable.py +11 -4
- airflow/providers/google/cloud/sensors/cloud_composer.py +533 -30
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +2 -3
- airflow/providers/google/cloud/sensors/dataflow.py +26 -10
- airflow/providers/google/cloud/sensors/dataform.py +2 -3
- airflow/providers/google/cloud/sensors/datafusion.py +4 -5
- airflow/providers/google/cloud/sensors/dataplex.py +2 -3
- airflow/providers/google/cloud/sensors/dataprep.py +2 -2
- airflow/providers/google/cloud/sensors/dataproc.py +2 -3
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +2 -3
- airflow/providers/google/cloud/sensors/gcs.py +4 -5
- airflow/providers/google/cloud/sensors/looker.py +2 -3
- airflow/providers/google/cloud/sensors/pubsub.py +4 -5
- airflow/providers/google/cloud/sensors/tasks.py +2 -2
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -3
- airflow/providers/google/cloud/sensors/workflows.py +2 -3
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +4 -3
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +10 -5
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
- airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -4
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +21 -13
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +4 -3
- airflow/providers/google/cloud/transfers/gcs_to_local.py +6 -4
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +11 -5
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
- airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
- airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +42 -9
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +13 -7
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +14 -5
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
- airflow/providers/google/cloud/triggers/bigquery.py +76 -35
- airflow/providers/google/cloud/triggers/cloud_build.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_composer.py +303 -47
- airflow/providers/google/cloud/triggers/cloud_run.py +3 -3
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +92 -2
- airflow/providers/google/cloud/triggers/dataflow.py +122 -0
- airflow/providers/google/cloud/triggers/datafusion.py +1 -1
- airflow/providers/google/cloud/triggers/dataplex.py +14 -2
- airflow/providers/google/cloud/triggers/dataproc.py +123 -53
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +47 -28
- airflow/providers/google/cloud/triggers/mlengine.py +1 -1
- airflow/providers/google/cloud/triggers/pubsub.py +15 -19
- airflow/providers/google/cloud/triggers/vertex_ai.py +1 -1
- airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
- airflow/providers/google/cloud/utils/credentials_provider.py +2 -2
- airflow/providers/google/cloud/utils/field_sanitizer.py +1 -1
- airflow/providers/google/cloud/utils/field_validator.py +2 -3
- airflow/providers/google/common/auth_backend/google_openid.py +4 -4
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +27 -9
- airflow/providers/google/common/hooks/operation_helpers.py +1 -1
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/common/utils/get_secret.py +31 -0
- airflow/providers/google/common/utils/id_token_credentials.py +3 -4
- airflow/providers/google/firebase/hooks/firestore.py +1 -1
- airflow/providers/google/firebase/operators/firestore.py +3 -3
- airflow/providers/google/get_provider_info.py +56 -52
- airflow/providers/google/go_module_utils.py +35 -3
- airflow/providers/google/leveldb/hooks/leveldb.py +27 -2
- airflow/providers/google/leveldb/operators/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/campaign_manager.py +1 -1
- airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
- airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +5 -14
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +2 -3
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +6 -6
- airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
- airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
- airflow/providers/google/marketing_platform/sensors/display_video.py +3 -64
- airflow/providers/google/suite/hooks/calendar.py +2 -2
- airflow/providers/google/suite/hooks/sheets.py +16 -2
- airflow/providers/google/suite/operators/sheets.py +8 -3
- airflow/providers/google/suite/sensors/drive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +3 -3
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
- airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
- airflow/providers/google/version_compat.py +15 -1
- {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.3.0.dist-info}/METADATA +90 -46
- apache_airflow_providers_google-19.3.0.dist-info/RECORD +331 -0
- apache_airflow_providers_google-19.3.0.dist-info/licenses/NOTICE +5 -0
- airflow/providers/google/cloud/hooks/automl.py +0 -673
- airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
- airflow/providers/google/cloud/links/automl.py +0 -193
- airflow/providers/google/cloud/operators/automl.py +0 -1362
- airflow/providers/google/cloud/operators/life_sciences.py +0 -119
- airflow/providers/google/cloud/operators/mlengine.py +0 -112
- apache_airflow_providers_google-15.1.0rc1.dist-info/RECORD +0 -321
- {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.3.0.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.3.0.dist-info}/entry_points.txt +0 -0
- {airflow/providers/google → apache_airflow_providers_google-19.3.0.dist-info/licenses}/LICENSE +0 -0
|
@@ -21,7 +21,7 @@ from __future__ import annotations
|
|
|
21
21
|
|
|
22
22
|
import time
|
|
23
23
|
from collections.abc import Sequence
|
|
24
|
-
from typing import TYPE_CHECKING
|
|
24
|
+
from typing import TYPE_CHECKING, Any
|
|
25
25
|
|
|
26
26
|
from google.api_core.exceptions import AlreadyExists
|
|
27
27
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
@@ -30,27 +30,18 @@ from google.cloud.metastore_v1 import MetadataExport, MetadataManagementActivity
|
|
|
30
30
|
from google.cloud.metastore_v1.types import Backup, MetadataImport, Service
|
|
31
31
|
from google.cloud.metastore_v1.types.metastore import DatabaseDumpSpec, Restore
|
|
32
32
|
|
|
33
|
-
from airflow.
|
|
33
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
34
34
|
from airflow.providers.google.cloud.hooks.dataproc_metastore import DataprocMetastoreHook
|
|
35
|
+
from airflow.providers.google.cloud.links.base import BaseGoogleLink
|
|
35
36
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
36
37
|
from airflow.providers.google.common.links.storage import StorageLink
|
|
37
38
|
|
|
38
39
|
if TYPE_CHECKING:
|
|
39
40
|
from google.protobuf.field_mask_pb2 import FieldMask
|
|
40
41
|
|
|
41
|
-
from airflow.models import BaseOperator
|
|
42
42
|
from airflow.models.taskinstancekey import TaskInstanceKey
|
|
43
|
-
from airflow.
|
|
44
|
-
|
|
45
|
-
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
|
46
|
-
|
|
47
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
48
|
-
from airflow.sdk import BaseOperatorLink
|
|
49
|
-
from airflow.sdk.execution_time.xcom import XCom
|
|
50
|
-
else:
|
|
51
|
-
from airflow.models import XCom # type: ignore[no-redef]
|
|
52
|
-
from airflow.models.baseoperatorlink import BaseOperatorLink # type: ignore[no-redef]
|
|
53
|
-
|
|
43
|
+
from airflow.providers.common.compat.sdk import Context
|
|
44
|
+
from airflow.providers.google.version_compat import BaseOperator
|
|
54
45
|
|
|
55
46
|
BASE_LINK = "https://console.cloud.google.com"
|
|
56
47
|
METASTORE_BASE_LINK = BASE_LINK + "/dataproc/metastore/services/{region}/{service_id}"
|
|
@@ -61,97 +52,50 @@ METASTORE_IMPORT_LINK = METASTORE_BASE_LINK + "/imports/{resource}?project={proj
|
|
|
61
52
|
METASTORE_SERVICE_LINK = METASTORE_BASE_LINK + "/config?project={project_id}"
|
|
62
53
|
|
|
63
54
|
|
|
64
|
-
class DataprocMetastoreLink(
|
|
55
|
+
class DataprocMetastoreLink(BaseGoogleLink):
|
|
65
56
|
"""Helper class for constructing Dataproc Metastore resource link."""
|
|
66
57
|
|
|
67
58
|
name = "Dataproc Metastore"
|
|
68
59
|
key = "conf"
|
|
69
60
|
|
|
70
|
-
@staticmethod
|
|
71
|
-
def persist(
|
|
72
|
-
context: Context,
|
|
73
|
-
task_instance: (
|
|
74
|
-
DataprocMetastoreCreateServiceOperator
|
|
75
|
-
| DataprocMetastoreGetServiceOperator
|
|
76
|
-
| DataprocMetastoreRestoreServiceOperator
|
|
77
|
-
| DataprocMetastoreUpdateServiceOperator
|
|
78
|
-
| DataprocMetastoreListBackupsOperator
|
|
79
|
-
| DataprocMetastoreExportMetadataOperator
|
|
80
|
-
),
|
|
81
|
-
url: str,
|
|
82
|
-
):
|
|
83
|
-
task_instance.xcom_push(
|
|
84
|
-
context=context,
|
|
85
|
-
key=DataprocMetastoreLink.key,
|
|
86
|
-
value={
|
|
87
|
-
"region": task_instance.region,
|
|
88
|
-
"service_id": task_instance.service_id,
|
|
89
|
-
"project_id": task_instance.project_id,
|
|
90
|
-
"url": url,
|
|
91
|
-
},
|
|
92
|
-
)
|
|
93
|
-
|
|
94
61
|
def get_link(
|
|
95
62
|
self,
|
|
96
63
|
operator: BaseOperator,
|
|
97
64
|
*,
|
|
98
65
|
ti_key: TaskInstanceKey,
|
|
99
66
|
) -> str:
|
|
100
|
-
conf =
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
else ""
|
|
67
|
+
conf = self.get_config(operator, ti_key)
|
|
68
|
+
if not conf:
|
|
69
|
+
return ""
|
|
70
|
+
|
|
71
|
+
return conf["url"].format(
|
|
72
|
+
region=conf["region"],
|
|
73
|
+
service_id=conf["service_id"],
|
|
74
|
+
project_id=conf["project_id"],
|
|
109
75
|
)
|
|
110
76
|
|
|
111
77
|
|
|
112
|
-
class DataprocMetastoreDetailedLink(
|
|
78
|
+
class DataprocMetastoreDetailedLink(BaseGoogleLink):
|
|
113
79
|
"""Helper class for constructing Dataproc Metastore detailed resource link."""
|
|
114
80
|
|
|
115
81
|
name = "Dataproc Metastore resource"
|
|
116
82
|
key = "config"
|
|
117
83
|
|
|
118
|
-
@staticmethod
|
|
119
|
-
def persist(
|
|
120
|
-
context: Context,
|
|
121
|
-
task_instance: (
|
|
122
|
-
DataprocMetastoreCreateBackupOperator | DataprocMetastoreCreateMetadataImportOperator
|
|
123
|
-
),
|
|
124
|
-
url: str,
|
|
125
|
-
resource: str,
|
|
126
|
-
):
|
|
127
|
-
task_instance.xcom_push(
|
|
128
|
-
context=context,
|
|
129
|
-
key=DataprocMetastoreDetailedLink.key,
|
|
130
|
-
value={
|
|
131
|
-
"region": task_instance.region,
|
|
132
|
-
"service_id": task_instance.service_id,
|
|
133
|
-
"project_id": task_instance.project_id,
|
|
134
|
-
"url": url,
|
|
135
|
-
"resource": resource,
|
|
136
|
-
},
|
|
137
|
-
)
|
|
138
|
-
|
|
139
84
|
def get_link(
|
|
140
85
|
self,
|
|
141
86
|
operator: BaseOperator,
|
|
142
87
|
*,
|
|
143
88
|
ti_key: TaskInstanceKey,
|
|
144
89
|
) -> str:
|
|
145
|
-
conf =
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
else ""
|
|
90
|
+
conf = self.get_config(operator, ti_key)
|
|
91
|
+
if not conf:
|
|
92
|
+
return ""
|
|
93
|
+
|
|
94
|
+
return conf["url"].format(
|
|
95
|
+
region=conf["region"],
|
|
96
|
+
service_id=conf["service_id"],
|
|
97
|
+
project_id=conf["project_id"],
|
|
98
|
+
resource=conf["resource"],
|
|
155
99
|
)
|
|
156
100
|
|
|
157
101
|
|
|
@@ -231,6 +175,14 @@ class DataprocMetastoreCreateBackupOperator(GoogleCloudBaseOperator):
|
|
|
231
175
|
self.gcp_conn_id = gcp_conn_id
|
|
232
176
|
self.impersonation_chain = impersonation_chain
|
|
233
177
|
|
|
178
|
+
@property
|
|
179
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
180
|
+
return {
|
|
181
|
+
"region": self.region,
|
|
182
|
+
"service_id": self.service_id,
|
|
183
|
+
"project_id": self.project_id,
|
|
184
|
+
}
|
|
185
|
+
|
|
234
186
|
def execute(self, context: Context) -> dict:
|
|
235
187
|
hook = DataprocMetastoreHook(
|
|
236
188
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -263,7 +215,7 @@ class DataprocMetastoreCreateBackupOperator(GoogleCloudBaseOperator):
|
|
|
263
215
|
metadata=self.metadata,
|
|
264
216
|
)
|
|
265
217
|
DataprocMetastoreDetailedLink.persist(
|
|
266
|
-
context=context,
|
|
218
|
+
context=context, url=METASTORE_BACKUP_LINK, resource=self.backup_id
|
|
267
219
|
)
|
|
268
220
|
return Backup.to_dict(backup)
|
|
269
221
|
|
|
@@ -344,6 +296,14 @@ class DataprocMetastoreCreateMetadataImportOperator(GoogleCloudBaseOperator):
|
|
|
344
296
|
self.gcp_conn_id = gcp_conn_id
|
|
345
297
|
self.impersonation_chain = impersonation_chain
|
|
346
298
|
|
|
299
|
+
@property
|
|
300
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
301
|
+
return {
|
|
302
|
+
"region": self.region,
|
|
303
|
+
"service_id": self.service_id,
|
|
304
|
+
"project_id": self.project_id,
|
|
305
|
+
}
|
|
306
|
+
|
|
347
307
|
def execute(self, context: Context):
|
|
348
308
|
hook = DataprocMetastoreHook(
|
|
349
309
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -364,7 +324,7 @@ class DataprocMetastoreCreateMetadataImportOperator(GoogleCloudBaseOperator):
|
|
|
364
324
|
self.log.info("Metadata import %s created successfully", self.metadata_import_id)
|
|
365
325
|
|
|
366
326
|
DataprocMetastoreDetailedLink.persist(
|
|
367
|
-
context=context,
|
|
327
|
+
context=context, url=METASTORE_IMPORT_LINK, resource=self.metadata_import_id
|
|
368
328
|
)
|
|
369
329
|
return MetadataImport.to_dict(metadata_import)
|
|
370
330
|
|
|
@@ -437,6 +397,14 @@ class DataprocMetastoreCreateServiceOperator(GoogleCloudBaseOperator):
|
|
|
437
397
|
self.gcp_conn_id = gcp_conn_id
|
|
438
398
|
self.impersonation_chain = impersonation_chain
|
|
439
399
|
|
|
400
|
+
@property
|
|
401
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
402
|
+
return {
|
|
403
|
+
"region": self.region,
|
|
404
|
+
"service_id": self.service_id,
|
|
405
|
+
"project_id": self.project_id,
|
|
406
|
+
}
|
|
407
|
+
|
|
440
408
|
def execute(self, context: Context) -> dict:
|
|
441
409
|
hook = DataprocMetastoreHook(
|
|
442
410
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -465,7 +433,7 @@ class DataprocMetastoreCreateServiceOperator(GoogleCloudBaseOperator):
|
|
|
465
433
|
timeout=self.timeout,
|
|
466
434
|
metadata=self.metadata,
|
|
467
435
|
)
|
|
468
|
-
DataprocMetastoreLink.persist(context=context,
|
|
436
|
+
DataprocMetastoreLink.persist(context=context, url=METASTORE_SERVICE_LINK)
|
|
469
437
|
return Service.to_dict(service)
|
|
470
438
|
|
|
471
439
|
|
|
@@ -689,6 +657,14 @@ class DataprocMetastoreExportMetadataOperator(GoogleCloudBaseOperator):
|
|
|
689
657
|
self.gcp_conn_id = gcp_conn_id
|
|
690
658
|
self.impersonation_chain = impersonation_chain
|
|
691
659
|
|
|
660
|
+
@property
|
|
661
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
662
|
+
return {
|
|
663
|
+
"region": self.region,
|
|
664
|
+
"service_id": self.service_id,
|
|
665
|
+
"project_id": self.project_id,
|
|
666
|
+
}
|
|
667
|
+
|
|
692
668
|
def execute(self, context: Context):
|
|
693
669
|
hook = DataprocMetastoreHook(
|
|
694
670
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -708,9 +684,9 @@ class DataprocMetastoreExportMetadataOperator(GoogleCloudBaseOperator):
|
|
|
708
684
|
metadata_export = self._wait_for_export_metadata(hook)
|
|
709
685
|
self.log.info("Metadata from service %s exported successfully", self.service_id)
|
|
710
686
|
|
|
711
|
-
DataprocMetastoreLink.persist(context=context,
|
|
687
|
+
DataprocMetastoreLink.persist(context=context, url=METASTORE_EXPORT_LINK)
|
|
712
688
|
uri = self._get_uri_from_destination(MetadataExport.to_dict(metadata_export)["destination_gcs_uri"])
|
|
713
|
-
StorageLink.persist(context=context,
|
|
689
|
+
StorageLink.persist(context=context, uri=uri, project_id=self.project_id)
|
|
714
690
|
return MetadataExport.to_dict(metadata_export)
|
|
715
691
|
|
|
716
692
|
def _get_uri_from_destination(self, destination_uri: str):
|
|
@@ -799,6 +775,14 @@ class DataprocMetastoreGetServiceOperator(GoogleCloudBaseOperator):
|
|
|
799
775
|
self.gcp_conn_id = gcp_conn_id
|
|
800
776
|
self.impersonation_chain = impersonation_chain
|
|
801
777
|
|
|
778
|
+
@property
|
|
779
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
780
|
+
return {
|
|
781
|
+
"region": self.region,
|
|
782
|
+
"service_id": self.service_id,
|
|
783
|
+
"project_id": self.project_id,
|
|
784
|
+
}
|
|
785
|
+
|
|
802
786
|
def execute(self, context: Context) -> dict:
|
|
803
787
|
hook = DataprocMetastoreHook(
|
|
804
788
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -812,7 +796,7 @@ class DataprocMetastoreGetServiceOperator(GoogleCloudBaseOperator):
|
|
|
812
796
|
timeout=self.timeout,
|
|
813
797
|
metadata=self.metadata,
|
|
814
798
|
)
|
|
815
|
-
DataprocMetastoreLink.persist(context=context,
|
|
799
|
+
DataprocMetastoreLink.persist(context=context, url=METASTORE_SERVICE_LINK)
|
|
816
800
|
return Service.to_dict(result)
|
|
817
801
|
|
|
818
802
|
|
|
@@ -880,6 +864,14 @@ class DataprocMetastoreListBackupsOperator(GoogleCloudBaseOperator):
|
|
|
880
864
|
self.gcp_conn_id = gcp_conn_id
|
|
881
865
|
self.impersonation_chain = impersonation_chain
|
|
882
866
|
|
|
867
|
+
@property
|
|
868
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
869
|
+
return {
|
|
870
|
+
"region": self.region,
|
|
871
|
+
"service_id": self.service_id,
|
|
872
|
+
"project_id": self.project_id,
|
|
873
|
+
}
|
|
874
|
+
|
|
883
875
|
def execute(self, context: Context) -> list[dict]:
|
|
884
876
|
hook = DataprocMetastoreHook(
|
|
885
877
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -897,7 +889,7 @@ class DataprocMetastoreListBackupsOperator(GoogleCloudBaseOperator):
|
|
|
897
889
|
timeout=self.timeout,
|
|
898
890
|
metadata=self.metadata,
|
|
899
891
|
)
|
|
900
|
-
DataprocMetastoreLink.persist(context=context,
|
|
892
|
+
DataprocMetastoreLink.persist(context=context, url=METASTORE_BACKUPS_LINK)
|
|
901
893
|
return [Backup.to_dict(backup) for backup in backups]
|
|
902
894
|
|
|
903
895
|
|
|
@@ -981,6 +973,14 @@ class DataprocMetastoreRestoreServiceOperator(GoogleCloudBaseOperator):
|
|
|
981
973
|
self.gcp_conn_id = gcp_conn_id
|
|
982
974
|
self.impersonation_chain = impersonation_chain
|
|
983
975
|
|
|
976
|
+
@property
|
|
977
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
978
|
+
return {
|
|
979
|
+
"region": self.region,
|
|
980
|
+
"service_id": self.service_id,
|
|
981
|
+
"project_id": self.project_id,
|
|
982
|
+
}
|
|
983
|
+
|
|
984
984
|
def execute(self, context: Context):
|
|
985
985
|
hook = DataprocMetastoreHook(
|
|
986
986
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -1004,7 +1004,7 @@ class DataprocMetastoreRestoreServiceOperator(GoogleCloudBaseOperator):
|
|
|
1004
1004
|
)
|
|
1005
1005
|
self._wait_for_restore_service(hook)
|
|
1006
1006
|
self.log.info("Service %s restored from backup %s", self.service_id, self.backup_id)
|
|
1007
|
-
DataprocMetastoreLink.persist(context=context,
|
|
1007
|
+
DataprocMetastoreLink.persist(context=context, url=METASTORE_SERVICE_LINK)
|
|
1008
1008
|
|
|
1009
1009
|
def _wait_for_restore_service(self, hook: DataprocMetastoreHook):
|
|
1010
1010
|
"""
|
|
@@ -1107,6 +1107,14 @@ class DataprocMetastoreUpdateServiceOperator(GoogleCloudBaseOperator):
|
|
|
1107
1107
|
self.gcp_conn_id = gcp_conn_id
|
|
1108
1108
|
self.impersonation_chain = impersonation_chain
|
|
1109
1109
|
|
|
1110
|
+
@property
|
|
1111
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
1112
|
+
return {
|
|
1113
|
+
"region": self.region,
|
|
1114
|
+
"service_id": self.service_id,
|
|
1115
|
+
"project_id": self.project_id,
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1110
1118
|
def execute(self, context: Context):
|
|
1111
1119
|
hook = DataprocMetastoreHook(
|
|
1112
1120
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -1126,4 +1134,4 @@ class DataprocMetastoreUpdateServiceOperator(GoogleCloudBaseOperator):
|
|
|
1126
1134
|
)
|
|
1127
1135
|
hook.wait_for_operation(self.timeout, operation)
|
|
1128
1136
|
self.log.info("Service %s updated successfully", self.service.get("name"))
|
|
1129
|
-
DataprocMetastoreLink.persist(context=context,
|
|
1137
|
+
DataprocMetastoreLink.persist(context=context, url=METASTORE_SERVICE_LINK)
|
|
@@ -22,7 +22,7 @@ from __future__ import annotations
|
|
|
22
22
|
from collections.abc import Sequence
|
|
23
23
|
from typing import TYPE_CHECKING, Any
|
|
24
24
|
|
|
25
|
-
from airflow.
|
|
25
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
26
26
|
from airflow.providers.google.cloud.hooks.datastore import DatastoreHook
|
|
27
27
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
|
28
28
|
from airflow.providers.google.cloud.links.datastore import (
|
|
@@ -34,7 +34,7 @@ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
|
34
34
|
from airflow.providers.google.common.links.storage import StorageLink
|
|
35
35
|
|
|
36
36
|
if TYPE_CHECKING:
|
|
37
|
-
from airflow.
|
|
37
|
+
from airflow.providers.common.compat.sdk import Context
|
|
38
38
|
|
|
39
39
|
|
|
40
40
|
class CloudDatastoreExportEntitiesOperator(GoogleCloudBaseOperator):
|
|
@@ -136,7 +136,6 @@ class CloudDatastoreExportEntitiesOperator(GoogleCloudBaseOperator):
|
|
|
136
136
|
raise AirflowException(f"Operation failed: result={result}")
|
|
137
137
|
StorageLink.persist(
|
|
138
138
|
context=context,
|
|
139
|
-
task_instance=self,
|
|
140
139
|
uri=f"{self.bucket}/{result['response']['outputUrl'].split('/')[3]}",
|
|
141
140
|
project_id=self.project_id or ds_hook.project_id,
|
|
142
141
|
)
|
|
@@ -211,6 +210,12 @@ class CloudDatastoreImportEntitiesOperator(GoogleCloudBaseOperator):
|
|
|
211
210
|
self.project_id = project_id
|
|
212
211
|
self.impersonation_chain = impersonation_chain
|
|
213
212
|
|
|
213
|
+
@property
|
|
214
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
215
|
+
return {
|
|
216
|
+
"project_id": self.project_id,
|
|
217
|
+
}
|
|
218
|
+
|
|
214
219
|
def execute(self, context: Context):
|
|
215
220
|
self.log.info("Importing data from Cloud Storage bucket %s", self.bucket)
|
|
216
221
|
ds_hook = DatastoreHook(
|
|
@@ -231,8 +236,7 @@ class CloudDatastoreImportEntitiesOperator(GoogleCloudBaseOperator):
|
|
|
231
236
|
state = result["metadata"]["common"]["state"]
|
|
232
237
|
if state != "SUCCESSFUL":
|
|
233
238
|
raise AirflowException(f"Operation failed: result={result}")
|
|
234
|
-
|
|
235
|
-
CloudDatastoreImportExportLink.persist(context=context, task_instance=self)
|
|
239
|
+
CloudDatastoreImportExportLink.persist(context=context)
|
|
236
240
|
return result
|
|
237
241
|
|
|
238
242
|
|
|
@@ -282,6 +286,12 @@ class CloudDatastoreAllocateIdsOperator(GoogleCloudBaseOperator):
|
|
|
282
286
|
self.project_id = project_id
|
|
283
287
|
self.impersonation_chain = impersonation_chain
|
|
284
288
|
|
|
289
|
+
@property
|
|
290
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
291
|
+
return {
|
|
292
|
+
"project_id": self.project_id,
|
|
293
|
+
}
|
|
294
|
+
|
|
285
295
|
def execute(self, context: Context) -> list:
|
|
286
296
|
hook = DatastoreHook(
|
|
287
297
|
gcp_conn_id=self.gcp_conn_id,
|
|
@@ -291,7 +301,7 @@ class CloudDatastoreAllocateIdsOperator(GoogleCloudBaseOperator):
|
|
|
291
301
|
partial_keys=self.partial_keys,
|
|
292
302
|
project_id=self.project_id,
|
|
293
303
|
)
|
|
294
|
-
CloudDatastoreEntitiesLink.persist(context=context
|
|
304
|
+
CloudDatastoreEntitiesLink.persist(context=context)
|
|
295
305
|
return keys
|
|
296
306
|
|
|
297
307
|
|
|
@@ -398,6 +408,12 @@ class CloudDatastoreCommitOperator(GoogleCloudBaseOperator):
|
|
|
398
408
|
self.project_id = project_id
|
|
399
409
|
self.impersonation_chain = impersonation_chain
|
|
400
410
|
|
|
411
|
+
@property
|
|
412
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
413
|
+
return {
|
|
414
|
+
"project_id": self.project_id,
|
|
415
|
+
}
|
|
416
|
+
|
|
401
417
|
def execute(self, context: Context) -> dict:
|
|
402
418
|
hook = DatastoreHook(
|
|
403
419
|
gcp_conn_id=self.gcp_conn_id,
|
|
@@ -407,7 +423,7 @@ class CloudDatastoreCommitOperator(GoogleCloudBaseOperator):
|
|
|
407
423
|
body=self.body,
|
|
408
424
|
project_id=self.project_id,
|
|
409
425
|
)
|
|
410
|
-
CloudDatastoreEntitiesLink.persist(context=context
|
|
426
|
+
CloudDatastoreEntitiesLink.persist(context=context)
|
|
411
427
|
return response
|
|
412
428
|
|
|
413
429
|
|
|
@@ -66,7 +66,7 @@ if TYPE_CHECKING:
|
|
|
66
66
|
from google.api_core.retry import Retry
|
|
67
67
|
from google.protobuf.field_mask_pb2 import FieldMask
|
|
68
68
|
|
|
69
|
-
from airflow.
|
|
69
|
+
from airflow.providers.common.compat.sdk import Context
|
|
70
70
|
|
|
71
71
|
|
|
72
72
|
class CloudDLPCancelDLPJobOperator(GoogleCloudBaseOperator):
|
|
@@ -144,7 +144,6 @@ class CloudDLPCancelDLPJobOperator(GoogleCloudBaseOperator):
|
|
|
144
144
|
if project_id:
|
|
145
145
|
CloudDLPJobDetailsLink.persist(
|
|
146
146
|
context=context,
|
|
147
|
-
task_instance=self,
|
|
148
147
|
project_id=project_id,
|
|
149
148
|
job_name=self.dlp_job_id,
|
|
150
149
|
)
|
|
@@ -251,7 +250,6 @@ class CloudDLPCreateDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
|
251
250
|
if project_id and template_id:
|
|
252
251
|
CloudDLPDeidentifyTemplateDetailsLink.persist(
|
|
253
252
|
context=context,
|
|
254
|
-
task_instance=self,
|
|
255
253
|
project_id=project_id,
|
|
256
254
|
template_name=template_id,
|
|
257
255
|
)
|
|
@@ -363,7 +361,6 @@ class CloudDLPCreateDLPJobOperator(GoogleCloudBaseOperator):
|
|
|
363
361
|
if project_id:
|
|
364
362
|
CloudDLPJobDetailsLink.persist(
|
|
365
363
|
context=context,
|
|
366
|
-
task_instance=self,
|
|
367
364
|
project_id=project_id,
|
|
368
365
|
job_name=result["name"].split("/")[-1] if result["name"] else None,
|
|
369
366
|
)
|
|
@@ -473,7 +470,6 @@ class CloudDLPCreateInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
|
473
470
|
if project_id and template_id:
|
|
474
471
|
CloudDLPInspectTemplateDetailsLink.persist(
|
|
475
472
|
context=context,
|
|
476
|
-
task_instance=self,
|
|
477
473
|
project_id=project_id,
|
|
478
474
|
template_name=template_id,
|
|
479
475
|
)
|
|
@@ -578,7 +574,6 @@ class CloudDLPCreateJobTriggerOperator(GoogleCloudBaseOperator):
|
|
|
578
574
|
if project_id:
|
|
579
575
|
CloudDLPJobTriggerDetailsLink.persist(
|
|
580
576
|
context=context,
|
|
581
|
-
task_instance=self,
|
|
582
577
|
project_id=project_id,
|
|
583
578
|
trigger_name=trigger_name,
|
|
584
579
|
)
|
|
@@ -692,7 +687,6 @@ class CloudDLPCreateStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
|
692
687
|
if project_id and stored_info_type_id:
|
|
693
688
|
CloudDLPInfoTypeDetailsLink.persist(
|
|
694
689
|
context=context,
|
|
695
|
-
task_instance=self,
|
|
696
690
|
project_id=project_id,
|
|
697
691
|
info_type_name=stored_info_type_id,
|
|
698
692
|
)
|
|
@@ -880,7 +874,6 @@ class CloudDLPDeleteDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
|
880
874
|
if project_id:
|
|
881
875
|
CloudDLPDeidentifyTemplatesListLink.persist(
|
|
882
876
|
context=context,
|
|
883
|
-
task_instance=self,
|
|
884
877
|
project_id=project_id,
|
|
885
878
|
)
|
|
886
879
|
except NotFound:
|
|
@@ -966,7 +959,6 @@ class CloudDLPDeleteDLPJobOperator(GoogleCloudBaseOperator):
|
|
|
966
959
|
if project_id:
|
|
967
960
|
CloudDLPJobsListLink.persist(
|
|
968
961
|
context=context,
|
|
969
|
-
task_instance=self,
|
|
970
962
|
project_id=project_id,
|
|
971
963
|
)
|
|
972
964
|
|
|
@@ -1056,7 +1048,6 @@ class CloudDLPDeleteInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
|
1056
1048
|
if project_id:
|
|
1057
1049
|
CloudDLPInspectTemplatesListLink.persist(
|
|
1058
1050
|
context=context,
|
|
1059
|
-
task_instance=self,
|
|
1060
1051
|
project_id=project_id,
|
|
1061
1052
|
)
|
|
1062
1053
|
|
|
@@ -1140,7 +1131,6 @@ class CloudDLPDeleteJobTriggerOperator(GoogleCloudBaseOperator):
|
|
|
1140
1131
|
if project_id:
|
|
1141
1132
|
CloudDLPJobTriggersListLink.persist(
|
|
1142
1133
|
context=context,
|
|
1143
|
-
task_instance=self,
|
|
1144
1134
|
project_id=project_id,
|
|
1145
1135
|
)
|
|
1146
1136
|
|
|
@@ -1232,7 +1222,6 @@ class CloudDLPDeleteStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
|
1232
1222
|
if project_id:
|
|
1233
1223
|
CloudDLPInfoTypesListLink.persist(
|
|
1234
1224
|
context=context,
|
|
1235
|
-
task_instance=self,
|
|
1236
1225
|
project_id=project_id,
|
|
1237
1226
|
)
|
|
1238
1227
|
|
|
@@ -1318,7 +1307,9 @@ class CloudDLPGetDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
|
1318
1307
|
project_id = self.project_id or hook.project_id
|
|
1319
1308
|
if project_id:
|
|
1320
1309
|
CloudDLPDeidentifyTemplateDetailsLink.persist(
|
|
1321
|
-
context=context,
|
|
1310
|
+
context=context,
|
|
1311
|
+
project_id=project_id,
|
|
1312
|
+
template_name=self.template_id,
|
|
1322
1313
|
)
|
|
1323
1314
|
|
|
1324
1315
|
return DeidentifyTemplate.to_dict(template)
|
|
@@ -1400,7 +1391,6 @@ class CloudDLPGetDLPJobOperator(GoogleCloudBaseOperator):
|
|
|
1400
1391
|
if project_id:
|
|
1401
1392
|
CloudDLPJobDetailsLink.persist(
|
|
1402
1393
|
context=context,
|
|
1403
|
-
task_instance=self,
|
|
1404
1394
|
project_id=project_id,
|
|
1405
1395
|
job_name=self.dlp_job_id,
|
|
1406
1396
|
)
|
|
@@ -1490,7 +1480,6 @@ class CloudDLPGetInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
|
1490
1480
|
if project_id:
|
|
1491
1481
|
CloudDLPInspectTemplateDetailsLink.persist(
|
|
1492
1482
|
context=context,
|
|
1493
|
-
task_instance=self,
|
|
1494
1483
|
project_id=project_id,
|
|
1495
1484
|
template_name=self.template_id,
|
|
1496
1485
|
)
|
|
@@ -1574,7 +1563,6 @@ class CloudDLPGetDLPJobTriggerOperator(GoogleCloudBaseOperator):
|
|
|
1574
1563
|
if project_id:
|
|
1575
1564
|
CloudDLPJobTriggerDetailsLink.persist(
|
|
1576
1565
|
context=context,
|
|
1577
|
-
task_instance=self,
|
|
1578
1566
|
project_id=project_id,
|
|
1579
1567
|
trigger_name=self.job_trigger_id,
|
|
1580
1568
|
)
|
|
@@ -1664,7 +1652,6 @@ class CloudDLPGetStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
|
1664
1652
|
if project_id:
|
|
1665
1653
|
CloudDLPInfoTypeDetailsLink.persist(
|
|
1666
1654
|
context=context,
|
|
1667
|
-
task_instance=self,
|
|
1668
1655
|
project_id=project_id,
|
|
1669
1656
|
info_type_name=self.stored_info_type_id,
|
|
1670
1657
|
)
|
|
@@ -1844,11 +1831,10 @@ class CloudDLPListDeidentifyTemplatesOperator(GoogleCloudBaseOperator):
|
|
|
1844
1831
|
if project_id:
|
|
1845
1832
|
CloudDLPDeidentifyTemplatesListLink.persist(
|
|
1846
1833
|
context=context,
|
|
1847
|
-
task_instance=self,
|
|
1848
1834
|
project_id=project_id,
|
|
1849
1835
|
)
|
|
1850
1836
|
|
|
1851
|
-
return [DeidentifyTemplate.to_dict(template) for template in templates]
|
|
1837
|
+
return [DeidentifyTemplate.to_dict(template) for template in templates]
|
|
1852
1838
|
|
|
1853
1839
|
|
|
1854
1840
|
class CloudDLPListDLPJobsOperator(GoogleCloudBaseOperator):
|
|
@@ -1940,12 +1926,11 @@ class CloudDLPListDLPJobsOperator(GoogleCloudBaseOperator):
|
|
|
1940
1926
|
if project_id:
|
|
1941
1927
|
CloudDLPJobsListLink.persist(
|
|
1942
1928
|
context=context,
|
|
1943
|
-
task_instance=self,
|
|
1944
1929
|
project_id=project_id,
|
|
1945
1930
|
)
|
|
1946
1931
|
|
|
1947
1932
|
# the DlpJob.to_dict does not have the right type defined as possible to pass in constructor
|
|
1948
|
-
return [DlpJob.to_dict(job) for job in jobs]
|
|
1933
|
+
return [DlpJob.to_dict(job) for job in jobs]
|
|
1949
1934
|
|
|
1950
1935
|
|
|
1951
1936
|
class CloudDLPListInfoTypesOperator(GoogleCloudBaseOperator):
|
|
@@ -2025,7 +2010,6 @@ class CloudDLPListInfoTypesOperator(GoogleCloudBaseOperator):
|
|
|
2025
2010
|
if project_id:
|
|
2026
2011
|
CloudDLPPossibleInfoTypesListLink.persist(
|
|
2027
2012
|
context=context,
|
|
2028
|
-
task_instance=self,
|
|
2029
2013
|
project_id=project_id,
|
|
2030
2014
|
)
|
|
2031
2015
|
|
|
@@ -2119,7 +2103,6 @@ class CloudDLPListInspectTemplatesOperator(GoogleCloudBaseOperator):
|
|
|
2119
2103
|
if project_id:
|
|
2120
2104
|
CloudDLPInspectTemplatesListLink.persist(
|
|
2121
2105
|
context=context,
|
|
2122
|
-
task_instance=self,
|
|
2123
2106
|
project_id=project_id,
|
|
2124
2107
|
)
|
|
2125
2108
|
|
|
@@ -2211,7 +2194,6 @@ class CloudDLPListJobTriggersOperator(GoogleCloudBaseOperator):
|
|
|
2211
2194
|
if project_id:
|
|
2212
2195
|
CloudDLPJobTriggersListLink.persist(
|
|
2213
2196
|
context=context,
|
|
2214
|
-
task_instance=self,
|
|
2215
2197
|
project_id=project_id,
|
|
2216
2198
|
)
|
|
2217
2199
|
|
|
@@ -2305,7 +2287,6 @@ class CloudDLPListStoredInfoTypesOperator(GoogleCloudBaseOperator):
|
|
|
2305
2287
|
if project_id:
|
|
2306
2288
|
CloudDLPInfoTypesListLink.persist(
|
|
2307
2289
|
context=context,
|
|
2308
|
-
task_instance=self,
|
|
2309
2290
|
project_id=project_id,
|
|
2310
2291
|
)
|
|
2311
2292
|
|
|
@@ -2592,7 +2573,6 @@ class CloudDLPUpdateDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
|
2592
2573
|
if project_id:
|
|
2593
2574
|
CloudDLPDeidentifyTemplateDetailsLink.persist(
|
|
2594
2575
|
context=context,
|
|
2595
|
-
task_instance=self,
|
|
2596
2576
|
project_id=project_id,
|
|
2597
2577
|
template_name=self.template_id,
|
|
2598
2578
|
)
|
|
@@ -2692,7 +2672,6 @@ class CloudDLPUpdateInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
|
2692
2672
|
if project_id:
|
|
2693
2673
|
CloudDLPInspectTemplateDetailsLink.persist(
|
|
2694
2674
|
context=context,
|
|
2695
|
-
task_instance=self,
|
|
2696
2675
|
project_id=project_id,
|
|
2697
2676
|
template_name=self.template_id,
|
|
2698
2677
|
)
|
|
@@ -2786,7 +2765,6 @@ class CloudDLPUpdateJobTriggerOperator(GoogleCloudBaseOperator):
|
|
|
2786
2765
|
if project_id:
|
|
2787
2766
|
CloudDLPJobTriggerDetailsLink.persist(
|
|
2788
2767
|
context=context,
|
|
2789
|
-
task_instance=self,
|
|
2790
2768
|
project_id=project_id,
|
|
2791
2769
|
trigger_name=self.job_trigger_id,
|
|
2792
2770
|
)
|
|
@@ -2887,7 +2865,6 @@ class CloudDLPUpdateStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
|
2887
2865
|
if project_id:
|
|
2888
2866
|
CloudDLPInfoTypeDetailsLink.persist(
|
|
2889
2867
|
context=context,
|
|
2890
|
-
task_instance=self,
|
|
2891
2868
|
project_id=project_id,
|
|
2892
2869
|
info_type_name=self.stored_info_type_id,
|
|
2893
2870
|
)
|