apache-airflow-providers-google 14.0.0__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
- airflow/providers/google/3rd-party-licenses/NOTICE +5 -0
- airflow/providers/google/__init__.py +3 -3
- airflow/providers/google/_vendor/__init__.py +0 -0
- airflow/providers/google/_vendor/json_merge_patch.py +91 -0
- airflow/providers/google/ads/hooks/ads.py +52 -43
- airflow/providers/google/ads/operators/ads.py +2 -2
- airflow/providers/google/ads/transfers/ads_to_gcs.py +3 -19
- airflow/providers/google/assets/gcs.py +1 -11
- airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
- airflow/providers/google/cloud/bundles/gcs.py +161 -0
- airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
- airflow/providers/google/cloud/hooks/bigquery.py +195 -318
- airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
- airflow/providers/google/cloud/hooks/bigtable.py +3 -2
- airflow/providers/google/cloud/hooks/cloud_batch.py +8 -9
- airflow/providers/google/cloud/hooks/cloud_build.py +6 -65
- airflow/providers/google/cloud/hooks/cloud_composer.py +292 -24
- airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
- airflow/providers/google/cloud/hooks/cloud_run.py +20 -11
- airflow/providers/google/cloud/hooks/cloud_sql.py +136 -64
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +35 -15
- airflow/providers/google/cloud/hooks/compute.py +7 -6
- airflow/providers/google/cloud/hooks/compute_ssh.py +7 -4
- airflow/providers/google/cloud/hooks/datacatalog.py +12 -3
- airflow/providers/google/cloud/hooks/dataflow.py +87 -242
- airflow/providers/google/cloud/hooks/dataform.py +9 -14
- airflow/providers/google/cloud/hooks/datafusion.py +7 -9
- airflow/providers/google/cloud/hooks/dataplex.py +13 -12
- airflow/providers/google/cloud/hooks/dataprep.py +2 -2
- airflow/providers/google/cloud/hooks/dataproc.py +76 -74
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/hooks/dlp.py +5 -4
- airflow/providers/google/cloud/hooks/gcs.py +144 -33
- airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
- airflow/providers/google/cloud/hooks/kms.py +3 -2
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +22 -17
- airflow/providers/google/cloud/hooks/looker.py +6 -1
- airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
- airflow/providers/google/cloud/hooks/mlengine.py +7 -8
- airflow/providers/google/cloud/hooks/natural_language.py +3 -2
- airflow/providers/google/cloud/hooks/os_login.py +3 -2
- airflow/providers/google/cloud/hooks/pubsub.py +6 -6
- airflow/providers/google/cloud/hooks/secret_manager.py +105 -12
- airflow/providers/google/cloud/hooks/spanner.py +75 -10
- airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
- airflow/providers/google/cloud/hooks/stackdriver.py +18 -18
- airflow/providers/google/cloud/hooks/tasks.py +4 -3
- airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
- airflow/providers/google/cloud/hooks/translate.py +8 -17
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +8 -222
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +9 -15
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +33 -283
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +311 -10
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +7 -13
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +8 -12
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
- airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
- airflow/providers/google/cloud/hooks/vision.py +7 -7
- airflow/providers/google/cloud/hooks/workflows.py +4 -3
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +77 -7
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -46
- airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -90
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +11 -89
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +11 -61
- airflow/providers/google/cloud/links/managed_kafka.py +11 -51
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +17 -187
- airflow/providers/google/cloud/links/vertex_ai.py +28 -195
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/log/gcs_task_handler.py +166 -118
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +14 -9
- airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
- airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
- airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
- airflow/providers/google/cloud/openlineage/facets.py +141 -40
- airflow/providers/google/cloud/openlineage/mixins.py +14 -13
- airflow/providers/google/cloud/openlineage/utils.py +19 -3
- airflow/providers/google/cloud/operators/alloy_db.py +76 -61
- airflow/providers/google/cloud/operators/bigquery.py +104 -667
- airflow/providers/google/cloud/operators/bigquery_dts.py +12 -12
- airflow/providers/google/cloud/operators/bigtable.py +38 -7
- airflow/providers/google/cloud/operators/cloud_base.py +22 -1
- airflow/providers/google/cloud/operators/cloud_batch.py +18 -18
- airflow/providers/google/cloud/operators/cloud_build.py +80 -36
- airflow/providers/google/cloud/operators/cloud_composer.py +157 -71
- airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
- airflow/providers/google/cloud/operators/cloud_memorystore.py +74 -46
- airflow/providers/google/cloud/operators/cloud_run.py +39 -20
- airflow/providers/google/cloud/operators/cloud_sql.py +46 -61
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +92 -14
- airflow/providers/google/cloud/operators/compute.py +18 -50
- airflow/providers/google/cloud/operators/datacatalog.py +167 -29
- airflow/providers/google/cloud/operators/dataflow.py +38 -15
- airflow/providers/google/cloud/operators/dataform.py +19 -7
- airflow/providers/google/cloud/operators/datafusion.py +43 -43
- airflow/providers/google/cloud/operators/dataplex.py +212 -126
- airflow/providers/google/cloud/operators/dataprep.py +1 -5
- airflow/providers/google/cloud/operators/dataproc.py +134 -207
- airflow/providers/google/cloud/operators/dataproc_metastore.py +102 -84
- airflow/providers/google/cloud/operators/datastore.py +22 -6
- airflow/providers/google/cloud/operators/dlp.py +24 -45
- airflow/providers/google/cloud/operators/functions.py +21 -14
- airflow/providers/google/cloud/operators/gcs.py +15 -12
- airflow/providers/google/cloud/operators/gen_ai.py +389 -0
- airflow/providers/google/cloud/operators/kubernetes_engine.py +115 -106
- airflow/providers/google/cloud/operators/looker.py +1 -1
- airflow/providers/google/cloud/operators/managed_kafka.py +362 -40
- airflow/providers/google/cloud/operators/natural_language.py +5 -3
- airflow/providers/google/cloud/operators/pubsub.py +69 -21
- airflow/providers/google/cloud/operators/spanner.py +53 -45
- airflow/providers/google/cloud/operators/speech_to_text.py +5 -4
- airflow/providers/google/cloud/operators/stackdriver.py +5 -11
- airflow/providers/google/cloud/operators/tasks.py +6 -15
- airflow/providers/google/cloud/operators/text_to_speech.py +4 -3
- airflow/providers/google/cloud/operators/translate.py +46 -20
- airflow/providers/google/cloud/operators/translate_speech.py +4 -3
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +44 -34
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +34 -12
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +62 -53
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +75 -11
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +48 -12
- airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -116
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +16 -12
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +62 -14
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +35 -10
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
- airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
- airflow/providers/google/cloud/operators/vision.py +7 -5
- airflow/providers/google/cloud/operators/workflows.py +24 -19
- airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
- airflow/providers/google/cloud/sensors/bigquery.py +2 -2
- airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -4
- airflow/providers/google/cloud/sensors/bigtable.py +14 -6
- airflow/providers/google/cloud/sensors/cloud_composer.py +535 -33
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -5
- airflow/providers/google/cloud/sensors/dataflow.py +27 -10
- airflow/providers/google/cloud/sensors/dataform.py +2 -2
- airflow/providers/google/cloud/sensors/datafusion.py +4 -4
- airflow/providers/google/cloud/sensors/dataplex.py +7 -5
- airflow/providers/google/cloud/sensors/dataprep.py +2 -2
- airflow/providers/google/cloud/sensors/dataproc.py +10 -9
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/sensors/gcs.py +22 -21
- airflow/providers/google/cloud/sensors/looker.py +5 -5
- airflow/providers/google/cloud/sensors/pubsub.py +20 -20
- airflow/providers/google/cloud/sensors/tasks.py +2 -2
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -2
- airflow/providers/google/cloud/sensors/workflows.py +6 -4
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +14 -13
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
- airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +18 -22
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -5
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +45 -38
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/gcs_to_local.py +5 -3
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +10 -4
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
- airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
- airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +44 -12
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +12 -6
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +36 -14
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
- airflow/providers/google/cloud/triggers/bigquery.py +75 -34
- airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +3 -2
- airflow/providers/google/cloud/triggers/cloud_composer.py +303 -47
- airflow/providers/google/cloud/triggers/cloud_run.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +96 -5
- airflow/providers/google/cloud/triggers/dataflow.py +125 -2
- airflow/providers/google/cloud/triggers/datafusion.py +1 -1
- airflow/providers/google/cloud/triggers/dataplex.py +16 -3
- airflow/providers/google/cloud/triggers/dataproc.py +124 -53
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +46 -28
- airflow/providers/google/cloud/triggers/mlengine.py +1 -1
- airflow/providers/google/cloud/triggers/pubsub.py +17 -20
- airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
- airflow/providers/google/cloud/utils/bigquery.py +5 -7
- airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
- airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
- airflow/providers/google/cloud/utils/field_validator.py +1 -2
- airflow/providers/google/cloud/utils/validators.py +43 -0
- airflow/providers/google/common/auth_backend/google_openid.py +26 -9
- airflow/providers/google/common/consts.py +2 -1
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +40 -43
- airflow/providers/google/common/hooks/operation_helpers.py +78 -0
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/common/utils/get_secret.py +31 -0
- airflow/providers/google/common/utils/id_token_credentials.py +4 -5
- airflow/providers/google/firebase/operators/firestore.py +2 -2
- airflow/providers/google/get_provider_info.py +61 -216
- airflow/providers/google/go_module_utils.py +35 -3
- airflow/providers/google/leveldb/hooks/leveldb.py +30 -6
- airflow/providers/google/leveldb/operators/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
- airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
- airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +4 -5
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -6
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +5 -5
- airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
- airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
- airflow/providers/google/marketing_platform/sensors/display_video.py +4 -64
- airflow/providers/google/suite/hooks/calendar.py +1 -1
- airflow/providers/google/suite/hooks/drive.py +2 -2
- airflow/providers/google/suite/hooks/sheets.py +15 -1
- airflow/providers/google/suite/operators/sheets.py +8 -3
- airflow/providers/google/suite/sensors/drive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
- airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
- airflow/providers/google/version_compat.py +15 -1
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/METADATA +117 -72
- apache_airflow_providers_google-19.1.0rc1.dist-info/RECORD +331 -0
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/WHEEL +1 -1
- apache_airflow_providers_google-19.1.0rc1.dist-info/licenses/NOTICE +5 -0
- airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
- airflow/providers/google/cloud/hooks/automl.py +0 -679
- airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
- airflow/providers/google/cloud/links/automl.py +0 -193
- airflow/providers/google/cloud/operators/automl.py +0 -1360
- airflow/providers/google/cloud/operators/life_sciences.py +0 -119
- airflow/providers/google/cloud/operators/mlengine.py +0 -1515
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +0 -273
- apache_airflow_providers_google-14.0.0.dist-info/RECORD +0 -318
- /airflow/providers/google/cloud/{example_dags → bundles}/__init__.py +0 -0
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/entry_points.txt +0 -0
- {airflow/providers/google → apache_airflow_providers_google-19.1.0rc1.dist-info/licenses}/LICENSE +0 -0
|
@@ -21,14 +21,8 @@ from __future__ import annotations
|
|
|
21
21
|
|
|
22
22
|
import time
|
|
23
23
|
from collections.abc import Sequence
|
|
24
|
-
from typing import TYPE_CHECKING
|
|
24
|
+
from typing import TYPE_CHECKING, Any
|
|
25
25
|
|
|
26
|
-
from airflow.exceptions import AirflowException
|
|
27
|
-
from airflow.models import BaseOperator, BaseOperatorLink
|
|
28
|
-
from airflow.models.xcom import XCom
|
|
29
|
-
from airflow.providers.google.cloud.hooks.dataproc_metastore import DataprocMetastoreHook
|
|
30
|
-
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
31
|
-
from airflow.providers.google.common.links.storage import StorageLink
|
|
32
26
|
from google.api_core.exceptions import AlreadyExists
|
|
33
27
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
34
28
|
from google.api_core.retry import Retry, exponential_sleep_generator
|
|
@@ -36,11 +30,18 @@ from google.cloud.metastore_v1 import MetadataExport, MetadataManagementActivity
|
|
|
36
30
|
from google.cloud.metastore_v1.types import Backup, MetadataImport, Service
|
|
37
31
|
from google.cloud.metastore_v1.types.metastore import DatabaseDumpSpec, Restore
|
|
38
32
|
|
|
33
|
+
from airflow.exceptions import AirflowException
|
|
34
|
+
from airflow.providers.google.cloud.hooks.dataproc_metastore import DataprocMetastoreHook
|
|
35
|
+
from airflow.providers.google.cloud.links.base import BaseGoogleLink
|
|
36
|
+
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
37
|
+
from airflow.providers.google.common.links.storage import StorageLink
|
|
38
|
+
|
|
39
39
|
if TYPE_CHECKING:
|
|
40
|
-
from airflow.models.taskinstancekey import TaskInstanceKey
|
|
41
|
-
from airflow.utils.context import Context
|
|
42
40
|
from google.protobuf.field_mask_pb2 import FieldMask
|
|
43
41
|
|
|
42
|
+
from airflow.models.taskinstancekey import TaskInstanceKey
|
|
43
|
+
from airflow.providers.common.compat.sdk import Context
|
|
44
|
+
from airflow.providers.google.version_compat import BaseOperator
|
|
44
45
|
|
|
45
46
|
BASE_LINK = "https://console.cloud.google.com"
|
|
46
47
|
METASTORE_BASE_LINK = BASE_LINK + "/dataproc/metastore/services/{region}/{service_id}"
|
|
@@ -51,97 +52,50 @@ METASTORE_IMPORT_LINK = METASTORE_BASE_LINK + "/imports/{resource}?project={proj
|
|
|
51
52
|
METASTORE_SERVICE_LINK = METASTORE_BASE_LINK + "/config?project={project_id}"
|
|
52
53
|
|
|
53
54
|
|
|
54
|
-
class DataprocMetastoreLink(
|
|
55
|
+
class DataprocMetastoreLink(BaseGoogleLink):
|
|
55
56
|
"""Helper class for constructing Dataproc Metastore resource link."""
|
|
56
57
|
|
|
57
58
|
name = "Dataproc Metastore"
|
|
58
59
|
key = "conf"
|
|
59
60
|
|
|
60
|
-
@staticmethod
|
|
61
|
-
def persist(
|
|
62
|
-
context: Context,
|
|
63
|
-
task_instance: (
|
|
64
|
-
DataprocMetastoreCreateServiceOperator
|
|
65
|
-
| DataprocMetastoreGetServiceOperator
|
|
66
|
-
| DataprocMetastoreRestoreServiceOperator
|
|
67
|
-
| DataprocMetastoreUpdateServiceOperator
|
|
68
|
-
| DataprocMetastoreListBackupsOperator
|
|
69
|
-
| DataprocMetastoreExportMetadataOperator
|
|
70
|
-
),
|
|
71
|
-
url: str,
|
|
72
|
-
):
|
|
73
|
-
task_instance.xcom_push(
|
|
74
|
-
context=context,
|
|
75
|
-
key=DataprocMetastoreLink.key,
|
|
76
|
-
value={
|
|
77
|
-
"region": task_instance.region,
|
|
78
|
-
"service_id": task_instance.service_id,
|
|
79
|
-
"project_id": task_instance.project_id,
|
|
80
|
-
"url": url,
|
|
81
|
-
},
|
|
82
|
-
)
|
|
83
|
-
|
|
84
61
|
def get_link(
|
|
85
62
|
self,
|
|
86
63
|
operator: BaseOperator,
|
|
87
64
|
*,
|
|
88
65
|
ti_key: TaskInstanceKey,
|
|
89
66
|
) -> str:
|
|
90
|
-
conf =
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
else ""
|
|
67
|
+
conf = self.get_config(operator, ti_key)
|
|
68
|
+
if not conf:
|
|
69
|
+
return ""
|
|
70
|
+
|
|
71
|
+
return conf["url"].format(
|
|
72
|
+
region=conf["region"],
|
|
73
|
+
service_id=conf["service_id"],
|
|
74
|
+
project_id=conf["project_id"],
|
|
99
75
|
)
|
|
100
76
|
|
|
101
77
|
|
|
102
|
-
class DataprocMetastoreDetailedLink(
|
|
78
|
+
class DataprocMetastoreDetailedLink(BaseGoogleLink):
|
|
103
79
|
"""Helper class for constructing Dataproc Metastore detailed resource link."""
|
|
104
80
|
|
|
105
81
|
name = "Dataproc Metastore resource"
|
|
106
82
|
key = "config"
|
|
107
83
|
|
|
108
|
-
@staticmethod
|
|
109
|
-
def persist(
|
|
110
|
-
context: Context,
|
|
111
|
-
task_instance: (
|
|
112
|
-
DataprocMetastoreCreateBackupOperator | DataprocMetastoreCreateMetadataImportOperator
|
|
113
|
-
),
|
|
114
|
-
url: str,
|
|
115
|
-
resource: str,
|
|
116
|
-
):
|
|
117
|
-
task_instance.xcom_push(
|
|
118
|
-
context=context,
|
|
119
|
-
key=DataprocMetastoreDetailedLink.key,
|
|
120
|
-
value={
|
|
121
|
-
"region": task_instance.region,
|
|
122
|
-
"service_id": task_instance.service_id,
|
|
123
|
-
"project_id": task_instance.project_id,
|
|
124
|
-
"url": url,
|
|
125
|
-
"resource": resource,
|
|
126
|
-
},
|
|
127
|
-
)
|
|
128
|
-
|
|
129
84
|
def get_link(
|
|
130
85
|
self,
|
|
131
86
|
operator: BaseOperator,
|
|
132
87
|
*,
|
|
133
88
|
ti_key: TaskInstanceKey,
|
|
134
89
|
) -> str:
|
|
135
|
-
conf =
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
else ""
|
|
90
|
+
conf = self.get_config(operator, ti_key)
|
|
91
|
+
if not conf:
|
|
92
|
+
return ""
|
|
93
|
+
|
|
94
|
+
return conf["url"].format(
|
|
95
|
+
region=conf["region"],
|
|
96
|
+
service_id=conf["service_id"],
|
|
97
|
+
project_id=conf["project_id"],
|
|
98
|
+
resource=conf["resource"],
|
|
145
99
|
)
|
|
146
100
|
|
|
147
101
|
|
|
@@ -221,6 +175,14 @@ class DataprocMetastoreCreateBackupOperator(GoogleCloudBaseOperator):
|
|
|
221
175
|
self.gcp_conn_id = gcp_conn_id
|
|
222
176
|
self.impersonation_chain = impersonation_chain
|
|
223
177
|
|
|
178
|
+
@property
|
|
179
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
180
|
+
return {
|
|
181
|
+
"region": self.region,
|
|
182
|
+
"service_id": self.service_id,
|
|
183
|
+
"project_id": self.project_id,
|
|
184
|
+
}
|
|
185
|
+
|
|
224
186
|
def execute(self, context: Context) -> dict:
|
|
225
187
|
hook = DataprocMetastoreHook(
|
|
226
188
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -253,7 +215,7 @@ class DataprocMetastoreCreateBackupOperator(GoogleCloudBaseOperator):
|
|
|
253
215
|
metadata=self.metadata,
|
|
254
216
|
)
|
|
255
217
|
DataprocMetastoreDetailedLink.persist(
|
|
256
|
-
context=context,
|
|
218
|
+
context=context, url=METASTORE_BACKUP_LINK, resource=self.backup_id
|
|
257
219
|
)
|
|
258
220
|
return Backup.to_dict(backup)
|
|
259
221
|
|
|
@@ -334,6 +296,14 @@ class DataprocMetastoreCreateMetadataImportOperator(GoogleCloudBaseOperator):
|
|
|
334
296
|
self.gcp_conn_id = gcp_conn_id
|
|
335
297
|
self.impersonation_chain = impersonation_chain
|
|
336
298
|
|
|
299
|
+
@property
|
|
300
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
301
|
+
return {
|
|
302
|
+
"region": self.region,
|
|
303
|
+
"service_id": self.service_id,
|
|
304
|
+
"project_id": self.project_id,
|
|
305
|
+
}
|
|
306
|
+
|
|
337
307
|
def execute(self, context: Context):
|
|
338
308
|
hook = DataprocMetastoreHook(
|
|
339
309
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -354,7 +324,7 @@ class DataprocMetastoreCreateMetadataImportOperator(GoogleCloudBaseOperator):
|
|
|
354
324
|
self.log.info("Metadata import %s created successfully", self.metadata_import_id)
|
|
355
325
|
|
|
356
326
|
DataprocMetastoreDetailedLink.persist(
|
|
357
|
-
context=context,
|
|
327
|
+
context=context, url=METASTORE_IMPORT_LINK, resource=self.metadata_import_id
|
|
358
328
|
)
|
|
359
329
|
return MetadataImport.to_dict(metadata_import)
|
|
360
330
|
|
|
@@ -427,6 +397,14 @@ class DataprocMetastoreCreateServiceOperator(GoogleCloudBaseOperator):
|
|
|
427
397
|
self.gcp_conn_id = gcp_conn_id
|
|
428
398
|
self.impersonation_chain = impersonation_chain
|
|
429
399
|
|
|
400
|
+
@property
|
|
401
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
402
|
+
return {
|
|
403
|
+
"region": self.region,
|
|
404
|
+
"service_id": self.service_id,
|
|
405
|
+
"project_id": self.project_id,
|
|
406
|
+
}
|
|
407
|
+
|
|
430
408
|
def execute(self, context: Context) -> dict:
|
|
431
409
|
hook = DataprocMetastoreHook(
|
|
432
410
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -455,7 +433,7 @@ class DataprocMetastoreCreateServiceOperator(GoogleCloudBaseOperator):
|
|
|
455
433
|
timeout=self.timeout,
|
|
456
434
|
metadata=self.metadata,
|
|
457
435
|
)
|
|
458
|
-
DataprocMetastoreLink.persist(context=context,
|
|
436
|
+
DataprocMetastoreLink.persist(context=context, url=METASTORE_SERVICE_LINK)
|
|
459
437
|
return Service.to_dict(service)
|
|
460
438
|
|
|
461
439
|
|
|
@@ -679,6 +657,14 @@ class DataprocMetastoreExportMetadataOperator(GoogleCloudBaseOperator):
|
|
|
679
657
|
self.gcp_conn_id = gcp_conn_id
|
|
680
658
|
self.impersonation_chain = impersonation_chain
|
|
681
659
|
|
|
660
|
+
@property
|
|
661
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
662
|
+
return {
|
|
663
|
+
"region": self.region,
|
|
664
|
+
"service_id": self.service_id,
|
|
665
|
+
"project_id": self.project_id,
|
|
666
|
+
}
|
|
667
|
+
|
|
682
668
|
def execute(self, context: Context):
|
|
683
669
|
hook = DataprocMetastoreHook(
|
|
684
670
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -698,9 +684,9 @@ class DataprocMetastoreExportMetadataOperator(GoogleCloudBaseOperator):
|
|
|
698
684
|
metadata_export = self._wait_for_export_metadata(hook)
|
|
699
685
|
self.log.info("Metadata from service %s exported successfully", self.service_id)
|
|
700
686
|
|
|
701
|
-
DataprocMetastoreLink.persist(context=context,
|
|
687
|
+
DataprocMetastoreLink.persist(context=context, url=METASTORE_EXPORT_LINK)
|
|
702
688
|
uri = self._get_uri_from_destination(MetadataExport.to_dict(metadata_export)["destination_gcs_uri"])
|
|
703
|
-
StorageLink.persist(context=context,
|
|
689
|
+
StorageLink.persist(context=context, uri=uri, project_id=self.project_id)
|
|
704
690
|
return MetadataExport.to_dict(metadata_export)
|
|
705
691
|
|
|
706
692
|
def _get_uri_from_destination(self, destination_uri: str):
|
|
@@ -789,6 +775,14 @@ class DataprocMetastoreGetServiceOperator(GoogleCloudBaseOperator):
|
|
|
789
775
|
self.gcp_conn_id = gcp_conn_id
|
|
790
776
|
self.impersonation_chain = impersonation_chain
|
|
791
777
|
|
|
778
|
+
@property
|
|
779
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
780
|
+
return {
|
|
781
|
+
"region": self.region,
|
|
782
|
+
"service_id": self.service_id,
|
|
783
|
+
"project_id": self.project_id,
|
|
784
|
+
}
|
|
785
|
+
|
|
792
786
|
def execute(self, context: Context) -> dict:
|
|
793
787
|
hook = DataprocMetastoreHook(
|
|
794
788
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -802,7 +796,7 @@ class DataprocMetastoreGetServiceOperator(GoogleCloudBaseOperator):
|
|
|
802
796
|
timeout=self.timeout,
|
|
803
797
|
metadata=self.metadata,
|
|
804
798
|
)
|
|
805
|
-
DataprocMetastoreLink.persist(context=context,
|
|
799
|
+
DataprocMetastoreLink.persist(context=context, url=METASTORE_SERVICE_LINK)
|
|
806
800
|
return Service.to_dict(result)
|
|
807
801
|
|
|
808
802
|
|
|
@@ -870,6 +864,14 @@ class DataprocMetastoreListBackupsOperator(GoogleCloudBaseOperator):
|
|
|
870
864
|
self.gcp_conn_id = gcp_conn_id
|
|
871
865
|
self.impersonation_chain = impersonation_chain
|
|
872
866
|
|
|
867
|
+
@property
|
|
868
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
869
|
+
return {
|
|
870
|
+
"region": self.region,
|
|
871
|
+
"service_id": self.service_id,
|
|
872
|
+
"project_id": self.project_id,
|
|
873
|
+
}
|
|
874
|
+
|
|
873
875
|
def execute(self, context: Context) -> list[dict]:
|
|
874
876
|
hook = DataprocMetastoreHook(
|
|
875
877
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -887,7 +889,7 @@ class DataprocMetastoreListBackupsOperator(GoogleCloudBaseOperator):
|
|
|
887
889
|
timeout=self.timeout,
|
|
888
890
|
metadata=self.metadata,
|
|
889
891
|
)
|
|
890
|
-
DataprocMetastoreLink.persist(context=context,
|
|
892
|
+
DataprocMetastoreLink.persist(context=context, url=METASTORE_BACKUPS_LINK)
|
|
891
893
|
return [Backup.to_dict(backup) for backup in backups]
|
|
892
894
|
|
|
893
895
|
|
|
@@ -971,6 +973,14 @@ class DataprocMetastoreRestoreServiceOperator(GoogleCloudBaseOperator):
|
|
|
971
973
|
self.gcp_conn_id = gcp_conn_id
|
|
972
974
|
self.impersonation_chain = impersonation_chain
|
|
973
975
|
|
|
976
|
+
@property
|
|
977
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
978
|
+
return {
|
|
979
|
+
"region": self.region,
|
|
980
|
+
"service_id": self.service_id,
|
|
981
|
+
"project_id": self.project_id,
|
|
982
|
+
}
|
|
983
|
+
|
|
974
984
|
def execute(self, context: Context):
|
|
975
985
|
hook = DataprocMetastoreHook(
|
|
976
986
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -994,7 +1004,7 @@ class DataprocMetastoreRestoreServiceOperator(GoogleCloudBaseOperator):
|
|
|
994
1004
|
)
|
|
995
1005
|
self._wait_for_restore_service(hook)
|
|
996
1006
|
self.log.info("Service %s restored from backup %s", self.service_id, self.backup_id)
|
|
997
|
-
DataprocMetastoreLink.persist(context=context,
|
|
1007
|
+
DataprocMetastoreLink.persist(context=context, url=METASTORE_SERVICE_LINK)
|
|
998
1008
|
|
|
999
1009
|
def _wait_for_restore_service(self, hook: DataprocMetastoreHook):
|
|
1000
1010
|
"""
|
|
@@ -1097,6 +1107,14 @@ class DataprocMetastoreUpdateServiceOperator(GoogleCloudBaseOperator):
|
|
|
1097
1107
|
self.gcp_conn_id = gcp_conn_id
|
|
1098
1108
|
self.impersonation_chain = impersonation_chain
|
|
1099
1109
|
|
|
1110
|
+
@property
|
|
1111
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
1112
|
+
return {
|
|
1113
|
+
"region": self.region,
|
|
1114
|
+
"service_id": self.service_id,
|
|
1115
|
+
"project_id": self.project_id,
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1100
1118
|
def execute(self, context: Context):
|
|
1101
1119
|
hook = DataprocMetastoreHook(
|
|
1102
1120
|
gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
|
|
@@ -1116,4 +1134,4 @@ class DataprocMetastoreUpdateServiceOperator(GoogleCloudBaseOperator):
|
|
|
1116
1134
|
)
|
|
1117
1135
|
hook.wait_for_operation(self.timeout, operation)
|
|
1118
1136
|
self.log.info("Service %s updated successfully", self.service.get("name"))
|
|
1119
|
-
DataprocMetastoreLink.persist(context=context,
|
|
1137
|
+
DataprocMetastoreLink.persist(context=context, url=METASTORE_SERVICE_LINK)
|
|
@@ -34,7 +34,7 @@ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
|
34
34
|
from airflow.providers.google.common.links.storage import StorageLink
|
|
35
35
|
|
|
36
36
|
if TYPE_CHECKING:
|
|
37
|
-
from airflow.
|
|
37
|
+
from airflow.providers.common.compat.sdk import Context
|
|
38
38
|
|
|
39
39
|
|
|
40
40
|
class CloudDatastoreExportEntitiesOperator(GoogleCloudBaseOperator):
|
|
@@ -136,7 +136,6 @@ class CloudDatastoreExportEntitiesOperator(GoogleCloudBaseOperator):
|
|
|
136
136
|
raise AirflowException(f"Operation failed: result={result}")
|
|
137
137
|
StorageLink.persist(
|
|
138
138
|
context=context,
|
|
139
|
-
task_instance=self,
|
|
140
139
|
uri=f"{self.bucket}/{result['response']['outputUrl'].split('/')[3]}",
|
|
141
140
|
project_id=self.project_id or ds_hook.project_id,
|
|
142
141
|
)
|
|
@@ -211,6 +210,12 @@ class CloudDatastoreImportEntitiesOperator(GoogleCloudBaseOperator):
|
|
|
211
210
|
self.project_id = project_id
|
|
212
211
|
self.impersonation_chain = impersonation_chain
|
|
213
212
|
|
|
213
|
+
@property
|
|
214
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
215
|
+
return {
|
|
216
|
+
"project_id": self.project_id,
|
|
217
|
+
}
|
|
218
|
+
|
|
214
219
|
def execute(self, context: Context):
|
|
215
220
|
self.log.info("Importing data from Cloud Storage bucket %s", self.bucket)
|
|
216
221
|
ds_hook = DatastoreHook(
|
|
@@ -231,8 +236,7 @@ class CloudDatastoreImportEntitiesOperator(GoogleCloudBaseOperator):
|
|
|
231
236
|
state = result["metadata"]["common"]["state"]
|
|
232
237
|
if state != "SUCCESSFUL":
|
|
233
238
|
raise AirflowException(f"Operation failed: result={result}")
|
|
234
|
-
|
|
235
|
-
CloudDatastoreImportExportLink.persist(context=context, task_instance=self)
|
|
239
|
+
CloudDatastoreImportExportLink.persist(context=context)
|
|
236
240
|
return result
|
|
237
241
|
|
|
238
242
|
|
|
@@ -282,6 +286,12 @@ class CloudDatastoreAllocateIdsOperator(GoogleCloudBaseOperator):
|
|
|
282
286
|
self.project_id = project_id
|
|
283
287
|
self.impersonation_chain = impersonation_chain
|
|
284
288
|
|
|
289
|
+
@property
|
|
290
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
291
|
+
return {
|
|
292
|
+
"project_id": self.project_id,
|
|
293
|
+
}
|
|
294
|
+
|
|
285
295
|
def execute(self, context: Context) -> list:
|
|
286
296
|
hook = DatastoreHook(
|
|
287
297
|
gcp_conn_id=self.gcp_conn_id,
|
|
@@ -291,7 +301,7 @@ class CloudDatastoreAllocateIdsOperator(GoogleCloudBaseOperator):
|
|
|
291
301
|
partial_keys=self.partial_keys,
|
|
292
302
|
project_id=self.project_id,
|
|
293
303
|
)
|
|
294
|
-
CloudDatastoreEntitiesLink.persist(context=context
|
|
304
|
+
CloudDatastoreEntitiesLink.persist(context=context)
|
|
295
305
|
return keys
|
|
296
306
|
|
|
297
307
|
|
|
@@ -398,6 +408,12 @@ class CloudDatastoreCommitOperator(GoogleCloudBaseOperator):
|
|
|
398
408
|
self.project_id = project_id
|
|
399
409
|
self.impersonation_chain = impersonation_chain
|
|
400
410
|
|
|
411
|
+
@property
|
|
412
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
413
|
+
return {
|
|
414
|
+
"project_id": self.project_id,
|
|
415
|
+
}
|
|
416
|
+
|
|
401
417
|
def execute(self, context: Context) -> dict:
|
|
402
418
|
hook = DatastoreHook(
|
|
403
419
|
gcp_conn_id=self.gcp_conn_id,
|
|
@@ -407,7 +423,7 @@ class CloudDatastoreCommitOperator(GoogleCloudBaseOperator):
|
|
|
407
423
|
body=self.body,
|
|
408
424
|
project_id=self.project_id,
|
|
409
425
|
)
|
|
410
|
-
CloudDatastoreEntitiesLink.persist(context=context
|
|
426
|
+
CloudDatastoreEntitiesLink.persist(context=context)
|
|
411
427
|
return response
|
|
412
428
|
|
|
413
429
|
|
|
@@ -22,22 +22,6 @@ from __future__ import annotations
|
|
|
22
22
|
from collections.abc import Sequence
|
|
23
23
|
from typing import TYPE_CHECKING
|
|
24
24
|
|
|
25
|
-
from airflow.providers.google.cloud.hooks.dlp import CloudDLPHook
|
|
26
|
-
from airflow.providers.google.cloud.links.data_loss_prevention import (
|
|
27
|
-
CloudDLPDeidentifyTemplateDetailsLink,
|
|
28
|
-
CloudDLPDeidentifyTemplatesListLink,
|
|
29
|
-
CloudDLPInfoTypeDetailsLink,
|
|
30
|
-
CloudDLPInfoTypesListLink,
|
|
31
|
-
CloudDLPInspectTemplateDetailsLink,
|
|
32
|
-
CloudDLPInspectTemplatesListLink,
|
|
33
|
-
CloudDLPJobDetailsLink,
|
|
34
|
-
CloudDLPJobsListLink,
|
|
35
|
-
CloudDLPJobTriggerDetailsLink,
|
|
36
|
-
CloudDLPJobTriggersListLink,
|
|
37
|
-
CloudDLPPossibleInfoTypesListLink,
|
|
38
|
-
)
|
|
39
|
-
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
40
|
-
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
41
25
|
from google.api_core.exceptions import AlreadyExists, InvalidArgument, NotFound
|
|
42
26
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
43
27
|
from google.cloud.dlp_v2.types import (
|
|
@@ -61,11 +45,29 @@ from google.cloud.dlp_v2.types import (
|
|
|
61
45
|
StoredInfoTypeConfig,
|
|
62
46
|
)
|
|
63
47
|
|
|
48
|
+
from airflow.providers.google.cloud.hooks.dlp import CloudDLPHook
|
|
49
|
+
from airflow.providers.google.cloud.links.data_loss_prevention import (
|
|
50
|
+
CloudDLPDeidentifyTemplateDetailsLink,
|
|
51
|
+
CloudDLPDeidentifyTemplatesListLink,
|
|
52
|
+
CloudDLPInfoTypeDetailsLink,
|
|
53
|
+
CloudDLPInfoTypesListLink,
|
|
54
|
+
CloudDLPInspectTemplateDetailsLink,
|
|
55
|
+
CloudDLPInspectTemplatesListLink,
|
|
56
|
+
CloudDLPJobDetailsLink,
|
|
57
|
+
CloudDLPJobsListLink,
|
|
58
|
+
CloudDLPJobTriggerDetailsLink,
|
|
59
|
+
CloudDLPJobTriggersListLink,
|
|
60
|
+
CloudDLPPossibleInfoTypesListLink,
|
|
61
|
+
)
|
|
62
|
+
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
63
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
64
|
+
|
|
64
65
|
if TYPE_CHECKING:
|
|
65
|
-
from airflow.utils.context import Context
|
|
66
66
|
from google.api_core.retry import Retry
|
|
67
67
|
from google.protobuf.field_mask_pb2 import FieldMask
|
|
68
68
|
|
|
69
|
+
from airflow.providers.common.compat.sdk import Context
|
|
70
|
+
|
|
69
71
|
|
|
70
72
|
class CloudDLPCancelDLPJobOperator(GoogleCloudBaseOperator):
|
|
71
73
|
"""
|
|
@@ -142,7 +144,6 @@ class CloudDLPCancelDLPJobOperator(GoogleCloudBaseOperator):
|
|
|
142
144
|
if project_id:
|
|
143
145
|
CloudDLPJobDetailsLink.persist(
|
|
144
146
|
context=context,
|
|
145
|
-
task_instance=self,
|
|
146
147
|
project_id=project_id,
|
|
147
148
|
job_name=self.dlp_job_id,
|
|
148
149
|
)
|
|
@@ -249,7 +250,6 @@ class CloudDLPCreateDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
|
249
250
|
if project_id and template_id:
|
|
250
251
|
CloudDLPDeidentifyTemplateDetailsLink.persist(
|
|
251
252
|
context=context,
|
|
252
|
-
task_instance=self,
|
|
253
253
|
project_id=project_id,
|
|
254
254
|
template_name=template_id,
|
|
255
255
|
)
|
|
@@ -361,7 +361,6 @@ class CloudDLPCreateDLPJobOperator(GoogleCloudBaseOperator):
|
|
|
361
361
|
if project_id:
|
|
362
362
|
CloudDLPJobDetailsLink.persist(
|
|
363
363
|
context=context,
|
|
364
|
-
task_instance=self,
|
|
365
364
|
project_id=project_id,
|
|
366
365
|
job_name=result["name"].split("/")[-1] if result["name"] else None,
|
|
367
366
|
)
|
|
@@ -471,7 +470,6 @@ class CloudDLPCreateInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
|
471
470
|
if project_id and template_id:
|
|
472
471
|
CloudDLPInspectTemplateDetailsLink.persist(
|
|
473
472
|
context=context,
|
|
474
|
-
task_instance=self,
|
|
475
473
|
project_id=project_id,
|
|
476
474
|
template_name=template_id,
|
|
477
475
|
)
|
|
@@ -576,7 +574,6 @@ class CloudDLPCreateJobTriggerOperator(GoogleCloudBaseOperator):
|
|
|
576
574
|
if project_id:
|
|
577
575
|
CloudDLPJobTriggerDetailsLink.persist(
|
|
578
576
|
context=context,
|
|
579
|
-
task_instance=self,
|
|
580
577
|
project_id=project_id,
|
|
581
578
|
trigger_name=trigger_name,
|
|
582
579
|
)
|
|
@@ -690,7 +687,6 @@ class CloudDLPCreateStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
|
690
687
|
if project_id and stored_info_type_id:
|
|
691
688
|
CloudDLPInfoTypeDetailsLink.persist(
|
|
692
689
|
context=context,
|
|
693
|
-
task_instance=self,
|
|
694
690
|
project_id=project_id,
|
|
695
691
|
info_type_name=stored_info_type_id,
|
|
696
692
|
)
|
|
@@ -878,7 +874,6 @@ class CloudDLPDeleteDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
|
878
874
|
if project_id:
|
|
879
875
|
CloudDLPDeidentifyTemplatesListLink.persist(
|
|
880
876
|
context=context,
|
|
881
|
-
task_instance=self,
|
|
882
877
|
project_id=project_id,
|
|
883
878
|
)
|
|
884
879
|
except NotFound:
|
|
@@ -964,7 +959,6 @@ class CloudDLPDeleteDLPJobOperator(GoogleCloudBaseOperator):
|
|
|
964
959
|
if project_id:
|
|
965
960
|
CloudDLPJobsListLink.persist(
|
|
966
961
|
context=context,
|
|
967
|
-
task_instance=self,
|
|
968
962
|
project_id=project_id,
|
|
969
963
|
)
|
|
970
964
|
|
|
@@ -1054,7 +1048,6 @@ class CloudDLPDeleteInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
|
1054
1048
|
if project_id:
|
|
1055
1049
|
CloudDLPInspectTemplatesListLink.persist(
|
|
1056
1050
|
context=context,
|
|
1057
|
-
task_instance=self,
|
|
1058
1051
|
project_id=project_id,
|
|
1059
1052
|
)
|
|
1060
1053
|
|
|
@@ -1138,7 +1131,6 @@ class CloudDLPDeleteJobTriggerOperator(GoogleCloudBaseOperator):
|
|
|
1138
1131
|
if project_id:
|
|
1139
1132
|
CloudDLPJobTriggersListLink.persist(
|
|
1140
1133
|
context=context,
|
|
1141
|
-
task_instance=self,
|
|
1142
1134
|
project_id=project_id,
|
|
1143
1135
|
)
|
|
1144
1136
|
|
|
@@ -1230,7 +1222,6 @@ class CloudDLPDeleteStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
|
1230
1222
|
if project_id:
|
|
1231
1223
|
CloudDLPInfoTypesListLink.persist(
|
|
1232
1224
|
context=context,
|
|
1233
|
-
task_instance=self,
|
|
1234
1225
|
project_id=project_id,
|
|
1235
1226
|
)
|
|
1236
1227
|
|
|
@@ -1316,7 +1307,9 @@ class CloudDLPGetDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
|
1316
1307
|
project_id = self.project_id or hook.project_id
|
|
1317
1308
|
if project_id:
|
|
1318
1309
|
CloudDLPDeidentifyTemplateDetailsLink.persist(
|
|
1319
|
-
context=context,
|
|
1310
|
+
context=context,
|
|
1311
|
+
project_id=project_id,
|
|
1312
|
+
template_name=self.template_id,
|
|
1320
1313
|
)
|
|
1321
1314
|
|
|
1322
1315
|
return DeidentifyTemplate.to_dict(template)
|
|
@@ -1398,7 +1391,6 @@ class CloudDLPGetDLPJobOperator(GoogleCloudBaseOperator):
|
|
|
1398
1391
|
if project_id:
|
|
1399
1392
|
CloudDLPJobDetailsLink.persist(
|
|
1400
1393
|
context=context,
|
|
1401
|
-
task_instance=self,
|
|
1402
1394
|
project_id=project_id,
|
|
1403
1395
|
job_name=self.dlp_job_id,
|
|
1404
1396
|
)
|
|
@@ -1488,7 +1480,6 @@ class CloudDLPGetInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
|
1488
1480
|
if project_id:
|
|
1489
1481
|
CloudDLPInspectTemplateDetailsLink.persist(
|
|
1490
1482
|
context=context,
|
|
1491
|
-
task_instance=self,
|
|
1492
1483
|
project_id=project_id,
|
|
1493
1484
|
template_name=self.template_id,
|
|
1494
1485
|
)
|
|
@@ -1572,7 +1563,6 @@ class CloudDLPGetDLPJobTriggerOperator(GoogleCloudBaseOperator):
|
|
|
1572
1563
|
if project_id:
|
|
1573
1564
|
CloudDLPJobTriggerDetailsLink.persist(
|
|
1574
1565
|
context=context,
|
|
1575
|
-
task_instance=self,
|
|
1576
1566
|
project_id=project_id,
|
|
1577
1567
|
trigger_name=self.job_trigger_id,
|
|
1578
1568
|
)
|
|
@@ -1662,7 +1652,6 @@ class CloudDLPGetStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
|
1662
1652
|
if project_id:
|
|
1663
1653
|
CloudDLPInfoTypeDetailsLink.persist(
|
|
1664
1654
|
context=context,
|
|
1665
|
-
task_instance=self,
|
|
1666
1655
|
project_id=project_id,
|
|
1667
1656
|
info_type_name=self.stored_info_type_id,
|
|
1668
1657
|
)
|
|
@@ -1842,11 +1831,10 @@ class CloudDLPListDeidentifyTemplatesOperator(GoogleCloudBaseOperator):
|
|
|
1842
1831
|
if project_id:
|
|
1843
1832
|
CloudDLPDeidentifyTemplatesListLink.persist(
|
|
1844
1833
|
context=context,
|
|
1845
|
-
task_instance=self,
|
|
1846
1834
|
project_id=project_id,
|
|
1847
1835
|
)
|
|
1848
1836
|
|
|
1849
|
-
return [DeidentifyTemplate.to_dict(template) for template in templates]
|
|
1837
|
+
return [DeidentifyTemplate.to_dict(template) for template in templates]
|
|
1850
1838
|
|
|
1851
1839
|
|
|
1852
1840
|
class CloudDLPListDLPJobsOperator(GoogleCloudBaseOperator):
|
|
@@ -1938,12 +1926,11 @@ class CloudDLPListDLPJobsOperator(GoogleCloudBaseOperator):
|
|
|
1938
1926
|
if project_id:
|
|
1939
1927
|
CloudDLPJobsListLink.persist(
|
|
1940
1928
|
context=context,
|
|
1941
|
-
task_instance=self,
|
|
1942
1929
|
project_id=project_id,
|
|
1943
1930
|
)
|
|
1944
1931
|
|
|
1945
1932
|
# the DlpJob.to_dict does not have the right type defined as possible to pass in constructor
|
|
1946
|
-
return [DlpJob.to_dict(job) for job in jobs]
|
|
1933
|
+
return [DlpJob.to_dict(job) for job in jobs]
|
|
1947
1934
|
|
|
1948
1935
|
|
|
1949
1936
|
class CloudDLPListInfoTypesOperator(GoogleCloudBaseOperator):
|
|
@@ -2023,7 +2010,6 @@ class CloudDLPListInfoTypesOperator(GoogleCloudBaseOperator):
|
|
|
2023
2010
|
if project_id:
|
|
2024
2011
|
CloudDLPPossibleInfoTypesListLink.persist(
|
|
2025
2012
|
context=context,
|
|
2026
|
-
task_instance=self,
|
|
2027
2013
|
project_id=project_id,
|
|
2028
2014
|
)
|
|
2029
2015
|
|
|
@@ -2117,7 +2103,6 @@ class CloudDLPListInspectTemplatesOperator(GoogleCloudBaseOperator):
|
|
|
2117
2103
|
if project_id:
|
|
2118
2104
|
CloudDLPInspectTemplatesListLink.persist(
|
|
2119
2105
|
context=context,
|
|
2120
|
-
task_instance=self,
|
|
2121
2106
|
project_id=project_id,
|
|
2122
2107
|
)
|
|
2123
2108
|
|
|
@@ -2209,7 +2194,6 @@ class CloudDLPListJobTriggersOperator(GoogleCloudBaseOperator):
|
|
|
2209
2194
|
if project_id:
|
|
2210
2195
|
CloudDLPJobTriggersListLink.persist(
|
|
2211
2196
|
context=context,
|
|
2212
|
-
task_instance=self,
|
|
2213
2197
|
project_id=project_id,
|
|
2214
2198
|
)
|
|
2215
2199
|
|
|
@@ -2303,7 +2287,6 @@ class CloudDLPListStoredInfoTypesOperator(GoogleCloudBaseOperator):
|
|
|
2303
2287
|
if project_id:
|
|
2304
2288
|
CloudDLPInfoTypesListLink.persist(
|
|
2305
2289
|
context=context,
|
|
2306
|
-
task_instance=self,
|
|
2307
2290
|
project_id=project_id,
|
|
2308
2291
|
)
|
|
2309
2292
|
|
|
@@ -2590,7 +2573,6 @@ class CloudDLPUpdateDeidentifyTemplateOperator(GoogleCloudBaseOperator):
|
|
|
2590
2573
|
if project_id:
|
|
2591
2574
|
CloudDLPDeidentifyTemplateDetailsLink.persist(
|
|
2592
2575
|
context=context,
|
|
2593
|
-
task_instance=self,
|
|
2594
2576
|
project_id=project_id,
|
|
2595
2577
|
template_name=self.template_id,
|
|
2596
2578
|
)
|
|
@@ -2690,7 +2672,6 @@ class CloudDLPUpdateInspectTemplateOperator(GoogleCloudBaseOperator):
|
|
|
2690
2672
|
if project_id:
|
|
2691
2673
|
CloudDLPInspectTemplateDetailsLink.persist(
|
|
2692
2674
|
context=context,
|
|
2693
|
-
task_instance=self,
|
|
2694
2675
|
project_id=project_id,
|
|
2695
2676
|
template_name=self.template_id,
|
|
2696
2677
|
)
|
|
@@ -2784,7 +2765,6 @@ class CloudDLPUpdateJobTriggerOperator(GoogleCloudBaseOperator):
|
|
|
2784
2765
|
if project_id:
|
|
2785
2766
|
CloudDLPJobTriggerDetailsLink.persist(
|
|
2786
2767
|
context=context,
|
|
2787
|
-
task_instance=self,
|
|
2788
2768
|
project_id=project_id,
|
|
2789
2769
|
trigger_name=self.job_trigger_id,
|
|
2790
2770
|
)
|
|
@@ -2885,7 +2865,6 @@ class CloudDLPUpdateStoredInfoTypeOperator(GoogleCloudBaseOperator):
|
|
|
2885
2865
|
if project_id:
|
|
2886
2866
|
CloudDLPInfoTypeDetailsLink.persist(
|
|
2887
2867
|
context=context,
|
|
2888
|
-
task_instance=self,
|
|
2889
2868
|
project_id=project_id,
|
|
2890
2869
|
info_type_name=self.stored_info_type_id,
|
|
2891
2870
|
)
|