apache-airflow-providers-google 14.0.0__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
- airflow/providers/google/3rd-party-licenses/NOTICE +5 -0
- airflow/providers/google/__init__.py +3 -3
- airflow/providers/google/_vendor/__init__.py +0 -0
- airflow/providers/google/_vendor/json_merge_patch.py +91 -0
- airflow/providers/google/ads/hooks/ads.py +52 -43
- airflow/providers/google/ads/operators/ads.py +2 -2
- airflow/providers/google/ads/transfers/ads_to_gcs.py +3 -19
- airflow/providers/google/assets/gcs.py +1 -11
- airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
- airflow/providers/google/cloud/bundles/gcs.py +161 -0
- airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
- airflow/providers/google/cloud/hooks/bigquery.py +195 -318
- airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
- airflow/providers/google/cloud/hooks/bigtable.py +3 -2
- airflow/providers/google/cloud/hooks/cloud_batch.py +8 -9
- airflow/providers/google/cloud/hooks/cloud_build.py +6 -65
- airflow/providers/google/cloud/hooks/cloud_composer.py +292 -24
- airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
- airflow/providers/google/cloud/hooks/cloud_run.py +20 -11
- airflow/providers/google/cloud/hooks/cloud_sql.py +136 -64
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +35 -15
- airflow/providers/google/cloud/hooks/compute.py +7 -6
- airflow/providers/google/cloud/hooks/compute_ssh.py +7 -4
- airflow/providers/google/cloud/hooks/datacatalog.py +12 -3
- airflow/providers/google/cloud/hooks/dataflow.py +87 -242
- airflow/providers/google/cloud/hooks/dataform.py +9 -14
- airflow/providers/google/cloud/hooks/datafusion.py +7 -9
- airflow/providers/google/cloud/hooks/dataplex.py +13 -12
- airflow/providers/google/cloud/hooks/dataprep.py +2 -2
- airflow/providers/google/cloud/hooks/dataproc.py +76 -74
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/hooks/dlp.py +5 -4
- airflow/providers/google/cloud/hooks/gcs.py +144 -33
- airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
- airflow/providers/google/cloud/hooks/kms.py +3 -2
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +22 -17
- airflow/providers/google/cloud/hooks/looker.py +6 -1
- airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
- airflow/providers/google/cloud/hooks/mlengine.py +7 -8
- airflow/providers/google/cloud/hooks/natural_language.py +3 -2
- airflow/providers/google/cloud/hooks/os_login.py +3 -2
- airflow/providers/google/cloud/hooks/pubsub.py +6 -6
- airflow/providers/google/cloud/hooks/secret_manager.py +105 -12
- airflow/providers/google/cloud/hooks/spanner.py +75 -10
- airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
- airflow/providers/google/cloud/hooks/stackdriver.py +18 -18
- airflow/providers/google/cloud/hooks/tasks.py +4 -3
- airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
- airflow/providers/google/cloud/hooks/translate.py +8 -17
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +8 -222
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +9 -15
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +33 -283
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +311 -10
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +7 -13
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +8 -12
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
- airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
- airflow/providers/google/cloud/hooks/vision.py +7 -7
- airflow/providers/google/cloud/hooks/workflows.py +4 -3
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +77 -7
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -46
- airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -90
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +11 -89
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +11 -61
- airflow/providers/google/cloud/links/managed_kafka.py +11 -51
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +17 -187
- airflow/providers/google/cloud/links/vertex_ai.py +28 -195
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/log/gcs_task_handler.py +166 -118
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +14 -9
- airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
- airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
- airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
- airflow/providers/google/cloud/openlineage/facets.py +141 -40
- airflow/providers/google/cloud/openlineage/mixins.py +14 -13
- airflow/providers/google/cloud/openlineage/utils.py +19 -3
- airflow/providers/google/cloud/operators/alloy_db.py +76 -61
- airflow/providers/google/cloud/operators/bigquery.py +104 -667
- airflow/providers/google/cloud/operators/bigquery_dts.py +12 -12
- airflow/providers/google/cloud/operators/bigtable.py +38 -7
- airflow/providers/google/cloud/operators/cloud_base.py +22 -1
- airflow/providers/google/cloud/operators/cloud_batch.py +18 -18
- airflow/providers/google/cloud/operators/cloud_build.py +80 -36
- airflow/providers/google/cloud/operators/cloud_composer.py +157 -71
- airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
- airflow/providers/google/cloud/operators/cloud_memorystore.py +74 -46
- airflow/providers/google/cloud/operators/cloud_run.py +39 -20
- airflow/providers/google/cloud/operators/cloud_sql.py +46 -61
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +92 -14
- airflow/providers/google/cloud/operators/compute.py +18 -50
- airflow/providers/google/cloud/operators/datacatalog.py +167 -29
- airflow/providers/google/cloud/operators/dataflow.py +38 -15
- airflow/providers/google/cloud/operators/dataform.py +19 -7
- airflow/providers/google/cloud/operators/datafusion.py +43 -43
- airflow/providers/google/cloud/operators/dataplex.py +212 -126
- airflow/providers/google/cloud/operators/dataprep.py +1 -5
- airflow/providers/google/cloud/operators/dataproc.py +134 -207
- airflow/providers/google/cloud/operators/dataproc_metastore.py +102 -84
- airflow/providers/google/cloud/operators/datastore.py +22 -6
- airflow/providers/google/cloud/operators/dlp.py +24 -45
- airflow/providers/google/cloud/operators/functions.py +21 -14
- airflow/providers/google/cloud/operators/gcs.py +15 -12
- airflow/providers/google/cloud/operators/gen_ai.py +389 -0
- airflow/providers/google/cloud/operators/kubernetes_engine.py +115 -106
- airflow/providers/google/cloud/operators/looker.py +1 -1
- airflow/providers/google/cloud/operators/managed_kafka.py +362 -40
- airflow/providers/google/cloud/operators/natural_language.py +5 -3
- airflow/providers/google/cloud/operators/pubsub.py +69 -21
- airflow/providers/google/cloud/operators/spanner.py +53 -45
- airflow/providers/google/cloud/operators/speech_to_text.py +5 -4
- airflow/providers/google/cloud/operators/stackdriver.py +5 -11
- airflow/providers/google/cloud/operators/tasks.py +6 -15
- airflow/providers/google/cloud/operators/text_to_speech.py +4 -3
- airflow/providers/google/cloud/operators/translate.py +46 -20
- airflow/providers/google/cloud/operators/translate_speech.py +4 -3
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +44 -34
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +34 -12
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +62 -53
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +75 -11
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +48 -12
- airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -116
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +16 -12
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +62 -14
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +35 -10
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
- airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
- airflow/providers/google/cloud/operators/vision.py +7 -5
- airflow/providers/google/cloud/operators/workflows.py +24 -19
- airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
- airflow/providers/google/cloud/sensors/bigquery.py +2 -2
- airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -4
- airflow/providers/google/cloud/sensors/bigtable.py +14 -6
- airflow/providers/google/cloud/sensors/cloud_composer.py +535 -33
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -5
- airflow/providers/google/cloud/sensors/dataflow.py +27 -10
- airflow/providers/google/cloud/sensors/dataform.py +2 -2
- airflow/providers/google/cloud/sensors/datafusion.py +4 -4
- airflow/providers/google/cloud/sensors/dataplex.py +7 -5
- airflow/providers/google/cloud/sensors/dataprep.py +2 -2
- airflow/providers/google/cloud/sensors/dataproc.py +10 -9
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/sensors/gcs.py +22 -21
- airflow/providers/google/cloud/sensors/looker.py +5 -5
- airflow/providers/google/cloud/sensors/pubsub.py +20 -20
- airflow/providers/google/cloud/sensors/tasks.py +2 -2
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -2
- airflow/providers/google/cloud/sensors/workflows.py +6 -4
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +14 -13
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
- airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +18 -22
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -5
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +45 -38
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/gcs_to_local.py +5 -3
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +10 -4
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
- airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
- airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +44 -12
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +12 -6
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +36 -14
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
- airflow/providers/google/cloud/triggers/bigquery.py +75 -34
- airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +3 -2
- airflow/providers/google/cloud/triggers/cloud_composer.py +303 -47
- airflow/providers/google/cloud/triggers/cloud_run.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +96 -5
- airflow/providers/google/cloud/triggers/dataflow.py +125 -2
- airflow/providers/google/cloud/triggers/datafusion.py +1 -1
- airflow/providers/google/cloud/triggers/dataplex.py +16 -3
- airflow/providers/google/cloud/triggers/dataproc.py +124 -53
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +46 -28
- airflow/providers/google/cloud/triggers/mlengine.py +1 -1
- airflow/providers/google/cloud/triggers/pubsub.py +17 -20
- airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
- airflow/providers/google/cloud/utils/bigquery.py +5 -7
- airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
- airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
- airflow/providers/google/cloud/utils/field_validator.py +1 -2
- airflow/providers/google/cloud/utils/validators.py +43 -0
- airflow/providers/google/common/auth_backend/google_openid.py +26 -9
- airflow/providers/google/common/consts.py +2 -1
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +40 -43
- airflow/providers/google/common/hooks/operation_helpers.py +78 -0
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/common/utils/get_secret.py +31 -0
- airflow/providers/google/common/utils/id_token_credentials.py +4 -5
- airflow/providers/google/firebase/operators/firestore.py +2 -2
- airflow/providers/google/get_provider_info.py +61 -216
- airflow/providers/google/go_module_utils.py +35 -3
- airflow/providers/google/leveldb/hooks/leveldb.py +30 -6
- airflow/providers/google/leveldb/operators/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
- airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
- airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +4 -5
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -6
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +5 -5
- airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
- airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
- airflow/providers/google/marketing_platform/sensors/display_video.py +4 -64
- airflow/providers/google/suite/hooks/calendar.py +1 -1
- airflow/providers/google/suite/hooks/drive.py +2 -2
- airflow/providers/google/suite/hooks/sheets.py +15 -1
- airflow/providers/google/suite/operators/sheets.py +8 -3
- airflow/providers/google/suite/sensors/drive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
- airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
- airflow/providers/google/version_compat.py +15 -1
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/METADATA +117 -72
- apache_airflow_providers_google-19.1.0rc1.dist-info/RECORD +331 -0
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/WHEEL +1 -1
- apache_airflow_providers_google-19.1.0rc1.dist-info/licenses/NOTICE +5 -0
- airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
- airflow/providers/google/cloud/hooks/automl.py +0 -679
- airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
- airflow/providers/google/cloud/links/automl.py +0 -193
- airflow/providers/google/cloud/operators/automl.py +0 -1360
- airflow/providers/google/cloud/operators/life_sciences.py +0 -119
- airflow/providers/google/cloud/operators/mlengine.py +0 -1515
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +0 -273
- apache_airflow_providers_google-14.0.0.dist-info/RECORD +0 -318
- /airflow/providers/google/cloud/{example_dags → bundles}/__init__.py +0 -0
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/entry_points.txt +0 -0
- {airflow/providers/google → apache_airflow_providers_google-19.1.0rc1.dist-info/licenses}/LICENSE +0 -0
|
@@ -23,34 +23,19 @@ from collections.abc import MutableSequence, Sequence
|
|
|
23
23
|
from functools import cached_property
|
|
24
24
|
from typing import TYPE_CHECKING, Any
|
|
25
25
|
|
|
26
|
+
from google.protobuf.json_format import MessageToDict
|
|
27
|
+
|
|
26
28
|
from airflow.exceptions import AirflowException
|
|
27
29
|
from airflow.providers.google.cloud.triggers.dataplex import (
|
|
28
30
|
DataplexDataProfileJobTrigger,
|
|
29
31
|
DataplexDataQualityJobTrigger,
|
|
30
32
|
)
|
|
31
|
-
from google.protobuf.json_format import MessageToDict
|
|
32
33
|
|
|
33
34
|
if TYPE_CHECKING:
|
|
34
|
-
from airflow.utils.context import Context
|
|
35
35
|
from google.protobuf.field_mask_pb2 import FieldMask
|
|
36
36
|
|
|
37
|
-
from
|
|
37
|
+
from airflow.providers.common.compat.sdk import Context
|
|
38
38
|
|
|
39
|
-
from airflow.configuration import conf
|
|
40
|
-
from airflow.providers.google.cloud.hooks.dataplex import AirflowDataQualityScanException, DataplexHook
|
|
41
|
-
from airflow.providers.google.cloud.links.dataplex import (
|
|
42
|
-
DataplexCatalogAspectTypeLink,
|
|
43
|
-
DataplexCatalogAspectTypesLink,
|
|
44
|
-
DataplexCatalogEntryGroupLink,
|
|
45
|
-
DataplexCatalogEntryGroupsLink,
|
|
46
|
-
DataplexCatalogEntryLink,
|
|
47
|
-
DataplexCatalogEntryTypeLink,
|
|
48
|
-
DataplexCatalogEntryTypesLink,
|
|
49
|
-
DataplexLakeLink,
|
|
50
|
-
DataplexTaskLink,
|
|
51
|
-
DataplexTasksLink,
|
|
52
|
-
)
|
|
53
|
-
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
54
39
|
from google.api_core.exceptions import AlreadyExists, GoogleAPICallError, NotFound
|
|
55
40
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
56
41
|
from google.api_core.retry import Retry, exponential_sleep_generator
|
|
@@ -72,6 +57,23 @@ from google.cloud.dataplex_v1.types import (
|
|
|
72
57
|
Task,
|
|
73
58
|
Zone,
|
|
74
59
|
)
|
|
60
|
+
from googleapiclient.errors import HttpError
|
|
61
|
+
|
|
62
|
+
from airflow.configuration import conf
|
|
63
|
+
from airflow.providers.google.cloud.hooks.dataplex import AirflowDataQualityScanException, DataplexHook
|
|
64
|
+
from airflow.providers.google.cloud.links.dataplex import (
|
|
65
|
+
DataplexCatalogAspectTypeLink,
|
|
66
|
+
DataplexCatalogAspectTypesLink,
|
|
67
|
+
DataplexCatalogEntryGroupLink,
|
|
68
|
+
DataplexCatalogEntryGroupsLink,
|
|
69
|
+
DataplexCatalogEntryLink,
|
|
70
|
+
DataplexCatalogEntryTypeLink,
|
|
71
|
+
DataplexCatalogEntryTypesLink,
|
|
72
|
+
DataplexLakeLink,
|
|
73
|
+
DataplexTaskLink,
|
|
74
|
+
DataplexTasksLink,
|
|
75
|
+
)
|
|
76
|
+
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
75
77
|
|
|
76
78
|
|
|
77
79
|
class DataplexCreateTaskOperator(GoogleCloudBaseOperator):
|
|
@@ -148,6 +150,15 @@ class DataplexCreateTaskOperator(GoogleCloudBaseOperator):
|
|
|
148
150
|
self.impersonation_chain = impersonation_chain
|
|
149
151
|
self.asynchronous = asynchronous
|
|
150
152
|
|
|
153
|
+
@property
|
|
154
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
155
|
+
return {
|
|
156
|
+
"lake_id": self.lake_id,
|
|
157
|
+
"task_id": self.dataplex_task_id,
|
|
158
|
+
"region": self.region,
|
|
159
|
+
"project_id": self.project_id,
|
|
160
|
+
}
|
|
161
|
+
|
|
151
162
|
def execute(self, context: Context) -> dict:
|
|
152
163
|
hook = DataplexHook(
|
|
153
164
|
gcp_conn_id=self.gcp_conn_id,
|
|
@@ -155,7 +166,7 @@ class DataplexCreateTaskOperator(GoogleCloudBaseOperator):
|
|
|
155
166
|
impersonation_chain=self.impersonation_chain,
|
|
156
167
|
)
|
|
157
168
|
self.log.info("Creating Dataplex task %s", self.dataplex_task_id)
|
|
158
|
-
DataplexTaskLink.persist(context=context
|
|
169
|
+
DataplexTaskLink.persist(context=context)
|
|
159
170
|
|
|
160
171
|
try:
|
|
161
172
|
operation = hook.create_task(
|
|
@@ -349,6 +360,14 @@ class DataplexListTasksOperator(GoogleCloudBaseOperator):
|
|
|
349
360
|
self.gcp_conn_id = gcp_conn_id
|
|
350
361
|
self.impersonation_chain = impersonation_chain
|
|
351
362
|
|
|
363
|
+
@property
|
|
364
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
365
|
+
return {
|
|
366
|
+
"project_id": self.project_id,
|
|
367
|
+
"lake_id": self.lake_id,
|
|
368
|
+
"region": self.region,
|
|
369
|
+
}
|
|
370
|
+
|
|
352
371
|
def execute(self, context: Context) -> list[dict]:
|
|
353
372
|
hook = DataplexHook(
|
|
354
373
|
gcp_conn_id=self.gcp_conn_id,
|
|
@@ -356,7 +375,7 @@ class DataplexListTasksOperator(GoogleCloudBaseOperator):
|
|
|
356
375
|
impersonation_chain=self.impersonation_chain,
|
|
357
376
|
)
|
|
358
377
|
self.log.info("Listing Dataplex tasks from lake %s", self.lake_id)
|
|
359
|
-
DataplexTasksLink.persist(context=context
|
|
378
|
+
DataplexTasksLink.persist(context=context)
|
|
360
379
|
|
|
361
380
|
tasks = hook.list_tasks(
|
|
362
381
|
project_id=self.project_id,
|
|
@@ -428,6 +447,15 @@ class DataplexGetTaskOperator(GoogleCloudBaseOperator):
|
|
|
428
447
|
self.gcp_conn_id = gcp_conn_id
|
|
429
448
|
self.impersonation_chain = impersonation_chain
|
|
430
449
|
|
|
450
|
+
@property
|
|
451
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
452
|
+
return {
|
|
453
|
+
"lake_id": self.lake_id,
|
|
454
|
+
"task_id": self.dataplex_task_id,
|
|
455
|
+
"region": self.region,
|
|
456
|
+
"project_id": self.project_id,
|
|
457
|
+
}
|
|
458
|
+
|
|
431
459
|
def execute(self, context: Context) -> dict:
|
|
432
460
|
hook = DataplexHook(
|
|
433
461
|
gcp_conn_id=self.gcp_conn_id,
|
|
@@ -435,7 +463,7 @@ class DataplexGetTaskOperator(GoogleCloudBaseOperator):
|
|
|
435
463
|
impersonation_chain=self.impersonation_chain,
|
|
436
464
|
)
|
|
437
465
|
self.log.info("Retrieving Dataplex task %s", self.dataplex_task_id)
|
|
438
|
-
DataplexTaskLink.persist(context=context
|
|
466
|
+
DataplexTaskLink.persist(context=context)
|
|
439
467
|
|
|
440
468
|
task = hook.get_task(
|
|
441
469
|
project_id=self.project_id,
|
|
@@ -446,7 +474,7 @@ class DataplexGetTaskOperator(GoogleCloudBaseOperator):
|
|
|
446
474
|
timeout=self.timeout,
|
|
447
475
|
metadata=self.metadata,
|
|
448
476
|
)
|
|
449
|
-
DataplexTasksLink.persist(context=context
|
|
477
|
+
DataplexTasksLink.persist(context=context)
|
|
450
478
|
return Task.to_dict(task)
|
|
451
479
|
|
|
452
480
|
|
|
@@ -520,6 +548,14 @@ class DataplexCreateLakeOperator(GoogleCloudBaseOperator):
|
|
|
520
548
|
self.impersonation_chain = impersonation_chain
|
|
521
549
|
self.asynchronous = asynchronous
|
|
522
550
|
|
|
551
|
+
@property
|
|
552
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
553
|
+
return {
|
|
554
|
+
"lake_id": self.lake_id,
|
|
555
|
+
"region": self.region,
|
|
556
|
+
"project_id": self.project_id,
|
|
557
|
+
}
|
|
558
|
+
|
|
523
559
|
def execute(self, context: Context) -> dict:
|
|
524
560
|
hook = DataplexHook(
|
|
525
561
|
gcp_conn_id=self.gcp_conn_id,
|
|
@@ -564,10 +600,7 @@ class DataplexCreateLakeOperator(GoogleCloudBaseOperator):
|
|
|
564
600
|
if lake["state"] != "CREATING":
|
|
565
601
|
break
|
|
566
602
|
time.sleep(time_to_wait)
|
|
567
|
-
DataplexLakeLink.persist(
|
|
568
|
-
context=context,
|
|
569
|
-
task_instance=self,
|
|
570
|
-
)
|
|
603
|
+
DataplexLakeLink.persist(context=context)
|
|
571
604
|
return Lake.to_dict(lake)
|
|
572
605
|
|
|
573
606
|
|
|
@@ -623,6 +656,14 @@ class DataplexDeleteLakeOperator(GoogleCloudBaseOperator):
|
|
|
623
656
|
self.gcp_conn_id = gcp_conn_id
|
|
624
657
|
self.impersonation_chain = impersonation_chain
|
|
625
658
|
|
|
659
|
+
@property
|
|
660
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
661
|
+
return {
|
|
662
|
+
"lake_id": self.lake_id,
|
|
663
|
+
"region": self.region,
|
|
664
|
+
"project_id": self.project_id,
|
|
665
|
+
}
|
|
666
|
+
|
|
626
667
|
def execute(self, context: Context) -> None:
|
|
627
668
|
hook = DataplexHook(
|
|
628
669
|
gcp_conn_id=self.gcp_conn_id,
|
|
@@ -640,7 +681,7 @@ class DataplexDeleteLakeOperator(GoogleCloudBaseOperator):
|
|
|
640
681
|
timeout=self.timeout,
|
|
641
682
|
metadata=self.metadata,
|
|
642
683
|
)
|
|
643
|
-
DataplexLakeLink.persist(context=context
|
|
684
|
+
DataplexLakeLink.persist(context=context)
|
|
644
685
|
hook.wait_for_operation(timeout=self.timeout, operation=operation)
|
|
645
686
|
self.log.info("Dataplex lake %s deleted successfully!", self.lake_id)
|
|
646
687
|
|
|
@@ -1041,11 +1082,11 @@ class DataplexRunDataQualityScanOperator(GoogleCloudBaseOperator):
|
|
|
1041
1082
|
"""
|
|
1042
1083
|
job_state = event["job_state"]
|
|
1043
1084
|
job_id = event["job_id"]
|
|
1044
|
-
if job_state == DataScanJob.State.FAILED:
|
|
1085
|
+
if job_state == DataScanJob.State.FAILED.name: # type: ignore
|
|
1045
1086
|
raise AirflowException(f"Job failed:\n{job_id}")
|
|
1046
|
-
if job_state == DataScanJob.State.CANCELLED:
|
|
1087
|
+
if job_state == DataScanJob.State.CANCELLED.name: # type: ignore
|
|
1047
1088
|
raise AirflowException(f"Job was cancelled:\n{job_id}")
|
|
1048
|
-
if job_state == DataScanJob.State.SUCCEEDED:
|
|
1089
|
+
if job_state == DataScanJob.State.SUCCEEDED.name: # type: ignore
|
|
1049
1090
|
job = event["job"]
|
|
1050
1091
|
if not job["data_quality_result"]["passed"]:
|
|
1051
1092
|
if self.fail_on_dq_failure:
|
|
@@ -1219,11 +1260,11 @@ class DataplexGetDataQualityScanResultOperator(GoogleCloudBaseOperator):
|
|
|
1219
1260
|
job_state = event["job_state"]
|
|
1220
1261
|
job_id = event["job_id"]
|
|
1221
1262
|
job = event["job"]
|
|
1222
|
-
if job_state == DataScanJob.State.FAILED:
|
|
1263
|
+
if job_state == DataScanJob.State.FAILED.name: # type: ignore
|
|
1223
1264
|
raise AirflowException(f"Job failed:\n{job_id}")
|
|
1224
|
-
if job_state == DataScanJob.State.CANCELLED:
|
|
1265
|
+
if job_state == DataScanJob.State.CANCELLED.name: # type: ignore
|
|
1225
1266
|
raise AirflowException(f"Job was cancelled:\n{job_id}")
|
|
1226
|
-
if job_state == DataScanJob.State.SUCCEEDED:
|
|
1267
|
+
if job_state == DataScanJob.State.SUCCEEDED.name: # type: ignore
|
|
1227
1268
|
if not job["data_quality_result"]["passed"]:
|
|
1228
1269
|
if self.fail_on_dq_failure:
|
|
1229
1270
|
raise AirflowDataQualityScanException(
|
|
@@ -1598,12 +1639,12 @@ class DataplexRunDataProfileScanOperator(GoogleCloudBaseOperator):
|
|
|
1598
1639
|
result_timeout=self.result_timeout,
|
|
1599
1640
|
)
|
|
1600
1641
|
|
|
1601
|
-
if job.state == DataScanJob.State.FAILED:
|
|
1642
|
+
if job.state == DataScanJob.State.FAILED.name: # type: ignore
|
|
1602
1643
|
raise AirflowException(f"Data Profile job failed: {job_id}")
|
|
1603
|
-
if job.state == DataScanJob.State.SUCCEEDED:
|
|
1644
|
+
if job.state == DataScanJob.State.SUCCEEDED.name: # type: ignore
|
|
1604
1645
|
self.log.info("Data Profile job executed successfully.")
|
|
1605
1646
|
else:
|
|
1606
|
-
self.log.info("Data Profile job execution returned status: %s", job.
|
|
1647
|
+
self.log.info("Data Profile job execution returned status: %s", job.state)
|
|
1607
1648
|
|
|
1608
1649
|
return job_id
|
|
1609
1650
|
|
|
@@ -1616,11 +1657,11 @@ class DataplexRunDataProfileScanOperator(GoogleCloudBaseOperator):
|
|
|
1616
1657
|
"""
|
|
1617
1658
|
job_state = event["job_state"]
|
|
1618
1659
|
job_id = event["job_id"]
|
|
1619
|
-
if job_state == DataScanJob.State.FAILED:
|
|
1660
|
+
if job_state == DataScanJob.State.FAILED.name: # type: ignore
|
|
1620
1661
|
raise AirflowException(f"Job failed:\n{job_id}")
|
|
1621
|
-
if job_state == DataScanJob.State.CANCELLED:
|
|
1662
|
+
if job_state == DataScanJob.State.CANCELLED.name: # type: ignore
|
|
1622
1663
|
raise AirflowException(f"Job was cancelled:\n{job_id}")
|
|
1623
|
-
if job_state == DataScanJob.State.SUCCEEDED:
|
|
1664
|
+
if job_state == DataScanJob.State.SUCCEEDED.name: # type: ignore
|
|
1624
1665
|
self.log.info("Data Profile job executed successfully.")
|
|
1625
1666
|
return job_id
|
|
1626
1667
|
|
|
@@ -2177,6 +2218,13 @@ class DataplexCatalogBaseOperator(GoogleCloudBaseOperator):
|
|
|
2177
2218
|
impersonation_chain=self.impersonation_chain,
|
|
2178
2219
|
)
|
|
2179
2220
|
|
|
2221
|
+
@property
|
|
2222
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
2223
|
+
return {
|
|
2224
|
+
"location": self.location,
|
|
2225
|
+
"project_id": self.project_id,
|
|
2226
|
+
}
|
|
2227
|
+
|
|
2180
2228
|
|
|
2181
2229
|
class DataplexCatalogCreateEntryGroupOperator(DataplexCatalogBaseOperator):
|
|
2182
2230
|
"""
|
|
@@ -2228,12 +2276,15 @@ class DataplexCatalogCreateEntryGroupOperator(DataplexCatalogBaseOperator):
|
|
|
2228
2276
|
self.entry_group_configuration = entry_group_configuration
|
|
2229
2277
|
self.validate_request = validate_request
|
|
2230
2278
|
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2279
|
+
@property
|
|
2280
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
2281
|
+
return {
|
|
2282
|
+
**super().extra_links_params,
|
|
2283
|
+
"entry_group_id": self.entry_group_id,
|
|
2284
|
+
}
|
|
2236
2285
|
|
|
2286
|
+
def execute(self, context: Context):
|
|
2287
|
+
DataplexCatalogEntryGroupLink.persist(context=context)
|
|
2237
2288
|
if self.validate_request:
|
|
2238
2289
|
self.log.info("Validating a Create Dataplex Catalog EntryGroup request.")
|
|
2239
2290
|
else:
|
|
@@ -2314,11 +2365,15 @@ class DataplexCatalogGetEntryGroupOperator(DataplexCatalogBaseOperator):
|
|
|
2314
2365
|
super().__init__(*args, **kwargs)
|
|
2315
2366
|
self.entry_group_id = entry_group_id
|
|
2316
2367
|
|
|
2368
|
+
@property
|
|
2369
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
2370
|
+
return {
|
|
2371
|
+
**super().extra_links_params,
|
|
2372
|
+
"entry_group_id": self.entry_group_id,
|
|
2373
|
+
}
|
|
2374
|
+
|
|
2317
2375
|
def execute(self, context: Context):
|
|
2318
|
-
DataplexCatalogEntryGroupLink.persist(
|
|
2319
|
-
context=context,
|
|
2320
|
-
task_instance=self,
|
|
2321
|
-
)
|
|
2376
|
+
DataplexCatalogEntryGroupLink.persist(context=context)
|
|
2322
2377
|
self.log.info(
|
|
2323
2378
|
"Retrieving Dataplex Catalog EntryGroup %s.",
|
|
2324
2379
|
self.entry_group_id,
|
|
@@ -2460,10 +2515,7 @@ class DataplexCatalogListEntryGroupsOperator(DataplexCatalogBaseOperator):
|
|
|
2460
2515
|
self.order_by = order_by
|
|
2461
2516
|
|
|
2462
2517
|
def execute(self, context: Context):
|
|
2463
|
-
DataplexCatalogEntryGroupsLink.persist(
|
|
2464
|
-
context=context,
|
|
2465
|
-
task_instance=self,
|
|
2466
|
-
)
|
|
2518
|
+
DataplexCatalogEntryGroupsLink.persist(context=context)
|
|
2467
2519
|
self.log.info(
|
|
2468
2520
|
"Listing Dataplex Catalog EntryGroup from location %s.",
|
|
2469
2521
|
self.location,
|
|
@@ -2481,8 +2533,7 @@ class DataplexCatalogListEntryGroupsOperator(DataplexCatalogBaseOperator):
|
|
|
2481
2533
|
metadata=self.metadata,
|
|
2482
2534
|
)
|
|
2483
2535
|
self.log.info("EntryGroup on page: %s", entry_group_on_page)
|
|
2484
|
-
|
|
2485
|
-
context=context,
|
|
2536
|
+
context["ti"].xcom_push(
|
|
2486
2537
|
key="entry_group_page",
|
|
2487
2538
|
value=ListEntryGroupsResponse.to_dict(entry_group_on_page._response),
|
|
2488
2539
|
)
|
|
@@ -2552,12 +2603,15 @@ class DataplexCatalogUpdateEntryGroupOperator(DataplexCatalogBaseOperator):
|
|
|
2552
2603
|
self.update_mask = update_mask
|
|
2553
2604
|
self.validate_request = validate_request
|
|
2554
2605
|
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2606
|
+
@property
|
|
2607
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
2608
|
+
return {
|
|
2609
|
+
**super().extra_links_params,
|
|
2610
|
+
"entry_group_id": self.entry_group_id,
|
|
2611
|
+
}
|
|
2560
2612
|
|
|
2613
|
+
def execute(self, context: Context):
|
|
2614
|
+
DataplexCatalogEntryGroupLink.persist(context=context)
|
|
2561
2615
|
if self.validate_request:
|
|
2562
2616
|
self.log.info("Validating an Update Dataplex Catalog EntryGroup request.")
|
|
2563
2617
|
else:
|
|
@@ -2642,12 +2696,15 @@ class DataplexCatalogCreateEntryTypeOperator(DataplexCatalogBaseOperator):
|
|
|
2642
2696
|
self.entry_type_configuration = entry_type_configuration
|
|
2643
2697
|
self.validate_request = validate_request
|
|
2644
2698
|
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
|
|
2699
|
+
@property
|
|
2700
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
2701
|
+
return {
|
|
2702
|
+
**super().extra_links_params,
|
|
2703
|
+
"entry_type_id": self.entry_type_id,
|
|
2704
|
+
}
|
|
2650
2705
|
|
|
2706
|
+
def execute(self, context: Context):
|
|
2707
|
+
DataplexCatalogEntryTypeLink.persist(context=context)
|
|
2651
2708
|
if self.validate_request:
|
|
2652
2709
|
self.log.info("Validating a Create Dataplex Catalog EntryType request.")
|
|
2653
2710
|
else:
|
|
@@ -2728,11 +2785,15 @@ class DataplexCatalogGetEntryTypeOperator(DataplexCatalogBaseOperator):
|
|
|
2728
2785
|
super().__init__(*args, **kwargs)
|
|
2729
2786
|
self.entry_type_id = entry_type_id
|
|
2730
2787
|
|
|
2788
|
+
@property
|
|
2789
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
2790
|
+
return {
|
|
2791
|
+
**super().extra_links_params,
|
|
2792
|
+
"entry_type_id": self.entry_type_id,
|
|
2793
|
+
}
|
|
2794
|
+
|
|
2731
2795
|
def execute(self, context: Context):
|
|
2732
|
-
DataplexCatalogEntryTypeLink.persist(
|
|
2733
|
-
context=context,
|
|
2734
|
-
task_instance=self,
|
|
2735
|
-
)
|
|
2796
|
+
DataplexCatalogEntryTypeLink.persist(context=context)
|
|
2736
2797
|
self.log.info(
|
|
2737
2798
|
"Retrieving Dataplex Catalog EntryType %s.",
|
|
2738
2799
|
self.entry_type_id,
|
|
@@ -2874,10 +2935,7 @@ class DataplexCatalogListEntryTypesOperator(DataplexCatalogBaseOperator):
|
|
|
2874
2935
|
self.order_by = order_by
|
|
2875
2936
|
|
|
2876
2937
|
def execute(self, context: Context):
|
|
2877
|
-
DataplexCatalogEntryTypesLink.persist(
|
|
2878
|
-
context=context,
|
|
2879
|
-
task_instance=self,
|
|
2880
|
-
)
|
|
2938
|
+
DataplexCatalogEntryTypesLink.persist(context=context)
|
|
2881
2939
|
self.log.info(
|
|
2882
2940
|
"Listing Dataplex Catalog EntryType from location %s.",
|
|
2883
2941
|
self.location,
|
|
@@ -2895,8 +2953,7 @@ class DataplexCatalogListEntryTypesOperator(DataplexCatalogBaseOperator):
|
|
|
2895
2953
|
metadata=self.metadata,
|
|
2896
2954
|
)
|
|
2897
2955
|
self.log.info("EntryType on page: %s", entry_type_on_page)
|
|
2898
|
-
|
|
2899
|
-
context=context,
|
|
2956
|
+
context["ti"].xcom_push(
|
|
2900
2957
|
key="entry_type_page",
|
|
2901
2958
|
value=ListEntryTypesResponse.to_dict(entry_type_on_page._response),
|
|
2902
2959
|
)
|
|
@@ -2966,12 +3023,15 @@ class DataplexCatalogUpdateEntryTypeOperator(DataplexCatalogBaseOperator):
|
|
|
2966
3023
|
self.update_mask = update_mask
|
|
2967
3024
|
self.validate_request = validate_request
|
|
2968
3025
|
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
3026
|
+
@property
|
|
3027
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
3028
|
+
return {
|
|
3029
|
+
**super().extra_links_params,
|
|
3030
|
+
"entry_type_id": self.entry_type_id,
|
|
3031
|
+
}
|
|
2974
3032
|
|
|
3033
|
+
def execute(self, context: Context):
|
|
3034
|
+
DataplexCatalogEntryTypeLink.persist(context=context)
|
|
2975
3035
|
if self.validate_request:
|
|
2976
3036
|
self.log.info("Validating an Update Dataplex Catalog EntryType request.")
|
|
2977
3037
|
else:
|
|
@@ -3056,12 +3116,15 @@ class DataplexCatalogCreateAspectTypeOperator(DataplexCatalogBaseOperator):
|
|
|
3056
3116
|
self.aspect_type_configuration = aspect_type_configuration
|
|
3057
3117
|
self.validate_request = validate_request
|
|
3058
3118
|
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
3119
|
+
@property
|
|
3120
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
3121
|
+
return {
|
|
3122
|
+
**super().extra_links_params,
|
|
3123
|
+
"aspect_type_id": self.aspect_type_id,
|
|
3124
|
+
}
|
|
3064
3125
|
|
|
3126
|
+
def execute(self, context: Context):
|
|
3127
|
+
DataplexCatalogAspectTypeLink.persist(context=context)
|
|
3065
3128
|
if self.validate_request:
|
|
3066
3129
|
self.log.info("Validating a Create Dataplex Catalog AspectType request.")
|
|
3067
3130
|
else:
|
|
@@ -3142,11 +3205,15 @@ class DataplexCatalogGetAspectTypeOperator(DataplexCatalogBaseOperator):
|
|
|
3142
3205
|
super().__init__(*args, **kwargs)
|
|
3143
3206
|
self.aspect_type_id = aspect_type_id
|
|
3144
3207
|
|
|
3208
|
+
@property
|
|
3209
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
3210
|
+
return {
|
|
3211
|
+
**super().extra_links_params,
|
|
3212
|
+
"aspect_type_id": self.aspect_type_id,
|
|
3213
|
+
}
|
|
3214
|
+
|
|
3145
3215
|
def execute(self, context: Context):
|
|
3146
|
-
DataplexCatalogAspectTypeLink.persist(
|
|
3147
|
-
context=context,
|
|
3148
|
-
task_instance=self,
|
|
3149
|
-
)
|
|
3216
|
+
DataplexCatalogAspectTypeLink.persist(context=context)
|
|
3150
3217
|
self.log.info(
|
|
3151
3218
|
"Retrieving Dataplex Catalog AspectType %s.",
|
|
3152
3219
|
self.aspect_type_id,
|
|
@@ -3221,10 +3288,7 @@ class DataplexCatalogListAspectTypesOperator(DataplexCatalogBaseOperator):
|
|
|
3221
3288
|
self.order_by = order_by
|
|
3222
3289
|
|
|
3223
3290
|
def execute(self, context: Context):
|
|
3224
|
-
DataplexCatalogAspectTypesLink.persist(
|
|
3225
|
-
context=context,
|
|
3226
|
-
task_instance=self,
|
|
3227
|
-
)
|
|
3291
|
+
DataplexCatalogAspectTypesLink.persist(context=context)
|
|
3228
3292
|
self.log.info(
|
|
3229
3293
|
"Listing Dataplex Catalog AspectType from location %s.",
|
|
3230
3294
|
self.location,
|
|
@@ -3242,8 +3306,7 @@ class DataplexCatalogListAspectTypesOperator(DataplexCatalogBaseOperator):
|
|
|
3242
3306
|
metadata=self.metadata,
|
|
3243
3307
|
)
|
|
3244
3308
|
self.log.info("AspectType on page: %s", aspect_type_on_page)
|
|
3245
|
-
|
|
3246
|
-
context=context,
|
|
3309
|
+
context["ti"].xcom_push(
|
|
3247
3310
|
key="aspect_type_page",
|
|
3248
3311
|
value=ListAspectTypesResponse.to_dict(aspect_type_on_page._response),
|
|
3249
3312
|
)
|
|
@@ -3313,12 +3376,15 @@ class DataplexCatalogUpdateAspectTypeOperator(DataplexCatalogBaseOperator):
|
|
|
3313
3376
|
self.update_mask = update_mask
|
|
3314
3377
|
self.validate_request = validate_request
|
|
3315
3378
|
|
|
3316
|
-
|
|
3317
|
-
|
|
3318
|
-
|
|
3319
|
-
|
|
3320
|
-
|
|
3379
|
+
@property
|
|
3380
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
3381
|
+
return {
|
|
3382
|
+
**super().extra_links_params,
|
|
3383
|
+
"aspect_type_id": self.aspect_type_id,
|
|
3384
|
+
}
|
|
3321
3385
|
|
|
3386
|
+
def execute(self, context: Context):
|
|
3387
|
+
DataplexCatalogAspectTypeLink.persist(context=context)
|
|
3322
3388
|
if self.validate_request:
|
|
3323
3389
|
self.log.info("Validating an Update Dataplex Catalog AspectType request.")
|
|
3324
3390
|
else:
|
|
@@ -3491,12 +3557,16 @@ class DataplexCatalogCreateEntryOperator(DataplexCatalogBaseOperator):
|
|
|
3491
3557
|
f"Missing required fields in Entry configuration: {', '.join(missing_fields)}. "
|
|
3492
3558
|
)
|
|
3493
3559
|
|
|
3494
|
-
|
|
3495
|
-
|
|
3496
|
-
|
|
3497
|
-
|
|
3498
|
-
|
|
3560
|
+
@property
|
|
3561
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
3562
|
+
return {
|
|
3563
|
+
**super().extra_links_params,
|
|
3564
|
+
"entry_id": self.entry_id,
|
|
3565
|
+
"entry_group_id": self.entry_group_id,
|
|
3566
|
+
}
|
|
3499
3567
|
|
|
3568
|
+
def execute(self, context: Context):
|
|
3569
|
+
DataplexCatalogEntryLink.persist(context=context)
|
|
3500
3570
|
self._validate_fields(self.entry_configuration)
|
|
3501
3571
|
try:
|
|
3502
3572
|
entry = self.hook.create_entry(
|
|
@@ -3597,11 +3667,16 @@ class DataplexCatalogGetEntryOperator(DataplexCatalogBaseOperator):
|
|
|
3597
3667
|
self.aspect_types = aspect_types
|
|
3598
3668
|
self.paths = paths
|
|
3599
3669
|
|
|
3670
|
+
@property
|
|
3671
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
3672
|
+
return {
|
|
3673
|
+
**super().extra_links_params,
|
|
3674
|
+
"entry_id": self.entry_id,
|
|
3675
|
+
"entry_group_id": self.entry_group_id,
|
|
3676
|
+
}
|
|
3677
|
+
|
|
3600
3678
|
def execute(self, context: Context):
|
|
3601
|
-
DataplexCatalogEntryLink.persist(
|
|
3602
|
-
context=context,
|
|
3603
|
-
task_instance=self,
|
|
3604
|
-
)
|
|
3679
|
+
DataplexCatalogEntryLink.persist(context=context)
|
|
3605
3680
|
self.log.info(
|
|
3606
3681
|
"Retrieving Dataplex Catalog Entry %s.",
|
|
3607
3682
|
self.entry_id,
|
|
@@ -3699,11 +3774,15 @@ class DataplexCatalogListEntriesOperator(DataplexCatalogBaseOperator):
|
|
|
3699
3774
|
self.page_token = page_token
|
|
3700
3775
|
self.filter_by = filter_by
|
|
3701
3776
|
|
|
3777
|
+
@property
|
|
3778
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
3779
|
+
return {
|
|
3780
|
+
**super().extra_links_params,
|
|
3781
|
+
"entry_group_id": self.entry_group_id,
|
|
3782
|
+
}
|
|
3783
|
+
|
|
3702
3784
|
def execute(self, context: Context):
|
|
3703
|
-
DataplexCatalogEntryGroupLink.persist(
|
|
3704
|
-
context=context,
|
|
3705
|
-
task_instance=self,
|
|
3706
|
-
)
|
|
3785
|
+
DataplexCatalogEntryGroupLink.persist(context=context)
|
|
3707
3786
|
self.log.info(
|
|
3708
3787
|
"Listing Dataplex Catalog Entry from location %s.",
|
|
3709
3788
|
self.location,
|
|
@@ -3721,8 +3800,7 @@ class DataplexCatalogListEntriesOperator(DataplexCatalogBaseOperator):
|
|
|
3721
3800
|
metadata=self.metadata,
|
|
3722
3801
|
)
|
|
3723
3802
|
self.log.info("Entries on page: %s", entries_on_page)
|
|
3724
|
-
|
|
3725
|
-
context=context,
|
|
3803
|
+
context["ti"].xcom_push(
|
|
3726
3804
|
key="entry_page",
|
|
3727
3805
|
value=ListEntriesResponse.to_dict(entries_on_page._response),
|
|
3728
3806
|
)
|
|
@@ -3819,8 +3897,7 @@ class DataplexCatalogSearchEntriesOperator(DataplexCatalogBaseOperator):
|
|
|
3819
3897
|
metadata=self.metadata,
|
|
3820
3898
|
)
|
|
3821
3899
|
self.log.info("Entries on page: %s", entries_on_page)
|
|
3822
|
-
|
|
3823
|
-
context=context,
|
|
3900
|
+
context["ti"].xcom_push(
|
|
3824
3901
|
key="entry_page",
|
|
3825
3902
|
value=SearchEntriesResponse.to_dict(entries_on_page._response),
|
|
3826
3903
|
)
|
|
@@ -3901,11 +3978,16 @@ class DataplexCatalogLookupEntryOperator(DataplexCatalogBaseOperator):
|
|
|
3901
3978
|
self.aspect_types = aspect_types
|
|
3902
3979
|
self.paths = paths
|
|
3903
3980
|
|
|
3981
|
+
@property
|
|
3982
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
3983
|
+
return {
|
|
3984
|
+
**super().extra_links_params,
|
|
3985
|
+
"entry_id": self.entry_id,
|
|
3986
|
+
"entry_group_id": self.entry_group_id,
|
|
3987
|
+
}
|
|
3988
|
+
|
|
3904
3989
|
def execute(self, context: Context):
|
|
3905
|
-
DataplexCatalogEntryLink.persist(
|
|
3906
|
-
context=context,
|
|
3907
|
-
task_instance=self,
|
|
3908
|
-
)
|
|
3990
|
+
DataplexCatalogEntryLink.persist(context=context)
|
|
3909
3991
|
self.log.info(
|
|
3910
3992
|
"Looking for Dataplex Catalog Entry %s.",
|
|
3911
3993
|
self.entry_id,
|
|
@@ -3961,7 +4043,7 @@ class DataplexCatalogUpdateEntryOperator(DataplexCatalogBaseOperator):
|
|
|
3961
4043
|
:param entry_configuration: Required. The updated configuration body of the Entry.
|
|
3962
4044
|
:param allow_missing: Optional. If set to true and entry doesn't exist, the service will create it.
|
|
3963
4045
|
:param delete_missing_aspects: Optional. If set to true and the aspect_keys specify aspect
|
|
3964
|
-
ranges, the service deletes any existing aspects from that range that
|
|
4046
|
+
ranges, the service deletes any existing aspects from that range that were not provided
|
|
3965
4047
|
in the request.
|
|
3966
4048
|
:param aspect_keys: Optional. The map keys of the Aspects which the service should modify.
|
|
3967
4049
|
It supports the following syntax:
|
|
@@ -4020,12 +4102,16 @@ class DataplexCatalogUpdateEntryOperator(DataplexCatalogBaseOperator):
|
|
|
4020
4102
|
self.delete_missing_aspects = delete_missing_aspects
|
|
4021
4103
|
self.aspect_keys = aspect_keys
|
|
4022
4104
|
|
|
4023
|
-
|
|
4024
|
-
|
|
4025
|
-
|
|
4026
|
-
|
|
4027
|
-
|
|
4105
|
+
@property
|
|
4106
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
4107
|
+
return {
|
|
4108
|
+
**super().extra_links_params,
|
|
4109
|
+
"entry_id": self.entry_id,
|
|
4110
|
+
"entry_group_id": self.entry_group_id,
|
|
4111
|
+
}
|
|
4028
4112
|
|
|
4113
|
+
def execute(self, context: Context):
|
|
4114
|
+
DataplexCatalogEntryLink.persist(context=context)
|
|
4029
4115
|
try:
|
|
4030
4116
|
entry = self.hook.update_entry(
|
|
4031
4117
|
location=self.location,
|
|
@@ -28,7 +28,7 @@ from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseO
|
|
|
28
28
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
29
29
|
|
|
30
30
|
if TYPE_CHECKING:
|
|
31
|
-
from airflow.
|
|
31
|
+
from airflow.providers.common.compat.sdk import Context
|
|
32
32
|
|
|
33
33
|
|
|
34
34
|
class DataprepGetJobsForJobGroupOperator(GoogleCloudBaseOperator):
|
|
@@ -113,7 +113,6 @@ class DataprepGetJobGroupOperator(GoogleCloudBaseOperator):
|
|
|
113
113
|
if self.project_id:
|
|
114
114
|
DataprepJobGroupLink.persist(
|
|
115
115
|
context=context,
|
|
116
|
-
task_instance=self,
|
|
117
116
|
project_id=self.project_id,
|
|
118
117
|
job_group_id=int(self.job_group_id),
|
|
119
118
|
)
|
|
@@ -170,7 +169,6 @@ class DataprepRunJobGroupOperator(GoogleCloudBaseOperator):
|
|
|
170
169
|
if self.project_id and job_group_id:
|
|
171
170
|
DataprepJobGroupLink.persist(
|
|
172
171
|
context=context,
|
|
173
|
-
task_instance=self,
|
|
174
172
|
project_id=self.project_id,
|
|
175
173
|
job_group_id=int(job_group_id),
|
|
176
174
|
)
|
|
@@ -230,7 +228,6 @@ class DataprepCopyFlowOperator(GoogleCloudBaseOperator):
|
|
|
230
228
|
if self.project_id and copied_flow_id:
|
|
231
229
|
DataprepFlowLink.persist(
|
|
232
230
|
context=context,
|
|
233
|
-
task_instance=self,
|
|
234
231
|
project_id=self.project_id,
|
|
235
232
|
flow_id=int(copied_flow_id),
|
|
236
233
|
)
|
|
@@ -303,7 +300,6 @@ class DataprepRunFlowOperator(GoogleCloudBaseOperator):
|
|
|
303
300
|
job_group_id = response["data"][0]["id"]
|
|
304
301
|
DataprepJobGroupLink.persist(
|
|
305
302
|
context=context,
|
|
306
|
-
task_instance=self,
|
|
307
303
|
project_id=self.project_id,
|
|
308
304
|
job_group_id=int(job_group_id),
|
|
309
305
|
)
|