apache-airflow-providers-google 16.0.0rc1__py3-none-any.whl → 16.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/ads/hooks/ads.py +9 -5
- airflow/providers/google/ads/operators/ads.py +1 -1
- airflow/providers/google/ads/transfers/ads_to_gcs.py +1 -1
- airflow/providers/google/cloud/hooks/bigquery.py +2 -3
- airflow/providers/google/cloud/hooks/cloud_sql.py +8 -4
- airflow/providers/google/cloud/hooks/datacatalog.py +9 -1
- airflow/providers/google/cloud/hooks/dataflow.py +2 -2
- airflow/providers/google/cloud/hooks/dataplex.py +1 -1
- airflow/providers/google/cloud/hooks/dataprep.py +4 -1
- airflow/providers/google/cloud/hooks/gcs.py +2 -2
- airflow/providers/google/cloud/hooks/looker.py +5 -1
- airflow/providers/google/cloud/hooks/mlengine.py +2 -1
- airflow/providers/google/cloud/hooks/secret_manager.py +102 -10
- airflow/providers/google/cloud/hooks/spanner.py +2 -2
- airflow/providers/google/cloud/hooks/translate.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +307 -7
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +43 -14
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +11 -2
- airflow/providers/google/cloud/hooks/vision.py +2 -2
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +75 -11
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/cloud_run.py +1 -33
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +16 -43
- airflow/providers/google/cloud/links/cloud_tasks.py +6 -25
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -96
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +14 -90
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +5 -59
- airflow/providers/google/cloud/links/life_sciences.py +0 -19
- airflow/providers/google/cloud/links/managed_kafka.py +0 -70
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +16 -186
- airflow/providers/google/cloud/links/vertex_ai.py +8 -224
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/operators/alloy_db.py +69 -54
- airflow/providers/google/cloud/operators/automl.py +16 -14
- airflow/providers/google/cloud/operators/bigquery.py +0 -15
- airflow/providers/google/cloud/operators/bigquery_dts.py +2 -4
- airflow/providers/google/cloud/operators/bigtable.py +35 -6
- airflow/providers/google/cloud/operators/cloud_base.py +21 -1
- airflow/providers/google/cloud/operators/cloud_build.py +74 -31
- airflow/providers/google/cloud/operators/cloud_composer.py +34 -35
- airflow/providers/google/cloud/operators/cloud_memorystore.py +68 -42
- airflow/providers/google/cloud/operators/cloud_run.py +0 -1
- airflow/providers/google/cloud/operators/cloud_sql.py +11 -15
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +0 -2
- airflow/providers/google/cloud/operators/compute.py +7 -39
- airflow/providers/google/cloud/operators/datacatalog.py +156 -20
- airflow/providers/google/cloud/operators/dataflow.py +37 -14
- airflow/providers/google/cloud/operators/dataform.py +14 -4
- airflow/providers/google/cloud/operators/datafusion.py +4 -12
- airflow/providers/google/cloud/operators/dataplex.py +180 -96
- airflow/providers/google/cloud/operators/dataprep.py +0 -4
- airflow/providers/google/cloud/operators/dataproc.py +10 -16
- airflow/providers/google/cloud/operators/dataproc_metastore.py +95 -87
- airflow/providers/google/cloud/operators/datastore.py +21 -5
- airflow/providers/google/cloud/operators/dlp.py +3 -26
- airflow/providers/google/cloud/operators/functions.py +15 -6
- airflow/providers/google/cloud/operators/gcs.py +0 -7
- airflow/providers/google/cloud/operators/kubernetes_engine.py +50 -7
- airflow/providers/google/cloud/operators/life_sciences.py +0 -1
- airflow/providers/google/cloud/operators/managed_kafka.py +106 -51
- airflow/providers/google/cloud/operators/mlengine.py +0 -1
- airflow/providers/google/cloud/operators/pubsub.py +2 -4
- airflow/providers/google/cloud/operators/spanner.py +0 -4
- airflow/providers/google/cloud/operators/speech_to_text.py +0 -1
- airflow/providers/google/cloud/operators/stackdriver.py +0 -8
- airflow/providers/google/cloud/operators/tasks.py +0 -11
- airflow/providers/google/cloud/operators/text_to_speech.py +0 -1
- airflow/providers/google/cloud/operators/translate.py +37 -13
- airflow/providers/google/cloud/operators/translate_speech.py +0 -1
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +31 -18
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +28 -8
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +38 -25
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +69 -7
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +42 -8
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +531 -0
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +93 -25
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +10 -8
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +56 -10
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +25 -6
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +9 -6
- airflow/providers/google/cloud/operators/workflows.py +1 -9
- airflow/providers/google/cloud/sensors/bigquery.py +1 -1
- airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -1
- airflow/providers/google/cloud/sensors/bigtable.py +15 -3
- airflow/providers/google/cloud/sensors/cloud_composer.py +6 -1
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -1
- airflow/providers/google/cloud/sensors/dataflow.py +3 -3
- airflow/providers/google/cloud/sensors/dataform.py +6 -1
- airflow/providers/google/cloud/sensors/datafusion.py +6 -1
- airflow/providers/google/cloud/sensors/dataplex.py +6 -1
- airflow/providers/google/cloud/sensors/dataprep.py +6 -1
- airflow/providers/google/cloud/sensors/dataproc.py +6 -1
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +6 -1
- airflow/providers/google/cloud/sensors/gcs.py +9 -3
- airflow/providers/google/cloud/sensors/looker.py +6 -1
- airflow/providers/google/cloud/sensors/pubsub.py +8 -3
- airflow/providers/google/cloud/sensors/tasks.py +6 -1
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +6 -1
- airflow/providers/google/cloud/sensors/workflows.py +6 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +1 -2
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +1 -2
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +0 -1
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +1 -1
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +1 -2
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/gcs_to_local.py +1 -1
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +5 -1
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +1 -1
- airflow/providers/google/cloud/transfers/http_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/local_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +1 -1
- airflow/providers/google/cloud/utils/field_validator.py +1 -2
- airflow/providers/google/common/auth_backend/google_openid.py +2 -1
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +7 -3
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/firebase/operators/firestore.py +1 -1
- airflow/providers/google/get_provider_info.py +0 -11
- airflow/providers/google/leveldb/hooks/leveldb.py +5 -1
- airflow/providers/google/leveldb/operators/leveldb.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +3 -6
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +0 -1
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +4 -4
- airflow/providers/google/marketing_platform/operators/display_video.py +6 -6
- airflow/providers/google/marketing_platform/operators/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +6 -1
- airflow/providers/google/marketing_platform/sensors/display_video.py +6 -1
- airflow/providers/google/suite/operators/sheets.py +3 -3
- airflow/providers/google/suite/sensors/drive.py +6 -1
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +1 -1
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +1 -1
- airflow/providers/google/version_compat.py +28 -0
- {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/METADATA +19 -20
- {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/RECORD +166 -166
- {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/entry_points.txt +0 -0
@@ -60,18 +60,6 @@ class CloudStorageTransferListLink(BaseGoogleLink):
|
|
60
60
|
key = "cloud_storage_transfer"
|
61
61
|
format_str = CLOUD_STORAGE_TRANSFER_LIST_LINK
|
62
62
|
|
63
|
-
@staticmethod
|
64
|
-
def persist(
|
65
|
-
context: Context,
|
66
|
-
task_instance,
|
67
|
-
project_id: str,
|
68
|
-
):
|
69
|
-
task_instance.xcom_push(
|
70
|
-
context,
|
71
|
-
key=CloudStorageTransferListLink.key,
|
72
|
-
value={"project_id": project_id},
|
73
|
-
)
|
74
|
-
|
75
63
|
|
76
64
|
class CloudStorageTransferJobLink(BaseGoogleLink):
|
77
65
|
"""Helper class for constructing Storage Transfer Job Link."""
|
@@ -80,22 +68,6 @@ class CloudStorageTransferJobLink(BaseGoogleLink):
|
|
80
68
|
key = "cloud_storage_transfer_job"
|
81
69
|
format_str = CLOUD_STORAGE_TRANSFER_JOB_LINK
|
82
70
|
|
83
|
-
@staticmethod
|
84
|
-
def persist(
|
85
|
-
context: Context,
|
86
|
-
project_id: str,
|
87
|
-
job_name: str,
|
88
|
-
):
|
89
|
-
job_name = job_name.split("/")[1] if job_name else ""
|
90
|
-
|
91
|
-
context["ti"].xcom_push(
|
92
|
-
key=CloudStorageTransferJobLink.key,
|
93
|
-
value={
|
94
|
-
"project_id": project_id,
|
95
|
-
"transfer_job": job_name,
|
96
|
-
},
|
97
|
-
)
|
98
|
-
|
99
71
|
|
100
72
|
class CloudStorageTransferDetailsLink(BaseGoogleLink):
|
101
73
|
"""Helper class for constructing Cloud Storage Transfer Operation Link."""
|
@@ -105,20 +77,21 @@ class CloudStorageTransferDetailsLink(BaseGoogleLink):
|
|
105
77
|
format_str = CLOUD_STORAGE_TRANSFER_OPERATION_LINK
|
106
78
|
|
107
79
|
@staticmethod
|
108
|
-
def
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
operation_name
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
80
|
+
def extract_parts(operation_name: str | None):
|
81
|
+
if not operation_name:
|
82
|
+
return "", ""
|
83
|
+
transfer_operation = operation_name.split("/")[1]
|
84
|
+
transfer_job = operation_name.split("-")[1]
|
85
|
+
return transfer_operation, transfer_job
|
86
|
+
|
87
|
+
@classmethod
|
88
|
+
def persist(cls, context: Context, **value):
|
89
|
+
operation_name = value.get("operation_name")
|
90
|
+
transfer_operation, transfer_job = cls.extract_parts(operation_name)
|
91
|
+
|
92
|
+
super().persist(
|
117
93
|
context,
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
"transfer_job": transfer_job,
|
122
|
-
"transfer_operation": transfer_operation,
|
123
|
-
},
|
94
|
+
project_id=value.get("project_id"),
|
95
|
+
transfer_job=transfer_job,
|
96
|
+
transfer_operation=transfer_operation,
|
124
97
|
)
|
@@ -24,7 +24,6 @@ from typing import TYPE_CHECKING
|
|
24
24
|
from airflow.providers.google.cloud.links.base import BaseGoogleLink
|
25
25
|
|
26
26
|
if TYPE_CHECKING:
|
27
|
-
from airflow.models import BaseOperator
|
28
27
|
from airflow.utils.context import Context
|
29
28
|
|
30
29
|
CLOUD_TASKS_BASE_LINK = "/cloudtasks"
|
@@ -51,18 +50,12 @@ class CloudTasksQueueLink(BaseGoogleLink):
|
|
51
50
|
parts = queue_name.split("/")
|
52
51
|
return parts[1], parts[3], parts[5]
|
53
52
|
|
54
|
-
@
|
55
|
-
def persist(
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
project_id, location, queue_id = CloudTasksQueueLink.extract_parts(queue_name)
|
61
|
-
operator_instance.xcom_push(
|
62
|
-
context,
|
63
|
-
key=CloudTasksQueueLink.key,
|
64
|
-
value={"project_id": project_id, "location": location, "queue_id": queue_id},
|
65
|
-
)
|
53
|
+
@classmethod
|
54
|
+
def persist(cls, context: Context, **value):
|
55
|
+
queue_name = value.get("queue_name")
|
56
|
+
project_id, location, queue_id = cls.extract_parts(queue_name)
|
57
|
+
|
58
|
+
super().persist(context, project_id=project_id, location=location, queue_id=queue_id)
|
66
59
|
|
67
60
|
|
68
61
|
class CloudTasksLink(BaseGoogleLink):
|
@@ -71,15 +64,3 @@ class CloudTasksLink(BaseGoogleLink):
|
|
71
64
|
name = "Cloud Tasks"
|
72
65
|
key = "cloud_task"
|
73
66
|
format_str = CLOUD_TASKS_LINK
|
74
|
-
|
75
|
-
@staticmethod
|
76
|
-
def persist(
|
77
|
-
operator_instance: BaseOperator,
|
78
|
-
context: Context,
|
79
|
-
project_id: str | None,
|
80
|
-
):
|
81
|
-
operator_instance.xcom_push(
|
82
|
-
context,
|
83
|
-
key=CloudTasksLink.key,
|
84
|
-
value={"project_id": project_id},
|
85
|
-
)
|
@@ -19,14 +19,8 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from typing import TYPE_CHECKING
|
23
|
-
|
24
22
|
from airflow.providers.google.cloud.links.base import BaseGoogleLink
|
25
23
|
|
26
|
-
if TYPE_CHECKING:
|
27
|
-
from airflow.models import BaseOperator
|
28
|
-
from airflow.utils.context import Context
|
29
|
-
|
30
24
|
COMPUTE_BASE_LINK = "https://console.cloud.google.com/compute"
|
31
25
|
COMPUTE_LINK = (
|
32
26
|
COMPUTE_BASE_LINK + "/instancesDetail/zones/{location_id}/instances/{resource_id}?project={project_id}"
|
@@ -44,24 +38,6 @@ class ComputeInstanceDetailsLink(BaseGoogleLink):
|
|
44
38
|
key = "compute_instance_details"
|
45
39
|
format_str = COMPUTE_LINK
|
46
40
|
|
47
|
-
@staticmethod
|
48
|
-
def persist(
|
49
|
-
context: Context,
|
50
|
-
task_instance: BaseOperator,
|
51
|
-
location_id: str,
|
52
|
-
resource_id: str | None,
|
53
|
-
project_id: str | None,
|
54
|
-
):
|
55
|
-
task_instance.xcom_push(
|
56
|
-
context,
|
57
|
-
key=ComputeInstanceDetailsLink.key,
|
58
|
-
value={
|
59
|
-
"location_id": location_id,
|
60
|
-
"resource_id": resource_id,
|
61
|
-
"project_id": project_id,
|
62
|
-
},
|
63
|
-
)
|
64
|
-
|
65
41
|
|
66
42
|
class ComputeInstanceTemplateDetailsLink(BaseGoogleLink):
|
67
43
|
"""Helper class for constructing Compute Instance Template details Link."""
|
@@ -70,22 +46,6 @@ class ComputeInstanceTemplateDetailsLink(BaseGoogleLink):
|
|
70
46
|
key = "compute_instance_template_details"
|
71
47
|
format_str = COMPUTE_TEMPLATE_LINK
|
72
48
|
|
73
|
-
@staticmethod
|
74
|
-
def persist(
|
75
|
-
context: Context,
|
76
|
-
task_instance: BaseOperator,
|
77
|
-
resource_id: str | None,
|
78
|
-
project_id: str | None,
|
79
|
-
):
|
80
|
-
task_instance.xcom_push(
|
81
|
-
context,
|
82
|
-
key=ComputeInstanceTemplateDetailsLink.key,
|
83
|
-
value={
|
84
|
-
"resource_id": resource_id,
|
85
|
-
"project_id": project_id,
|
86
|
-
},
|
87
|
-
)
|
88
|
-
|
89
49
|
|
90
50
|
class ComputeInstanceGroupManagerDetailsLink(BaseGoogleLink):
|
91
51
|
"""Helper class for constructing Compute Instance Group Manager details Link."""
|
@@ -93,21 +53,3 @@ class ComputeInstanceGroupManagerDetailsLink(BaseGoogleLink):
|
|
93
53
|
name = "Compute Instance Group Manager"
|
94
54
|
key = "compute_instance_group_manager_details"
|
95
55
|
format_str = COMPUTE_GROUP_MANAGER_LINK
|
96
|
-
|
97
|
-
@staticmethod
|
98
|
-
def persist(
|
99
|
-
context: Context,
|
100
|
-
task_instance: BaseOperator,
|
101
|
-
location_id: str,
|
102
|
-
resource_id: str | None,
|
103
|
-
project_id: str | None,
|
104
|
-
):
|
105
|
-
task_instance.xcom_push(
|
106
|
-
context,
|
107
|
-
key=ComputeInstanceGroupManagerDetailsLink.key,
|
108
|
-
value={
|
109
|
-
"location_id": location_id,
|
110
|
-
"resource_id": resource_id,
|
111
|
-
"project_id": project_id,
|
112
|
-
},
|
113
|
-
)
|
@@ -17,13 +17,8 @@
|
|
17
17
|
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
from typing import TYPE_CHECKING
|
21
|
-
|
22
20
|
from airflow.providers.google.cloud.links.base import BaseGoogleLink
|
23
21
|
|
24
|
-
if TYPE_CHECKING:
|
25
|
-
from airflow.utils.context import Context
|
26
|
-
|
27
22
|
BASE_LINK = "https://console.cloud.google.com"
|
28
23
|
|
29
24
|
DLP_BASE_LINK = BASE_LINK + "/security/dlp"
|
@@ -73,20 +68,6 @@ class CloudDLPDeidentifyTemplatesListLink(BaseGoogleLink):
|
|
73
68
|
key = "cloud_dlp_deidentify_templates_list_key"
|
74
69
|
format_str = DLP_DEIDENTIFY_TEMPLATES_LIST_LINK
|
75
70
|
|
76
|
-
@staticmethod
|
77
|
-
def persist(
|
78
|
-
context: Context,
|
79
|
-
task_instance,
|
80
|
-
project_id: str,
|
81
|
-
):
|
82
|
-
task_instance.xcom_push(
|
83
|
-
context=context,
|
84
|
-
key=CloudDLPDeidentifyTemplatesListLink.key,
|
85
|
-
value={
|
86
|
-
"project_id": project_id,
|
87
|
-
},
|
88
|
-
)
|
89
|
-
|
90
71
|
|
91
72
|
class CloudDLPDeidentifyTemplateDetailsLink(BaseGoogleLink):
|
92
73
|
"""Helper class for constructing Cloud Data Loss Prevention link."""
|
@@ -95,22 +76,6 @@ class CloudDLPDeidentifyTemplateDetailsLink(BaseGoogleLink):
|
|
95
76
|
key = "cloud_dlp_deidentify_template_details_key"
|
96
77
|
format_str = DLP_DEIDENTIFY_TEMPLATE_DETAILS_LINK
|
97
78
|
|
98
|
-
@staticmethod
|
99
|
-
def persist(
|
100
|
-
context: Context,
|
101
|
-
task_instance,
|
102
|
-
project_id: str,
|
103
|
-
template_name: str,
|
104
|
-
):
|
105
|
-
task_instance.xcom_push(
|
106
|
-
context=context,
|
107
|
-
key=CloudDLPDeidentifyTemplateDetailsLink.key,
|
108
|
-
value={
|
109
|
-
"project_id": project_id,
|
110
|
-
"template_name": template_name,
|
111
|
-
},
|
112
|
-
)
|
113
|
-
|
114
79
|
|
115
80
|
class CloudDLPJobTriggersListLink(BaseGoogleLink):
|
116
81
|
"""Helper class for constructing Cloud Data Loss Prevention link."""
|
@@ -119,20 +84,6 @@ class CloudDLPJobTriggersListLink(BaseGoogleLink):
|
|
119
84
|
key = "cloud_dlp_job_triggers_list_key"
|
120
85
|
format_str = DLP_JOB_TRIGGER_LIST_LINK
|
121
86
|
|
122
|
-
@staticmethod
|
123
|
-
def persist(
|
124
|
-
context: Context,
|
125
|
-
task_instance,
|
126
|
-
project_id: str,
|
127
|
-
):
|
128
|
-
task_instance.xcom_push(
|
129
|
-
context=context,
|
130
|
-
key=CloudDLPJobTriggersListLink.key,
|
131
|
-
value={
|
132
|
-
"project_id": project_id,
|
133
|
-
},
|
134
|
-
)
|
135
|
-
|
136
87
|
|
137
88
|
class CloudDLPJobTriggerDetailsLink(BaseGoogleLink):
|
138
89
|
"""Helper class for constructing Cloud Data Loss Prevention link."""
|
@@ -141,22 +92,6 @@ class CloudDLPJobTriggerDetailsLink(BaseGoogleLink):
|
|
141
92
|
key = "cloud_dlp_job_trigger_details_key"
|
142
93
|
format_str = DLP_JOB_TRIGGER_DETAILS_LINK
|
143
94
|
|
144
|
-
@staticmethod
|
145
|
-
def persist(
|
146
|
-
context: Context,
|
147
|
-
task_instance,
|
148
|
-
project_id: str,
|
149
|
-
trigger_name: str,
|
150
|
-
):
|
151
|
-
task_instance.xcom_push(
|
152
|
-
context=context,
|
153
|
-
key=CloudDLPJobTriggerDetailsLink.key,
|
154
|
-
value={
|
155
|
-
"project_id": project_id,
|
156
|
-
"trigger_name": trigger_name,
|
157
|
-
},
|
158
|
-
)
|
159
|
-
|
160
95
|
|
161
96
|
class CloudDLPJobsListLink(BaseGoogleLink):
|
162
97
|
"""Helper class for constructing Cloud Data Loss Prevention link."""
|
@@ -165,20 +100,6 @@ class CloudDLPJobsListLink(BaseGoogleLink):
|
|
165
100
|
key = "cloud_dlp_jobs_list_key"
|
166
101
|
format_str = DLP_JOBS_LIST_LINK
|
167
102
|
|
168
|
-
@staticmethod
|
169
|
-
def persist(
|
170
|
-
context: Context,
|
171
|
-
task_instance,
|
172
|
-
project_id: str,
|
173
|
-
):
|
174
|
-
task_instance.xcom_push(
|
175
|
-
context=context,
|
176
|
-
key=CloudDLPJobsListLink.key,
|
177
|
-
value={
|
178
|
-
"project_id": project_id,
|
179
|
-
},
|
180
|
-
)
|
181
|
-
|
182
103
|
|
183
104
|
class CloudDLPJobDetailsLink(BaseGoogleLink):
|
184
105
|
"""Helper class for constructing Cloud Data Loss Prevention link."""
|
@@ -187,22 +108,6 @@ class CloudDLPJobDetailsLink(BaseGoogleLink):
|
|
187
108
|
key = "cloud_dlp_job_details_key"
|
188
109
|
format_str = DLP_JOB_DETAILS_LINK
|
189
110
|
|
190
|
-
@staticmethod
|
191
|
-
def persist(
|
192
|
-
context: Context,
|
193
|
-
task_instance,
|
194
|
-
project_id: str,
|
195
|
-
job_name: str,
|
196
|
-
):
|
197
|
-
task_instance.xcom_push(
|
198
|
-
context=context,
|
199
|
-
key=CloudDLPJobDetailsLink.key,
|
200
|
-
value={
|
201
|
-
"project_id": project_id,
|
202
|
-
"job_name": job_name,
|
203
|
-
},
|
204
|
-
)
|
205
|
-
|
206
111
|
|
207
112
|
class CloudDLPInspectTemplatesListLink(BaseGoogleLink):
|
208
113
|
"""Helper class for constructing Cloud Data Loss Prevention link."""
|
@@ -211,20 +116,6 @@ class CloudDLPInspectTemplatesListLink(BaseGoogleLink):
|
|
211
116
|
key = "cloud_dlp_inspect_templates_list_key"
|
212
117
|
format_str = DLP_INSPECT_TEMPLATES_LIST_LINK
|
213
118
|
|
214
|
-
@staticmethod
|
215
|
-
def persist(
|
216
|
-
context: Context,
|
217
|
-
task_instance,
|
218
|
-
project_id: str,
|
219
|
-
):
|
220
|
-
task_instance.xcom_push(
|
221
|
-
context=context,
|
222
|
-
key=CloudDLPInspectTemplatesListLink.key,
|
223
|
-
value={
|
224
|
-
"project_id": project_id,
|
225
|
-
},
|
226
|
-
)
|
227
|
-
|
228
119
|
|
229
120
|
class CloudDLPInspectTemplateDetailsLink(BaseGoogleLink):
|
230
121
|
"""Helper class for constructing Cloud Data Loss Prevention link."""
|
@@ -233,22 +124,6 @@ class CloudDLPInspectTemplateDetailsLink(BaseGoogleLink):
|
|
233
124
|
key = "cloud_dlp_inspect_template_details_key"
|
234
125
|
format_str = DLP_INSPECT_TEMPLATE_DETAILS_LINK
|
235
126
|
|
236
|
-
@staticmethod
|
237
|
-
def persist(
|
238
|
-
context: Context,
|
239
|
-
task_instance,
|
240
|
-
project_id: str,
|
241
|
-
template_name: str,
|
242
|
-
):
|
243
|
-
task_instance.xcom_push(
|
244
|
-
context=context,
|
245
|
-
key=CloudDLPInspectTemplateDetailsLink.key,
|
246
|
-
value={
|
247
|
-
"project_id": project_id,
|
248
|
-
"template_name": template_name,
|
249
|
-
},
|
250
|
-
)
|
251
|
-
|
252
127
|
|
253
128
|
class CloudDLPInfoTypesListLink(BaseGoogleLink):
|
254
129
|
"""Helper class for constructing Cloud Data Loss Prevention link."""
|
@@ -257,20 +132,6 @@ class CloudDLPInfoTypesListLink(BaseGoogleLink):
|
|
257
132
|
key = "cloud_dlp_info_types_list_key"
|
258
133
|
format_str = DLP_INFO_TYPES_LIST_LINK
|
259
134
|
|
260
|
-
@staticmethod
|
261
|
-
def persist(
|
262
|
-
context: Context,
|
263
|
-
task_instance,
|
264
|
-
project_id: str,
|
265
|
-
):
|
266
|
-
task_instance.xcom_push(
|
267
|
-
context=context,
|
268
|
-
key=CloudDLPInfoTypesListLink.key,
|
269
|
-
value={
|
270
|
-
"project_id": project_id,
|
271
|
-
},
|
272
|
-
)
|
273
|
-
|
274
135
|
|
275
136
|
class CloudDLPInfoTypeDetailsLink(BaseGoogleLink):
|
276
137
|
"""Helper class for constructing Cloud Data Loss Prevention link."""
|
@@ -279,22 +140,6 @@ class CloudDLPInfoTypeDetailsLink(BaseGoogleLink):
|
|
279
140
|
key = "cloud_dlp_info_type_details_key"
|
280
141
|
format_str = DLP_INFO_TYPE_DETAILS_LINK
|
281
142
|
|
282
|
-
@staticmethod
|
283
|
-
def persist(
|
284
|
-
context: Context,
|
285
|
-
task_instance,
|
286
|
-
project_id: str,
|
287
|
-
info_type_name: str,
|
288
|
-
):
|
289
|
-
task_instance.xcom_push(
|
290
|
-
context=context,
|
291
|
-
key=CloudDLPInfoTypeDetailsLink.key,
|
292
|
-
value={
|
293
|
-
"project_id": project_id,
|
294
|
-
"info_type_name": info_type_name,
|
295
|
-
},
|
296
|
-
)
|
297
|
-
|
298
143
|
|
299
144
|
class CloudDLPPossibleInfoTypesListLink(BaseGoogleLink):
|
300
145
|
"""Helper class for constructing Cloud Data Loss Prevention link."""
|
@@ -302,17 +147,3 @@ class CloudDLPPossibleInfoTypesListLink(BaseGoogleLink):
|
|
302
147
|
name = "Cloud DLP Possible Info Types List"
|
303
148
|
key = "cloud_dlp_possible_info_types_list_key"
|
304
149
|
format_str = DLP_POSSIBLE_INFO_TYPES_LIST_LINK
|
305
|
-
|
306
|
-
@staticmethod
|
307
|
-
def persist(
|
308
|
-
context: Context,
|
309
|
-
task_instance,
|
310
|
-
project_id: str,
|
311
|
-
):
|
312
|
-
task_instance.xcom_push(
|
313
|
-
context=context,
|
314
|
-
key=CloudDLPPossibleInfoTypesListLink.key,
|
315
|
-
value={
|
316
|
-
"project_id": project_id,
|
317
|
-
},
|
318
|
-
)
|
@@ -19,13 +19,9 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from
|
23
|
-
|
22
|
+
from airflow.exceptions import AirflowProviderDeprecationWarning
|
24
23
|
from airflow.providers.google.cloud.links.base import BaseGoogleLink
|
25
|
-
|
26
|
-
if TYPE_CHECKING:
|
27
|
-
from airflow.models import BaseOperator
|
28
|
-
from airflow.utils.context import Context
|
24
|
+
from airflow.providers.google.common.deprecated import deprecated
|
29
25
|
|
30
26
|
DATACATALOG_BASE_LINK = "/datacatalog"
|
31
27
|
ENTRY_GROUP_LINK = (
|
@@ -43,6 +39,13 @@ TAG_TEMPLATE_LINK = (
|
|
43
39
|
)
|
44
40
|
|
45
41
|
|
42
|
+
@deprecated(
|
43
|
+
planned_removal_date="January 30, 2026",
|
44
|
+
use_instead="airflow.providers.google.cloud.links.dataplex.DataplexCatalogEntryGroupLink",
|
45
|
+
reason="The Data Catalog will be discontinued on January 30, 2026 "
|
46
|
+
"in favor of Dataplex Universal Catalog.",
|
47
|
+
category=AirflowProviderDeprecationWarning,
|
48
|
+
)
|
46
49
|
class DataCatalogEntryGroupLink(BaseGoogleLink):
|
47
50
|
"""Helper class for constructing Data Catalog Entry Group Link."""
|
48
51
|
|
@@ -50,21 +53,14 @@ class DataCatalogEntryGroupLink(BaseGoogleLink):
|
|
50
53
|
key = "data_catalog_entry_group"
|
51
54
|
format_str = ENTRY_GROUP_LINK
|
52
55
|
|
53
|
-
@staticmethod
|
54
|
-
def persist(
|
55
|
-
context: Context,
|
56
|
-
task_instance: BaseOperator,
|
57
|
-
entry_group_id: str,
|
58
|
-
location_id: str,
|
59
|
-
project_id: str | None,
|
60
|
-
):
|
61
|
-
task_instance.xcom_push(
|
62
|
-
context,
|
63
|
-
key=DataCatalogEntryGroupLink.key,
|
64
|
-
value={"entry_group_id": entry_group_id, "location_id": location_id, "project_id": project_id},
|
65
|
-
)
|
66
|
-
|
67
56
|
|
57
|
+
@deprecated(
|
58
|
+
planned_removal_date="January 30, 2026",
|
59
|
+
use_instead="airflow.providers.google.cloud.links.dataplex.DataplexCatalogEntryLink",
|
60
|
+
reason="The Data Catalog will be discontinued on January 30, 2026 "
|
61
|
+
"in favor of Dataplex Universal Catalog.",
|
62
|
+
category=AirflowProviderDeprecationWarning,
|
63
|
+
)
|
68
64
|
class DataCatalogEntryLink(BaseGoogleLink):
|
69
65
|
"""Helper class for constructing Data Catalog Entry Link."""
|
70
66
|
|
@@ -72,44 +68,17 @@ class DataCatalogEntryLink(BaseGoogleLink):
|
|
72
68
|
key = "data_catalog_entry"
|
73
69
|
format_str = ENTRY_LINK
|
74
70
|
|
75
|
-
@staticmethod
|
76
|
-
def persist(
|
77
|
-
context: Context,
|
78
|
-
task_instance: BaseOperator,
|
79
|
-
entry_id: str,
|
80
|
-
entry_group_id: str,
|
81
|
-
location_id: str,
|
82
|
-
project_id: str | None,
|
83
|
-
):
|
84
|
-
task_instance.xcom_push(
|
85
|
-
context,
|
86
|
-
key=DataCatalogEntryLink.key,
|
87
|
-
value={
|
88
|
-
"entry_id": entry_id,
|
89
|
-
"entry_group_id": entry_group_id,
|
90
|
-
"location_id": location_id,
|
91
|
-
"project_id": project_id,
|
92
|
-
},
|
93
|
-
)
|
94
|
-
|
95
71
|
|
72
|
+
@deprecated(
|
73
|
+
planned_removal_date="January 30, 2026",
|
74
|
+
use_instead="airflow.providers.google.cloud.links.dataplex.DataplexCatalogAspectTypeLink",
|
75
|
+
reason="The Data Catalog will be discontinued on January 30, 2026 "
|
76
|
+
"in favor of Dataplex Universal Catalog.",
|
77
|
+
category=AirflowProviderDeprecationWarning,
|
78
|
+
)
|
96
79
|
class DataCatalogTagTemplateLink(BaseGoogleLink):
|
97
80
|
"""Helper class for constructing Data Catalog Tag Template Link."""
|
98
81
|
|
99
82
|
name = "Data Catalog Tag Template"
|
100
83
|
key = "data_catalog_tag_template"
|
101
84
|
format_str = TAG_TEMPLATE_LINK
|
102
|
-
|
103
|
-
@staticmethod
|
104
|
-
def persist(
|
105
|
-
context: Context,
|
106
|
-
task_instance: BaseOperator,
|
107
|
-
tag_template_id: str,
|
108
|
-
location_id: str,
|
109
|
-
project_id: str | None,
|
110
|
-
):
|
111
|
-
task_instance.xcom_push(
|
112
|
-
context,
|
113
|
-
key=DataCatalogTagTemplateLink.key,
|
114
|
-
value={"tag_template_id": tag_template_id, "location_id": location_id, "project_id": project_id},
|
115
|
-
)
|
@@ -19,14 +19,8 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from typing import TYPE_CHECKING
|
23
|
-
|
24
22
|
from airflow.providers.google.cloud.links.base import BaseGoogleLink
|
25
23
|
|
26
|
-
if TYPE_CHECKING:
|
27
|
-
from airflow.models import BaseOperator
|
28
|
-
from airflow.utils.context import Context
|
29
|
-
|
30
24
|
DATAFLOW_BASE_LINK = "/dataflow/jobs"
|
31
25
|
DATAFLOW_JOB_LINK = DATAFLOW_BASE_LINK + "/{region}/{job_id}?project={project_id}"
|
32
26
|
|
@@ -41,20 +35,6 @@ class DataflowJobLink(BaseGoogleLink):
|
|
41
35
|
key = "dataflow_job_config"
|
42
36
|
format_str = DATAFLOW_JOB_LINK
|
43
37
|
|
44
|
-
@staticmethod
|
45
|
-
def persist(
|
46
|
-
operator_instance: BaseOperator,
|
47
|
-
context: Context,
|
48
|
-
project_id: str | None,
|
49
|
-
region: str | None,
|
50
|
-
job_id: str | None,
|
51
|
-
):
|
52
|
-
operator_instance.xcom_push(
|
53
|
-
context,
|
54
|
-
key=DataflowJobLink.key,
|
55
|
-
value={"project_id": project_id, "region": region, "job_id": job_id},
|
56
|
-
)
|
57
|
-
|
58
38
|
|
59
39
|
class DataflowPipelineLink(BaseGoogleLink):
|
60
40
|
"""Helper class for constructing Dataflow Pipeline Link."""
|
@@ -62,17 +42,3 @@ class DataflowPipelineLink(BaseGoogleLink):
|
|
62
42
|
name = "Dataflow Pipeline"
|
63
43
|
key = "dataflow_pipeline_config"
|
64
44
|
format_str = DATAFLOW_PIPELINE_LINK
|
65
|
-
|
66
|
-
@staticmethod
|
67
|
-
def persist(
|
68
|
-
operator_instance: BaseOperator,
|
69
|
-
context: Context,
|
70
|
-
project_id: str | None,
|
71
|
-
location: str | None,
|
72
|
-
pipeline_name: str | None,
|
73
|
-
):
|
74
|
-
operator_instance.xcom_push(
|
75
|
-
context,
|
76
|
-
key=DataflowPipelineLink.key,
|
77
|
-
value={"project_id": project_id, "location": location, "pipeline_name": pipeline_name},
|
78
|
-
)
|