apache-airflow-providers-google 16.0.0rc1__py3-none-any.whl → 16.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/ads/hooks/ads.py +9 -5
- airflow/providers/google/ads/operators/ads.py +1 -1
- airflow/providers/google/ads/transfers/ads_to_gcs.py +1 -1
- airflow/providers/google/cloud/hooks/bigquery.py +2 -3
- airflow/providers/google/cloud/hooks/cloud_sql.py +8 -4
- airflow/providers/google/cloud/hooks/datacatalog.py +9 -1
- airflow/providers/google/cloud/hooks/dataflow.py +2 -2
- airflow/providers/google/cloud/hooks/dataplex.py +1 -1
- airflow/providers/google/cloud/hooks/dataprep.py +4 -1
- airflow/providers/google/cloud/hooks/gcs.py +2 -2
- airflow/providers/google/cloud/hooks/looker.py +5 -1
- airflow/providers/google/cloud/hooks/mlengine.py +2 -1
- airflow/providers/google/cloud/hooks/secret_manager.py +102 -10
- airflow/providers/google/cloud/hooks/spanner.py +2 -2
- airflow/providers/google/cloud/hooks/translate.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +307 -7
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +43 -14
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +11 -2
- airflow/providers/google/cloud/hooks/vision.py +2 -2
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +75 -11
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/cloud_run.py +1 -33
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +16 -43
- airflow/providers/google/cloud/links/cloud_tasks.py +6 -25
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -96
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +14 -90
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +5 -59
- airflow/providers/google/cloud/links/life_sciences.py +0 -19
- airflow/providers/google/cloud/links/managed_kafka.py +0 -70
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +16 -186
- airflow/providers/google/cloud/links/vertex_ai.py +8 -224
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/operators/alloy_db.py +69 -54
- airflow/providers/google/cloud/operators/automl.py +16 -14
- airflow/providers/google/cloud/operators/bigquery.py +0 -15
- airflow/providers/google/cloud/operators/bigquery_dts.py +2 -4
- airflow/providers/google/cloud/operators/bigtable.py +35 -6
- airflow/providers/google/cloud/operators/cloud_base.py +21 -1
- airflow/providers/google/cloud/operators/cloud_build.py +74 -31
- airflow/providers/google/cloud/operators/cloud_composer.py +34 -35
- airflow/providers/google/cloud/operators/cloud_memorystore.py +68 -42
- airflow/providers/google/cloud/operators/cloud_run.py +0 -1
- airflow/providers/google/cloud/operators/cloud_sql.py +11 -15
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +0 -2
- airflow/providers/google/cloud/operators/compute.py +7 -39
- airflow/providers/google/cloud/operators/datacatalog.py +156 -20
- airflow/providers/google/cloud/operators/dataflow.py +37 -14
- airflow/providers/google/cloud/operators/dataform.py +14 -4
- airflow/providers/google/cloud/operators/datafusion.py +4 -12
- airflow/providers/google/cloud/operators/dataplex.py +180 -96
- airflow/providers/google/cloud/operators/dataprep.py +0 -4
- airflow/providers/google/cloud/operators/dataproc.py +10 -16
- airflow/providers/google/cloud/operators/dataproc_metastore.py +95 -87
- airflow/providers/google/cloud/operators/datastore.py +21 -5
- airflow/providers/google/cloud/operators/dlp.py +3 -26
- airflow/providers/google/cloud/operators/functions.py +15 -6
- airflow/providers/google/cloud/operators/gcs.py +0 -7
- airflow/providers/google/cloud/operators/kubernetes_engine.py +50 -7
- airflow/providers/google/cloud/operators/life_sciences.py +0 -1
- airflow/providers/google/cloud/operators/managed_kafka.py +106 -51
- airflow/providers/google/cloud/operators/mlengine.py +0 -1
- airflow/providers/google/cloud/operators/pubsub.py +2 -4
- airflow/providers/google/cloud/operators/spanner.py +0 -4
- airflow/providers/google/cloud/operators/speech_to_text.py +0 -1
- airflow/providers/google/cloud/operators/stackdriver.py +0 -8
- airflow/providers/google/cloud/operators/tasks.py +0 -11
- airflow/providers/google/cloud/operators/text_to_speech.py +0 -1
- airflow/providers/google/cloud/operators/translate.py +37 -13
- airflow/providers/google/cloud/operators/translate_speech.py +0 -1
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +31 -18
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +28 -8
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +38 -25
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +69 -7
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +42 -8
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +531 -0
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +93 -25
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +10 -8
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +56 -10
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +25 -6
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +9 -6
- airflow/providers/google/cloud/operators/workflows.py +1 -9
- airflow/providers/google/cloud/sensors/bigquery.py +1 -1
- airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -1
- airflow/providers/google/cloud/sensors/bigtable.py +15 -3
- airflow/providers/google/cloud/sensors/cloud_composer.py +6 -1
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -1
- airflow/providers/google/cloud/sensors/dataflow.py +3 -3
- airflow/providers/google/cloud/sensors/dataform.py +6 -1
- airflow/providers/google/cloud/sensors/datafusion.py +6 -1
- airflow/providers/google/cloud/sensors/dataplex.py +6 -1
- airflow/providers/google/cloud/sensors/dataprep.py +6 -1
- airflow/providers/google/cloud/sensors/dataproc.py +6 -1
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +6 -1
- airflow/providers/google/cloud/sensors/gcs.py +9 -3
- airflow/providers/google/cloud/sensors/looker.py +6 -1
- airflow/providers/google/cloud/sensors/pubsub.py +8 -3
- airflow/providers/google/cloud/sensors/tasks.py +6 -1
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +6 -1
- airflow/providers/google/cloud/sensors/workflows.py +6 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +1 -2
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +1 -2
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +0 -1
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +1 -1
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +1 -2
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/gcs_to_local.py +1 -1
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +5 -1
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +1 -1
- airflow/providers/google/cloud/transfers/http_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/local_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +1 -1
- airflow/providers/google/cloud/utils/field_validator.py +1 -2
- airflow/providers/google/common/auth_backend/google_openid.py +2 -1
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +7 -3
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/firebase/operators/firestore.py +1 -1
- airflow/providers/google/get_provider_info.py +0 -11
- airflow/providers/google/leveldb/hooks/leveldb.py +5 -1
- airflow/providers/google/leveldb/operators/leveldb.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +3 -6
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +0 -1
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +4 -4
- airflow/providers/google/marketing_platform/operators/display_video.py +6 -6
- airflow/providers/google/marketing_platform/operators/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +6 -1
- airflow/providers/google/marketing_platform/sensors/display_video.py +6 -1
- airflow/providers/google/suite/operators/sheets.py +3 -3
- airflow/providers/google/suite/sensors/drive.py +6 -1
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +1 -1
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +1 -1
- airflow/providers/google/version_compat.py +28 -0
- {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/METADATA +19 -20
- {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/RECORD +166 -166
- {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/entry_points.txt +0 -0
@@ -16,13 +16,8 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from typing import TYPE_CHECKING
|
20
|
-
|
21
19
|
from airflow.providers.google.cloud.links.base import BaseGoogleLink
|
22
20
|
|
23
|
-
if TYPE_CHECKING:
|
24
|
-
from airflow.utils.context import Context
|
25
|
-
|
26
21
|
DATASTORE_BASE_LINK = "/datastore"
|
27
22
|
DATASTORE_IMPORT_EXPORT_LINK = DATASTORE_BASE_LINK + "/import-export?project={project_id}"
|
28
23
|
DATASTORE_EXPORT_ENTITIES_LINK = "/storage/browser/{bucket_name}/{export_name}?project={project_id}"
|
@@ -36,19 +31,6 @@ class CloudDatastoreImportExportLink(BaseGoogleLink):
|
|
36
31
|
key = "import_export_conf"
|
37
32
|
format_str = DATASTORE_IMPORT_EXPORT_LINK
|
38
33
|
|
39
|
-
@staticmethod
|
40
|
-
def persist(
|
41
|
-
context: Context,
|
42
|
-
task_instance,
|
43
|
-
):
|
44
|
-
task_instance.xcom_push(
|
45
|
-
context=context,
|
46
|
-
key=CloudDatastoreImportExportLink.key,
|
47
|
-
value={
|
48
|
-
"project_id": task_instance.project_id,
|
49
|
-
},
|
50
|
-
)
|
51
|
-
|
52
34
|
|
53
35
|
class CloudDatastoreEntitiesLink(BaseGoogleLink):
|
54
36
|
"""Helper class for constructing Cloud Datastore Entities Link."""
|
@@ -56,16 +38,3 @@ class CloudDatastoreEntitiesLink(BaseGoogleLink):
|
|
56
38
|
name = "Entities"
|
57
39
|
key = "entities_conf"
|
58
40
|
format_str = DATASTORE_ENTITIES_LINK
|
59
|
-
|
60
|
-
@staticmethod
|
61
|
-
def persist(
|
62
|
-
context: Context,
|
63
|
-
task_instance,
|
64
|
-
):
|
65
|
-
task_instance.xcom_push(
|
66
|
-
context=context,
|
67
|
-
key=CloudDatastoreEntitiesLink.key,
|
68
|
-
value={
|
69
|
-
"project_id": task_instance.project_id,
|
70
|
-
},
|
71
|
-
)
|
@@ -51,19 +51,15 @@ class KubernetesEngineClusterLink(BaseGoogleLink):
|
|
51
51
|
key = "kubernetes_cluster_conf"
|
52
52
|
format_str = KUBERNETES_CLUSTER_LINK
|
53
53
|
|
54
|
-
@
|
55
|
-
def persist(context: Context,
|
54
|
+
@classmethod
|
55
|
+
def persist(cls, context: Context, **value):
|
56
|
+
cluster = value.get("cluster")
|
56
57
|
if isinstance(cluster, dict):
|
57
58
|
cluster = Cluster.from_json(json.dumps(cluster))
|
58
59
|
|
59
|
-
|
60
|
+
super().persist(
|
60
61
|
context=context,
|
61
|
-
|
62
|
-
value={
|
63
|
-
"location": task_instance.location,
|
64
|
-
"cluster_name": cluster.name, # type: ignore
|
65
|
-
"project_id": task_instance.project_id,
|
66
|
-
},
|
62
|
+
cluster_name=cluster.name,
|
67
63
|
)
|
68
64
|
|
69
65
|
|
@@ -74,23 +70,6 @@ class KubernetesEnginePodLink(BaseGoogleLink):
|
|
74
70
|
key = "kubernetes_pod_conf"
|
75
71
|
format_str = KUBERNETES_POD_LINK
|
76
72
|
|
77
|
-
@staticmethod
|
78
|
-
def persist(
|
79
|
-
context: Context,
|
80
|
-
task_instance,
|
81
|
-
):
|
82
|
-
task_instance.xcom_push(
|
83
|
-
context=context,
|
84
|
-
key=KubernetesEnginePodLink.key,
|
85
|
-
value={
|
86
|
-
"location": task_instance.location,
|
87
|
-
"cluster_name": task_instance.cluster_name,
|
88
|
-
"namespace": task_instance.pod.metadata.namespace,
|
89
|
-
"pod_name": task_instance.pod.metadata.name,
|
90
|
-
"project_id": task_instance.project_id,
|
91
|
-
},
|
92
|
-
)
|
93
|
-
|
94
73
|
|
95
74
|
class KubernetesEngineJobLink(BaseGoogleLink):
|
96
75
|
"""Helper class for constructing Kubernetes Engine Job Link."""
|
@@ -99,23 +78,6 @@ class KubernetesEngineJobLink(BaseGoogleLink):
|
|
99
78
|
key = "kubernetes_job_conf"
|
100
79
|
format_str = KUBERNETES_JOB_LINK
|
101
80
|
|
102
|
-
@staticmethod
|
103
|
-
def persist(
|
104
|
-
context: Context,
|
105
|
-
task_instance,
|
106
|
-
):
|
107
|
-
task_instance.xcom_push(
|
108
|
-
context=context,
|
109
|
-
key=KubernetesEngineJobLink.key,
|
110
|
-
value={
|
111
|
-
"location": task_instance.location,
|
112
|
-
"cluster_name": task_instance.cluster_name,
|
113
|
-
"namespace": task_instance.job.metadata.namespace,
|
114
|
-
"job_name": task_instance.job.metadata.name,
|
115
|
-
"project_id": task_instance.project_id,
|
116
|
-
},
|
117
|
-
)
|
118
|
-
|
119
81
|
|
120
82
|
class KubernetesEngineWorkloadsLink(BaseGoogleLink):
|
121
83
|
"""Helper class for constructing Kubernetes Engine Workloads Link."""
|
@@ -123,19 +85,3 @@ class KubernetesEngineWorkloadsLink(BaseGoogleLink):
|
|
123
85
|
name = "Kubernetes Workloads"
|
124
86
|
key = "kubernetes_workloads_conf"
|
125
87
|
format_str = KUBERNETES_WORKLOADS_LINK
|
126
|
-
|
127
|
-
@staticmethod
|
128
|
-
def persist(
|
129
|
-
context: Context,
|
130
|
-
task_instance,
|
131
|
-
):
|
132
|
-
task_instance.xcom_push(
|
133
|
-
context=context,
|
134
|
-
key=KubernetesEngineWorkloadsLink.key,
|
135
|
-
value={
|
136
|
-
"location": task_instance.location,
|
137
|
-
"cluster_name": task_instance.cluster_name,
|
138
|
-
"namespace": task_instance.namespace,
|
139
|
-
"project_id": task_instance.project_id,
|
140
|
-
},
|
141
|
-
)
|
@@ -16,13 +16,8 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from typing import TYPE_CHECKING
|
20
|
-
|
21
19
|
from airflow.providers.google.cloud.links.base import BaseGoogleLink
|
22
20
|
|
23
|
-
if TYPE_CHECKING:
|
24
|
-
from airflow.utils.context import Context
|
25
|
-
|
26
21
|
BASE_LINK = "https://console.cloud.google.com/lifesciences"
|
27
22
|
LIFESCIENCES_LIST_LINK = BASE_LINK + "/pipelines?project={project_id}"
|
28
23
|
|
@@ -33,17 +28,3 @@ class LifeSciencesLink(BaseGoogleLink):
|
|
33
28
|
name = "Life Sciences"
|
34
29
|
key = "lifesciences_key"
|
35
30
|
format_str = LIFESCIENCES_LIST_LINK
|
36
|
-
|
37
|
-
@staticmethod
|
38
|
-
def persist(
|
39
|
-
context: Context,
|
40
|
-
task_instance,
|
41
|
-
project_id: str,
|
42
|
-
):
|
43
|
-
task_instance.xcom_push(
|
44
|
-
context=context,
|
45
|
-
key=LifeSciencesLink.key,
|
46
|
-
value={
|
47
|
-
"project_id": project_id,
|
48
|
-
},
|
49
|
-
)
|
@@ -16,13 +16,8 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from typing import TYPE_CHECKING
|
20
|
-
|
21
19
|
from airflow.providers.google.cloud.links.base import BaseGoogleLink
|
22
20
|
|
23
|
-
if TYPE_CHECKING:
|
24
|
-
from airflow.utils.context import Context
|
25
|
-
|
26
21
|
MANAGED_KAFKA_BASE_LINK = "/managedkafka"
|
27
22
|
MANAGED_KAFKA_CLUSTER_LINK = (
|
28
23
|
MANAGED_KAFKA_BASE_LINK + "/{location}/clusters/{cluster_id}?project={project_id}"
|
@@ -44,22 +39,6 @@ class ApacheKafkaClusterLink(BaseGoogleLink):
|
|
44
39
|
key = "cluster_conf"
|
45
40
|
format_str = MANAGED_KAFKA_CLUSTER_LINK
|
46
41
|
|
47
|
-
@staticmethod
|
48
|
-
def persist(
|
49
|
-
context: Context,
|
50
|
-
task_instance,
|
51
|
-
cluster_id: str,
|
52
|
-
):
|
53
|
-
task_instance.xcom_push(
|
54
|
-
context=context,
|
55
|
-
key=ApacheKafkaClusterLink.key,
|
56
|
-
value={
|
57
|
-
"location": task_instance.location,
|
58
|
-
"cluster_id": cluster_id,
|
59
|
-
"project_id": task_instance.project_id,
|
60
|
-
},
|
61
|
-
)
|
62
|
-
|
63
42
|
|
64
43
|
class ApacheKafkaClusterListLink(BaseGoogleLink):
|
65
44
|
"""Helper class for constructing Apache Kafka Clusters link."""
|
@@ -68,19 +47,6 @@ class ApacheKafkaClusterListLink(BaseGoogleLink):
|
|
68
47
|
key = "cluster_list_conf"
|
69
48
|
format_str = MANAGED_KAFKA_CLUSTER_LIST_LINK
|
70
49
|
|
71
|
-
@staticmethod
|
72
|
-
def persist(
|
73
|
-
context: Context,
|
74
|
-
task_instance,
|
75
|
-
):
|
76
|
-
task_instance.xcom_push(
|
77
|
-
context=context,
|
78
|
-
key=ApacheKafkaClusterListLink.key,
|
79
|
-
value={
|
80
|
-
"project_id": task_instance.project_id,
|
81
|
-
},
|
82
|
-
)
|
83
|
-
|
84
50
|
|
85
51
|
class ApacheKafkaTopicLink(BaseGoogleLink):
|
86
52
|
"""Helper class for constructing Apache Kafka Topic link."""
|
@@ -89,24 +55,6 @@ class ApacheKafkaTopicLink(BaseGoogleLink):
|
|
89
55
|
key = "topic_conf"
|
90
56
|
format_str = MANAGED_KAFKA_TOPIC_LINK
|
91
57
|
|
92
|
-
@staticmethod
|
93
|
-
def persist(
|
94
|
-
context: Context,
|
95
|
-
task_instance,
|
96
|
-
cluster_id: str,
|
97
|
-
topic_id: str,
|
98
|
-
):
|
99
|
-
task_instance.xcom_push(
|
100
|
-
context=context,
|
101
|
-
key=ApacheKafkaTopicLink.key,
|
102
|
-
value={
|
103
|
-
"location": task_instance.location,
|
104
|
-
"cluster_id": cluster_id,
|
105
|
-
"topic_id": topic_id,
|
106
|
-
"project_id": task_instance.project_id,
|
107
|
-
},
|
108
|
-
)
|
109
|
-
|
110
58
|
|
111
59
|
class ApacheKafkaConsumerGroupLink(BaseGoogleLink):
|
112
60
|
"""Helper class for constructing Apache Kafka Consumer Group link."""
|
@@ -114,21 +62,3 @@ class ApacheKafkaConsumerGroupLink(BaseGoogleLink):
|
|
114
62
|
name = "Apache Kafka Consumer Group"
|
115
63
|
key = "consumer_group_conf"
|
116
64
|
format_str = MANAGED_KAFKA_CONSUMER_GROUP_LINK
|
117
|
-
|
118
|
-
@staticmethod
|
119
|
-
def persist(
|
120
|
-
context: Context,
|
121
|
-
task_instance,
|
122
|
-
cluster_id: str,
|
123
|
-
consumer_group_id: str,
|
124
|
-
):
|
125
|
-
task_instance.xcom_push(
|
126
|
-
context=context,
|
127
|
-
key=ApacheKafkaConsumerGroupLink.key,
|
128
|
-
value={
|
129
|
-
"location": task_instance.location,
|
130
|
-
"cluster_id": cluster_id,
|
131
|
-
"consumer_group_id": consumer_group_id,
|
132
|
-
"project_id": task_instance.project_id,
|
133
|
-
},
|
134
|
-
)
|
@@ -19,14 +19,8 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from typing import TYPE_CHECKING
|
23
|
-
|
24
22
|
from airflow.providers.google.cloud.links.base import BaseGoogleLink
|
25
23
|
|
26
|
-
if TYPE_CHECKING:
|
27
|
-
from airflow.utils.context import Context
|
28
|
-
|
29
|
-
|
30
24
|
MLENGINE_BASE_LINK = "https://console.cloud.google.com/ai-platform"
|
31
25
|
MLENGINE_MODEL_DETAILS_LINK = MLENGINE_BASE_LINK + "/models/{model_id}/versions?project={project_id}"
|
32
26
|
MLENGINE_MODEL_VERSION_DETAILS_LINK = (
|
@@ -44,19 +38,6 @@ class MLEngineModelLink(BaseGoogleLink):
|
|
44
38
|
key = "ml_engine_model"
|
45
39
|
format_str = MLENGINE_MODEL_DETAILS_LINK
|
46
40
|
|
47
|
-
@staticmethod
|
48
|
-
def persist(
|
49
|
-
context: Context,
|
50
|
-
task_instance,
|
51
|
-
model_id: str,
|
52
|
-
project_id: str,
|
53
|
-
):
|
54
|
-
task_instance.xcom_push(
|
55
|
-
context,
|
56
|
-
key=MLEngineModelLink.key,
|
57
|
-
value={"model_id": model_id, "project_id": project_id},
|
58
|
-
)
|
59
|
-
|
60
41
|
|
61
42
|
class MLEngineModelsListLink(BaseGoogleLink):
|
62
43
|
"""Helper class for constructing ML Engine link."""
|
@@ -65,18 +46,6 @@ class MLEngineModelsListLink(BaseGoogleLink):
|
|
65
46
|
key = "ml_engine_models_list"
|
66
47
|
format_str = MLENGINE_MODELS_LIST_LINK
|
67
48
|
|
68
|
-
@staticmethod
|
69
|
-
def persist(
|
70
|
-
context: Context,
|
71
|
-
task_instance,
|
72
|
-
project_id: str,
|
73
|
-
):
|
74
|
-
task_instance.xcom_push(
|
75
|
-
context,
|
76
|
-
key=MLEngineModelsListLink.key,
|
77
|
-
value={"project_id": project_id},
|
78
|
-
)
|
79
|
-
|
80
49
|
|
81
50
|
class MLEngineJobDetailsLink(BaseGoogleLink):
|
82
51
|
"""Helper class for constructing ML Engine link."""
|
@@ -85,19 +54,6 @@ class MLEngineJobDetailsLink(BaseGoogleLink):
|
|
85
54
|
key = "ml_engine_job_details"
|
86
55
|
format_str = MLENGINE_JOB_DETAILS_LINK
|
87
56
|
|
88
|
-
@staticmethod
|
89
|
-
def persist(
|
90
|
-
context: Context,
|
91
|
-
task_instance,
|
92
|
-
job_id: str,
|
93
|
-
project_id: str,
|
94
|
-
):
|
95
|
-
task_instance.xcom_push(
|
96
|
-
context,
|
97
|
-
key=MLEngineJobDetailsLink.key,
|
98
|
-
value={"job_id": job_id, "project_id": project_id},
|
99
|
-
)
|
100
|
-
|
101
57
|
|
102
58
|
class MLEngineModelVersionDetailsLink(BaseGoogleLink):
|
103
59
|
"""Helper class for constructing ML Engine link."""
|
@@ -106,20 +62,6 @@ class MLEngineModelVersionDetailsLink(BaseGoogleLink):
|
|
106
62
|
key = "ml_engine_version_details"
|
107
63
|
format_str = MLENGINE_MODEL_VERSION_DETAILS_LINK
|
108
64
|
|
109
|
-
@staticmethod
|
110
|
-
def persist(
|
111
|
-
context: Context,
|
112
|
-
task_instance,
|
113
|
-
model_id: str,
|
114
|
-
project_id: str,
|
115
|
-
version_id: str,
|
116
|
-
):
|
117
|
-
task_instance.xcom_push(
|
118
|
-
context,
|
119
|
-
key=MLEngineModelVersionDetailsLink.key,
|
120
|
-
value={"model_id": model_id, "project_id": project_id, "version_id": version_id},
|
121
|
-
)
|
122
|
-
|
123
65
|
|
124
66
|
class MLEngineJobSListLink(BaseGoogleLink):
|
125
67
|
"""Helper class for constructing ML Engine link."""
|
@@ -127,15 +69,3 @@ class MLEngineJobSListLink(BaseGoogleLink):
|
|
127
69
|
name = "MLEngine Jobs List"
|
128
70
|
key = "ml_engine_jobs_list"
|
129
71
|
format_str = MLENGINE_JOBS_LIST_LINK
|
130
|
-
|
131
|
-
@staticmethod
|
132
|
-
def persist(
|
133
|
-
context: Context,
|
134
|
-
task_instance,
|
135
|
-
project_id: str,
|
136
|
-
):
|
137
|
-
task_instance.xcom_push(
|
138
|
-
context,
|
139
|
-
key=MLEngineJobSListLink.key,
|
140
|
-
value={"project_id": project_id},
|
141
|
-
)
|
@@ -19,14 +19,8 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from typing import TYPE_CHECKING
|
23
|
-
|
24
22
|
from airflow.providers.google.cloud.links.base import BaseGoogleLink
|
25
23
|
|
26
|
-
if TYPE_CHECKING:
|
27
|
-
from airflow.models import BaseOperator
|
28
|
-
from airflow.utils.context import Context
|
29
|
-
|
30
24
|
PUBSUB_BASE_LINK = "/cloudpubsub"
|
31
25
|
PUBSUB_TOPIC_LINK = PUBSUB_BASE_LINK + "/topic/detail/{topic_id}?project={project_id}"
|
32
26
|
PUBSUB_SUBSCRIPTION_LINK = PUBSUB_BASE_LINK + "/subscription/detail/{subscription_id}?project={project_id}"
|
@@ -39,19 +33,6 @@ class PubSubTopicLink(BaseGoogleLink):
|
|
39
33
|
key = "pubsub_topic"
|
40
34
|
format_str = PUBSUB_TOPIC_LINK
|
41
35
|
|
42
|
-
@staticmethod
|
43
|
-
def persist(
|
44
|
-
context: Context,
|
45
|
-
task_instance: BaseOperator,
|
46
|
-
topic_id: str,
|
47
|
-
project_id: str | None,
|
48
|
-
):
|
49
|
-
task_instance.xcom_push(
|
50
|
-
context,
|
51
|
-
key=PubSubTopicLink.key,
|
52
|
-
value={"topic_id": topic_id, "project_id": project_id},
|
53
|
-
)
|
54
|
-
|
55
36
|
|
56
37
|
class PubSubSubscriptionLink(BaseGoogleLink):
|
57
38
|
"""Helper class for constructing Pub/Sub Subscription Link."""
|
@@ -59,16 +40,3 @@ class PubSubSubscriptionLink(BaseGoogleLink):
|
|
59
40
|
name = "Pub/Sub Subscription"
|
60
41
|
key = "pubsub_subscription"
|
61
42
|
format_str = PUBSUB_SUBSCRIPTION_LINK
|
62
|
-
|
63
|
-
@staticmethod
|
64
|
-
def persist(
|
65
|
-
context: Context,
|
66
|
-
task_instance: BaseOperator,
|
67
|
-
subscription_id: str | None,
|
68
|
-
project_id: str | None,
|
69
|
-
):
|
70
|
-
task_instance.xcom_push(
|
71
|
-
context,
|
72
|
-
key=PubSubSubscriptionLink.key,
|
73
|
-
value={"subscription_id": subscription_id, "project_id": project_id},
|
74
|
-
)
|
@@ -19,14 +19,8 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from typing import TYPE_CHECKING
|
23
|
-
|
24
22
|
from airflow.providers.google.cloud.links.base import BaseGoogleLink
|
25
23
|
|
26
|
-
if TYPE_CHECKING:
|
27
|
-
from airflow.models import BaseOperator
|
28
|
-
from airflow.utils.context import Context
|
29
|
-
|
30
24
|
SPANNER_BASE_LINK = "/spanner/instances"
|
31
25
|
SPANNER_INSTANCE_LINK = SPANNER_BASE_LINK + "/{instance_id}/details/databases?project={project_id}"
|
32
26
|
SPANNER_DATABASE_LINK = (
|
@@ -41,19 +35,6 @@ class SpannerInstanceLink(BaseGoogleLink):
|
|
41
35
|
key = "spanner_instance"
|
42
36
|
format_str = SPANNER_INSTANCE_LINK
|
43
37
|
|
44
|
-
@staticmethod
|
45
|
-
def persist(
|
46
|
-
context: Context,
|
47
|
-
task_instance: BaseOperator,
|
48
|
-
instance_id: str,
|
49
|
-
project_id: str | None,
|
50
|
-
):
|
51
|
-
task_instance.xcom_push(
|
52
|
-
context,
|
53
|
-
key=SpannerInstanceLink.key,
|
54
|
-
value={"instance_id": instance_id, "project_id": project_id},
|
55
|
-
)
|
56
|
-
|
57
38
|
|
58
39
|
class SpannerDatabaseLink(BaseGoogleLink):
|
59
40
|
"""Helper class for constructing Spanner Database Link."""
|
@@ -61,17 +42,3 @@ class SpannerDatabaseLink(BaseGoogleLink):
|
|
61
42
|
name = "Spanner Database"
|
62
43
|
key = "spanner_database"
|
63
44
|
format_str = SPANNER_DATABASE_LINK
|
64
|
-
|
65
|
-
@staticmethod
|
66
|
-
def persist(
|
67
|
-
context: Context,
|
68
|
-
task_instance: BaseOperator,
|
69
|
-
instance_id: str,
|
70
|
-
database_id: str,
|
71
|
-
project_id: str | None,
|
72
|
-
):
|
73
|
-
task_instance.xcom_push(
|
74
|
-
context,
|
75
|
-
key=SpannerDatabaseLink.key,
|
76
|
-
value={"instance_id": instance_id, "database_id": database_id, "project_id": project_id},
|
77
|
-
)
|
@@ -19,14 +19,8 @@
|
|
19
19
|
|
20
20
|
from __future__ import annotations
|
21
21
|
|
22
|
-
from typing import TYPE_CHECKING
|
23
|
-
|
24
22
|
from airflow.providers.google.cloud.links.base import BaseGoogleLink
|
25
23
|
|
26
|
-
if TYPE_CHECKING:
|
27
|
-
from airflow.models import BaseOperator
|
28
|
-
from airflow.utils.context import Context
|
29
|
-
|
30
24
|
STACKDRIVER_BASE_LINK = "/monitoring/alerting"
|
31
25
|
STACKDRIVER_NOTIFICATIONS_LINK = STACKDRIVER_BASE_LINK + "/notifications?project={project_id}"
|
32
26
|
STACKDRIVER_POLICIES_LINK = STACKDRIVER_BASE_LINK + "/policies?project={project_id}"
|
@@ -39,18 +33,6 @@ class StackdriverNotificationsLink(BaseGoogleLink):
|
|
39
33
|
key = "stackdriver_notifications"
|
40
34
|
format_str = STACKDRIVER_NOTIFICATIONS_LINK
|
41
35
|
|
42
|
-
@staticmethod
|
43
|
-
def persist(
|
44
|
-
operator_instance: BaseOperator,
|
45
|
-
context: Context,
|
46
|
-
project_id: str | None,
|
47
|
-
):
|
48
|
-
operator_instance.xcom_push(
|
49
|
-
context,
|
50
|
-
key=StackdriverNotificationsLink.key,
|
51
|
-
value={"project_id": project_id},
|
52
|
-
)
|
53
|
-
|
54
36
|
|
55
37
|
class StackdriverPoliciesLink(BaseGoogleLink):
|
56
38
|
"""Helper class for constructing Stackdriver Policies Link."""
|
@@ -58,15 +40,3 @@ class StackdriverPoliciesLink(BaseGoogleLink):
|
|
58
40
|
name = "Cloud Monitoring Policies"
|
59
41
|
key = "stackdriver_policies"
|
60
42
|
format_str = STACKDRIVER_POLICIES_LINK
|
61
|
-
|
62
|
-
@staticmethod
|
63
|
-
def persist(
|
64
|
-
operator_instance: BaseOperator,
|
65
|
-
context: Context,
|
66
|
-
project_id: str | None,
|
67
|
-
):
|
68
|
-
operator_instance.xcom_push(
|
69
|
-
context,
|
70
|
-
key=StackdriverPoliciesLink.key,
|
71
|
-
value={"project_id": project_id},
|
72
|
-
)
|