apache-airflow-providers-google 15.1.0rc1__py3-none-any.whl → 19.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/3rd-party-licenses/NOTICE +2 -12
- airflow/providers/google/__init__.py +3 -3
- airflow/providers/google/ads/hooks/ads.py +39 -6
- airflow/providers/google/ads/operators/ads.py +2 -2
- airflow/providers/google/ads/transfers/ads_to_gcs.py +2 -2
- airflow/providers/google/assets/gcs.py +1 -11
- airflow/providers/google/cloud/bundles/__init__.py +16 -0
- airflow/providers/google/cloud/bundles/gcs.py +161 -0
- airflow/providers/google/cloud/hooks/alloy_db.py +1 -1
- airflow/providers/google/cloud/hooks/bigquery.py +176 -293
- airflow/providers/google/cloud/hooks/cloud_batch.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_build.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_composer.py +288 -15
- airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_run.py +18 -10
- airflow/providers/google/cloud/hooks/cloud_sql.py +102 -23
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +29 -7
- airflow/providers/google/cloud/hooks/compute.py +1 -1
- airflow/providers/google/cloud/hooks/compute_ssh.py +6 -2
- airflow/providers/google/cloud/hooks/datacatalog.py +10 -1
- airflow/providers/google/cloud/hooks/dataflow.py +72 -95
- airflow/providers/google/cloud/hooks/dataform.py +1 -1
- airflow/providers/google/cloud/hooks/datafusion.py +21 -19
- airflow/providers/google/cloud/hooks/dataplex.py +2 -2
- airflow/providers/google/cloud/hooks/dataprep.py +1 -1
- airflow/providers/google/cloud/hooks/dataproc.py +73 -72
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +1 -1
- airflow/providers/google/cloud/hooks/dlp.py +1 -1
- airflow/providers/google/cloud/hooks/functions.py +1 -1
- airflow/providers/google/cloud/hooks/gcs.py +112 -15
- airflow/providers/google/cloud/hooks/gdm.py +1 -1
- airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +3 -3
- airflow/providers/google/cloud/hooks/looker.py +6 -2
- airflow/providers/google/cloud/hooks/managed_kafka.py +1 -1
- airflow/providers/google/cloud/hooks/mlengine.py +4 -3
- airflow/providers/google/cloud/hooks/pubsub.py +3 -0
- airflow/providers/google/cloud/hooks/secret_manager.py +102 -10
- airflow/providers/google/cloud/hooks/spanner.py +74 -9
- airflow/providers/google/cloud/hooks/stackdriver.py +11 -9
- airflow/providers/google/cloud/hooks/tasks.py +1 -1
- airflow/providers/google/cloud/hooks/translate.py +2 -2
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +2 -210
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +3 -3
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +28 -2
- airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +308 -8
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
- airflow/providers/google/cloud/hooks/vision.py +3 -3
- airflow/providers/google/cloud/hooks/workflows.py +1 -1
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +77 -13
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -44
- airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -96
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +11 -95
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +9 -60
- airflow/providers/google/cloud/links/managed_kafka.py +0 -70
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +17 -187
- airflow/providers/google/cloud/links/vertex_ai.py +28 -195
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/log/gcs_task_handler.py +58 -22
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +9 -6
- airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
- airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
- airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
- airflow/providers/google/cloud/openlineage/facets.py +102 -1
- airflow/providers/google/cloud/openlineage/mixins.py +10 -8
- airflow/providers/google/cloud/openlineage/utils.py +15 -1
- airflow/providers/google/cloud/operators/alloy_db.py +71 -56
- airflow/providers/google/cloud/operators/bigquery.py +73 -636
- airflow/providers/google/cloud/operators/bigquery_dts.py +4 -6
- airflow/providers/google/cloud/operators/bigtable.py +37 -8
- airflow/providers/google/cloud/operators/cloud_base.py +21 -1
- airflow/providers/google/cloud/operators/cloud_batch.py +3 -3
- airflow/providers/google/cloud/operators/cloud_build.py +76 -33
- airflow/providers/google/cloud/operators/cloud_composer.py +129 -41
- airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
- airflow/providers/google/cloud/operators/cloud_memorystore.py +69 -43
- airflow/providers/google/cloud/operators/cloud_run.py +24 -6
- airflow/providers/google/cloud/operators/cloud_sql.py +8 -17
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +93 -12
- airflow/providers/google/cloud/operators/compute.py +9 -41
- airflow/providers/google/cloud/operators/datacatalog.py +157 -21
- airflow/providers/google/cloud/operators/dataflow.py +40 -16
- airflow/providers/google/cloud/operators/dataform.py +15 -5
- airflow/providers/google/cloud/operators/datafusion.py +42 -21
- airflow/providers/google/cloud/operators/dataplex.py +194 -110
- airflow/providers/google/cloud/operators/dataprep.py +1 -5
- airflow/providers/google/cloud/operators/dataproc.py +80 -36
- airflow/providers/google/cloud/operators/dataproc_metastore.py +97 -89
- airflow/providers/google/cloud/operators/datastore.py +23 -7
- airflow/providers/google/cloud/operators/dlp.py +6 -29
- airflow/providers/google/cloud/operators/functions.py +17 -8
- airflow/providers/google/cloud/operators/gcs.py +12 -9
- airflow/providers/google/cloud/operators/gen_ai.py +389 -0
- airflow/providers/google/cloud/operators/kubernetes_engine.py +62 -100
- airflow/providers/google/cloud/operators/looker.py +2 -2
- airflow/providers/google/cloud/operators/managed_kafka.py +108 -53
- airflow/providers/google/cloud/operators/natural_language.py +1 -1
- airflow/providers/google/cloud/operators/pubsub.py +68 -15
- airflow/providers/google/cloud/operators/spanner.py +26 -13
- airflow/providers/google/cloud/operators/speech_to_text.py +2 -3
- airflow/providers/google/cloud/operators/stackdriver.py +1 -9
- airflow/providers/google/cloud/operators/tasks.py +1 -12
- airflow/providers/google/cloud/operators/text_to_speech.py +2 -3
- airflow/providers/google/cloud/operators/translate.py +41 -17
- airflow/providers/google/cloud/operators/translate_speech.py +2 -3
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +39 -19
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +30 -10
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +55 -27
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +70 -8
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +43 -9
- airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -115
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +12 -10
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +57 -11
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +31 -8
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
- airflow/providers/google/cloud/operators/video_intelligence.py +1 -1
- airflow/providers/google/cloud/operators/vision.py +2 -2
- airflow/providers/google/cloud/operators/workflows.py +18 -15
- airflow/providers/google/cloud/secrets/secret_manager.py +3 -2
- airflow/providers/google/cloud/sensors/bigquery.py +3 -3
- airflow/providers/google/cloud/sensors/bigquery_dts.py +2 -3
- airflow/providers/google/cloud/sensors/bigtable.py +11 -4
- airflow/providers/google/cloud/sensors/cloud_composer.py +533 -30
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +2 -3
- airflow/providers/google/cloud/sensors/dataflow.py +26 -10
- airflow/providers/google/cloud/sensors/dataform.py +2 -3
- airflow/providers/google/cloud/sensors/datafusion.py +4 -5
- airflow/providers/google/cloud/sensors/dataplex.py +2 -3
- airflow/providers/google/cloud/sensors/dataprep.py +2 -2
- airflow/providers/google/cloud/sensors/dataproc.py +2 -3
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +2 -3
- airflow/providers/google/cloud/sensors/gcs.py +4 -5
- airflow/providers/google/cloud/sensors/looker.py +2 -3
- airflow/providers/google/cloud/sensors/pubsub.py +4 -5
- airflow/providers/google/cloud/sensors/tasks.py +2 -2
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -3
- airflow/providers/google/cloud/sensors/workflows.py +2 -3
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +4 -3
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +10 -5
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
- airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -4
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +21 -13
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +4 -3
- airflow/providers/google/cloud/transfers/gcs_to_local.py +6 -4
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +11 -5
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
- airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
- airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +42 -9
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +13 -7
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +14 -5
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
- airflow/providers/google/cloud/triggers/bigquery.py +76 -35
- airflow/providers/google/cloud/triggers/cloud_build.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_composer.py +303 -47
- airflow/providers/google/cloud/triggers/cloud_run.py +3 -3
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +92 -2
- airflow/providers/google/cloud/triggers/dataflow.py +122 -0
- airflow/providers/google/cloud/triggers/datafusion.py +1 -1
- airflow/providers/google/cloud/triggers/dataplex.py +14 -2
- airflow/providers/google/cloud/triggers/dataproc.py +123 -53
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +47 -28
- airflow/providers/google/cloud/triggers/mlengine.py +1 -1
- airflow/providers/google/cloud/triggers/pubsub.py +15 -19
- airflow/providers/google/cloud/triggers/vertex_ai.py +1 -1
- airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
- airflow/providers/google/cloud/utils/credentials_provider.py +2 -2
- airflow/providers/google/cloud/utils/field_sanitizer.py +1 -1
- airflow/providers/google/cloud/utils/field_validator.py +2 -3
- airflow/providers/google/common/auth_backend/google_openid.py +4 -4
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +27 -9
- airflow/providers/google/common/hooks/operation_helpers.py +1 -1
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/common/utils/get_secret.py +31 -0
- airflow/providers/google/common/utils/id_token_credentials.py +3 -4
- airflow/providers/google/firebase/hooks/firestore.py +1 -1
- airflow/providers/google/firebase/operators/firestore.py +3 -3
- airflow/providers/google/get_provider_info.py +56 -52
- airflow/providers/google/go_module_utils.py +35 -3
- airflow/providers/google/leveldb/hooks/leveldb.py +27 -2
- airflow/providers/google/leveldb/operators/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/campaign_manager.py +1 -1
- airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
- airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +5 -14
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +2 -3
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +6 -6
- airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
- airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
- airflow/providers/google/marketing_platform/sensors/display_video.py +3 -64
- airflow/providers/google/suite/hooks/calendar.py +2 -2
- airflow/providers/google/suite/hooks/sheets.py +16 -2
- airflow/providers/google/suite/operators/sheets.py +8 -3
- airflow/providers/google/suite/sensors/drive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +3 -3
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
- airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
- airflow/providers/google/version_compat.py +15 -1
- {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.3.0.dist-info}/METADATA +90 -46
- apache_airflow_providers_google-19.3.0.dist-info/RECORD +331 -0
- apache_airflow_providers_google-19.3.0.dist-info/licenses/NOTICE +5 -0
- airflow/providers/google/cloud/hooks/automl.py +0 -673
- airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
- airflow/providers/google/cloud/links/automl.py +0 -193
- airflow/providers/google/cloud/operators/automl.py +0 -1362
- airflow/providers/google/cloud/operators/life_sciences.py +0 -119
- airflow/providers/google/cloud/operators/mlengine.py +0 -112
- apache_airflow_providers_google-15.1.0rc1.dist-info/RECORD +0 -321
- {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.3.0.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.3.0.dist-info}/entry_points.txt +0 -0
- {airflow/providers/google → apache_airflow_providers_google-19.3.0.dist-info/licenses}/LICENSE +0 -0
|
@@ -29,7 +29,7 @@ from kubernetes.client import V1JobList, models as k8s
|
|
|
29
29
|
from packaging.version import parse as parse_version
|
|
30
30
|
|
|
31
31
|
from airflow.configuration import conf
|
|
32
|
-
from airflow.exceptions import
|
|
32
|
+
from airflow.exceptions import AirflowProviderDeprecationWarning
|
|
33
33
|
from airflow.providers.cncf.kubernetes.operators.job import KubernetesJobOperator
|
|
34
34
|
from airflow.providers.cncf.kubernetes.operators.kueue import (
|
|
35
35
|
KubernetesInstallKueueOperator,
|
|
@@ -41,6 +41,7 @@ from airflow.providers.cncf.kubernetes.operators.resource import (
|
|
|
41
41
|
KubernetesDeleteResourceOperator,
|
|
42
42
|
)
|
|
43
43
|
from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction
|
|
44
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
44
45
|
from airflow.providers.google.cloud.hooks.kubernetes_engine import (
|
|
45
46
|
GKEHook,
|
|
46
47
|
GKEKubernetesHook,
|
|
@@ -57,7 +58,6 @@ from airflow.providers.google.cloud.triggers.kubernetes_engine import (
|
|
|
57
58
|
GKEOperationTrigger,
|
|
58
59
|
GKEStartPodTrigger,
|
|
59
60
|
)
|
|
60
|
-
from airflow.providers.google.common.deprecated import deprecated
|
|
61
61
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
62
62
|
from airflow.providers_manager import ProvidersManager
|
|
63
63
|
from airflow.utils.timezone import utcnow
|
|
@@ -77,7 +77,7 @@ if TYPE_CHECKING:
|
|
|
77
77
|
from kubernetes.client.models import V1Job
|
|
78
78
|
from pendulum import DateTime
|
|
79
79
|
|
|
80
|
-
from airflow.
|
|
80
|
+
from airflow.providers.common.compat.sdk import Context
|
|
81
81
|
|
|
82
82
|
KUBE_CONFIG_ENV_VAR = "KUBECONFIG"
|
|
83
83
|
|
|
@@ -222,7 +222,6 @@ class GKEDeleteClusterOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
|
222
222
|
If set as a sequence, the identities from the list must grant
|
|
223
223
|
Service Account Token Creator IAM role to the directly preceding identity, with first
|
|
224
224
|
account from the list granting this role to the originating account (templated).
|
|
225
|
-
:param name: (Deprecated) The name of the resource to delete, in this case cluster name
|
|
226
225
|
:param api_version: The api version to use
|
|
227
226
|
:param deferrable: Run operator in the deferrable mode.
|
|
228
227
|
:param poll_interval: Interval size which defines how often operation status is checked.
|
|
@@ -241,7 +240,6 @@ class GKEDeleteClusterOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
|
241
240
|
gcp_conn_id: str = "google_cloud_default",
|
|
242
241
|
impersonation_chain: str | Sequence[str] | None = None,
|
|
243
242
|
cluster_name: str | None = None,
|
|
244
|
-
name: str | None = None,
|
|
245
243
|
api_version: str = "v2",
|
|
246
244
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
|
247
245
|
poll_interval: int = 10,
|
|
@@ -251,36 +249,17 @@ class GKEDeleteClusterOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
|
251
249
|
super().__init__(*args, **kwargs)
|
|
252
250
|
|
|
253
251
|
self.location = location
|
|
254
|
-
self.cluster_name = cluster_name
|
|
252
|
+
self.cluster_name = cluster_name
|
|
255
253
|
self.use_internal_ip = use_internal_ip
|
|
256
254
|
self.use_dns_endpoint = use_dns_endpoint
|
|
257
255
|
self.project_id = project_id
|
|
258
256
|
self.gcp_conn_id = gcp_conn_id
|
|
259
257
|
self.impersonation_chain = impersonation_chain
|
|
260
|
-
self._name = name
|
|
261
258
|
self.api_version = api_version
|
|
262
259
|
self.deferrable = deferrable
|
|
263
260
|
self.poll_interval = poll_interval
|
|
264
261
|
self._check_input()
|
|
265
262
|
|
|
266
|
-
@property
|
|
267
|
-
@deprecated(
|
|
268
|
-
planned_removal_date="May 01, 2025",
|
|
269
|
-
use_instead="cluster_name",
|
|
270
|
-
category=AirflowProviderDeprecationWarning,
|
|
271
|
-
)
|
|
272
|
-
def name(self) -> str | None:
|
|
273
|
-
return self._name
|
|
274
|
-
|
|
275
|
-
@name.setter
|
|
276
|
-
@deprecated(
|
|
277
|
-
planned_removal_date="May 01, 2025",
|
|
278
|
-
use_instead="cluster_name",
|
|
279
|
-
category=AirflowProviderDeprecationWarning,
|
|
280
|
-
)
|
|
281
|
-
def name(self, name: str) -> None:
|
|
282
|
-
self._name = name
|
|
283
|
-
|
|
284
263
|
def _check_input(self) -> None:
|
|
285
264
|
if not all([self.project_id, self.cluster_name, self.location]):
|
|
286
265
|
self.log.error("One of (project_id, cluster_name, location) is missing or incorrect")
|
|
@@ -475,8 +454,15 @@ class GKECreateClusterOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
|
475
454
|
stacklevel=2,
|
|
476
455
|
)
|
|
477
456
|
|
|
457
|
+
@property
|
|
458
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
459
|
+
return {
|
|
460
|
+
"project_id": self.project_id,
|
|
461
|
+
"location": self.location,
|
|
462
|
+
}
|
|
463
|
+
|
|
478
464
|
def execute(self, context: Context) -> str:
|
|
479
|
-
KubernetesEngineClusterLink.persist(context=context,
|
|
465
|
+
KubernetesEngineClusterLink.persist(context=context, cluster=self.body)
|
|
480
466
|
|
|
481
467
|
try:
|
|
482
468
|
operation = self.cluster_hook.create_cluster(
|
|
@@ -575,9 +561,16 @@ class GKEStartKueueInsideClusterOperator(GKEOperatorMixin, KubernetesInstallKueu
|
|
|
575
561
|
self.use_dns_endpoint = use_dns_endpoint
|
|
576
562
|
self.impersonation_chain = impersonation_chain
|
|
577
563
|
|
|
564
|
+
@property
|
|
565
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
566
|
+
return {
|
|
567
|
+
"project_id": self.project_id,
|
|
568
|
+
"location": self.location,
|
|
569
|
+
}
|
|
570
|
+
|
|
578
571
|
def execute(self, context: Context):
|
|
579
572
|
cluster = self.cluster_hook.get_cluster(name=self.cluster_name, project_id=self.project_id)
|
|
580
|
-
KubernetesEngineClusterLink.persist(context=context,
|
|
573
|
+
KubernetesEngineClusterLink.persist(context=context, cluster=cluster)
|
|
581
574
|
|
|
582
575
|
if self.cluster_hook.check_cluster_autoscaling_ability(cluster=cluster):
|
|
583
576
|
super().execute(context)
|
|
@@ -622,21 +615,15 @@ class GKEStartPodOperator(GKEOperatorMixin, KubernetesPodOperator):
|
|
|
622
615
|
If set as a sequence, the identities from the list must grant
|
|
623
616
|
Service Account Token Creator IAM role to the directly preceding identity, with first
|
|
624
617
|
account from the list granting this role to the originating account (templated).
|
|
625
|
-
:param regional: (Deprecated) The location param is region name.
|
|
626
618
|
:param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted.
|
|
627
619
|
If "delete_pod", the pod will be deleted regardless its state; if "delete_succeeded_pod",
|
|
628
620
|
only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod.
|
|
629
|
-
Current default is `
|
|
630
|
-
:param is_delete_operator_pod: (Deprecated) What to do when the pod reaches its final
|
|
631
|
-
state, or the execution is interrupted. If True, delete the
|
|
632
|
-
pod; if False, leave the pod. Current default is False, but this will be
|
|
633
|
-
changed in the next major release of this provider.
|
|
634
|
-
Deprecated - use `on_finish_action` instead.
|
|
621
|
+
Current default is `delete_pod`, but this will be changed in the next major release of this provider.
|
|
635
622
|
:param deferrable: Run operator in the deferrable mode.
|
|
636
623
|
"""
|
|
637
624
|
|
|
638
625
|
template_fields: Sequence[str] = tuple(
|
|
639
|
-
{"
|
|
626
|
+
{"deferrable"}
|
|
640
627
|
| (set(KubernetesPodOperator.template_fields) - {"is_delete_operator_pod", "regional"})
|
|
641
628
|
| set(GKEOperatorMixin.template_fields)
|
|
642
629
|
)
|
|
@@ -651,30 +638,15 @@ class GKEStartPodOperator(GKEOperatorMixin, KubernetesPodOperator):
|
|
|
651
638
|
project_id: str = PROVIDE_PROJECT_ID,
|
|
652
639
|
gcp_conn_id: str = "google_cloud_default",
|
|
653
640
|
impersonation_chain: str | Sequence[str] | None = None,
|
|
654
|
-
regional: bool | None = None,
|
|
655
641
|
on_finish_action: str | None = None,
|
|
656
|
-
is_delete_operator_pod: bool | None = None,
|
|
657
642
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
|
658
643
|
*args,
|
|
659
644
|
**kwargs,
|
|
660
645
|
) -> None:
|
|
661
|
-
if
|
|
662
|
-
kwargs["on_finish_action"] = (
|
|
663
|
-
OnFinishAction.DELETE_POD if is_delete_operator_pod else OnFinishAction.KEEP_POD
|
|
664
|
-
)
|
|
665
|
-
elif on_finish_action is not None:
|
|
646
|
+
if on_finish_action is not None:
|
|
666
647
|
kwargs["on_finish_action"] = OnFinishAction(on_finish_action)
|
|
667
648
|
else:
|
|
668
|
-
|
|
669
|
-
f"You have not set parameter `on_finish_action` in class {self.__class__.__name__}. "
|
|
670
|
-
"Currently the default for this parameter is `keep_pod` but in a future release"
|
|
671
|
-
" the default will be changed to `delete_pod`. To ensure pods are not deleted in"
|
|
672
|
-
" the future you will need to set `on_finish_action=keep_pod` explicitly.",
|
|
673
|
-
AirflowProviderDeprecationWarning,
|
|
674
|
-
stacklevel=2,
|
|
675
|
-
)
|
|
676
|
-
kwargs["on_finish_action"] = OnFinishAction.KEEP_POD
|
|
677
|
-
|
|
649
|
+
kwargs["on_finish_action"] = OnFinishAction.DELETE_POD
|
|
678
650
|
super().__init__(*args, **kwargs)
|
|
679
651
|
self.project_id = project_id
|
|
680
652
|
self.location = location
|
|
@@ -683,9 +655,6 @@ class GKEStartPodOperator(GKEOperatorMixin, KubernetesPodOperator):
|
|
|
683
655
|
self.use_internal_ip = use_internal_ip
|
|
684
656
|
self.use_dns_endpoint = use_dns_endpoint
|
|
685
657
|
self.impersonation_chain = impersonation_chain
|
|
686
|
-
self._regional = regional
|
|
687
|
-
if is_delete_operator_pod is not None:
|
|
688
|
-
self.is_delete_operator_pod = is_delete_operator_pod
|
|
689
658
|
self.deferrable = deferrable
|
|
690
659
|
|
|
691
660
|
# There is no need to manage the kube_config file, as it will be generated automatically.
|
|
@@ -693,42 +662,6 @@ class GKEStartPodOperator(GKEOperatorMixin, KubernetesPodOperator):
|
|
|
693
662
|
if self.config_file:
|
|
694
663
|
raise AirflowException("config_file is not an allowed parameter for the GKEStartPodOperator.")
|
|
695
664
|
|
|
696
|
-
@property
|
|
697
|
-
@deprecated(
|
|
698
|
-
planned_removal_date="May 01, 2025",
|
|
699
|
-
use_instead="on_finish_action",
|
|
700
|
-
category=AirflowProviderDeprecationWarning,
|
|
701
|
-
)
|
|
702
|
-
def is_delete_operator_pod(self) -> bool | None:
|
|
703
|
-
return self._is_delete_operator_pod
|
|
704
|
-
|
|
705
|
-
@is_delete_operator_pod.setter
|
|
706
|
-
@deprecated(
|
|
707
|
-
planned_removal_date="May 01, 2025",
|
|
708
|
-
use_instead="on_finish_action",
|
|
709
|
-
category=AirflowProviderDeprecationWarning,
|
|
710
|
-
)
|
|
711
|
-
def is_delete_operator_pod(self, is_delete_operator_pod) -> None:
|
|
712
|
-
self._is_delete_operator_pod = is_delete_operator_pod
|
|
713
|
-
|
|
714
|
-
@property
|
|
715
|
-
@deprecated(
|
|
716
|
-
planned_removal_date="May 01, 2025",
|
|
717
|
-
reason="The parameter is not in actual use.",
|
|
718
|
-
category=AirflowProviderDeprecationWarning,
|
|
719
|
-
)
|
|
720
|
-
def regional(self) -> bool | None:
|
|
721
|
-
return self._regional
|
|
722
|
-
|
|
723
|
-
@regional.setter
|
|
724
|
-
@deprecated(
|
|
725
|
-
planned_removal_date="May 01, 2025",
|
|
726
|
-
reason="The parameter is not in actual use.",
|
|
727
|
-
category=AirflowProviderDeprecationWarning,
|
|
728
|
-
)
|
|
729
|
-
def regional(self, regional) -> None:
|
|
730
|
-
self._regional = regional
|
|
731
|
-
|
|
732
665
|
def invoke_defer_method(self, last_log_time: DateTime | None = None):
|
|
733
666
|
"""Redefine triggers which are being used in child classes."""
|
|
734
667
|
trigger_start_time = utcnow()
|
|
@@ -855,10 +788,10 @@ class GKEStartJobOperator(GKEOperatorMixin, KubernetesJobOperator):
|
|
|
855
788
|
trigger=GKEJobTrigger(
|
|
856
789
|
cluster_url=self.cluster_url,
|
|
857
790
|
ssl_ca_cert=self.ssl_ca_cert,
|
|
858
|
-
job_name=self.job.metadata.name,
|
|
859
|
-
job_namespace=self.job.metadata.namespace,
|
|
860
|
-
|
|
861
|
-
pod_namespace=self.
|
|
791
|
+
job_name=self.job.metadata.name,
|
|
792
|
+
job_namespace=self.job.metadata.namespace,
|
|
793
|
+
pod_names=[pod.metadata.name for pod in self.pods],
|
|
794
|
+
pod_namespace=self.pods[0].metadata.namespace,
|
|
862
795
|
base_container_name=self.base_container_name,
|
|
863
796
|
gcp_conn_id=self.gcp_conn_id,
|
|
864
797
|
poll_interval=self.job_poll_interval,
|
|
@@ -936,7 +869,14 @@ class GKEDescribeJobOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
|
936
869
|
self.cluster_name,
|
|
937
870
|
self.job,
|
|
938
871
|
)
|
|
939
|
-
KubernetesEngineJobLink.persist(
|
|
872
|
+
KubernetesEngineJobLink.persist(
|
|
873
|
+
context=context,
|
|
874
|
+
location=self.location,
|
|
875
|
+
cluster_name=self.cluster_name,
|
|
876
|
+
namespace=self.job.metadata.namespace,
|
|
877
|
+
job_name=self.job.metadata.name,
|
|
878
|
+
project_id=self.project_id,
|
|
879
|
+
)
|
|
940
880
|
return None
|
|
941
881
|
|
|
942
882
|
|
|
@@ -1000,6 +940,15 @@ class GKEListJobsOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
|
1000
940
|
self.namespace = namespace
|
|
1001
941
|
self.do_xcom_push = do_xcom_push
|
|
1002
942
|
|
|
943
|
+
@property
|
|
944
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
945
|
+
return {
|
|
946
|
+
"location": self.location,
|
|
947
|
+
"cluster_name": self.cluster_name,
|
|
948
|
+
"namespace": self.namespace,
|
|
949
|
+
"project_id": self.project_id,
|
|
950
|
+
}
|
|
951
|
+
|
|
1003
952
|
def execute(self, context: Context) -> dict:
|
|
1004
953
|
if self.namespace:
|
|
1005
954
|
jobs = self.hook.list_jobs_from_namespace(namespace=self.namespace)
|
|
@@ -1010,7 +959,7 @@ class GKEListJobsOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
|
1010
959
|
if self.do_xcom_push:
|
|
1011
960
|
ti = context["ti"]
|
|
1012
961
|
ti.xcom_push(key="jobs_list", value=V1JobList.to_dict(jobs))
|
|
1013
|
-
KubernetesEngineWorkloadsLink.persist(context=context
|
|
962
|
+
KubernetesEngineWorkloadsLink.persist(context=context)
|
|
1014
963
|
return V1JobList.to_dict(jobs)
|
|
1015
964
|
|
|
1016
965
|
|
|
@@ -1352,8 +1301,14 @@ class GKESuspendJobOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
|
1352
1301
|
self.name,
|
|
1353
1302
|
self.cluster_name,
|
|
1354
1303
|
)
|
|
1355
|
-
KubernetesEngineJobLink.persist(
|
|
1356
|
-
|
|
1304
|
+
KubernetesEngineJobLink.persist(
|
|
1305
|
+
context=context,
|
|
1306
|
+
location=self.location,
|
|
1307
|
+
cluster_name=self.cluster_name,
|
|
1308
|
+
namespace=self.job.metadata.namespace,
|
|
1309
|
+
job_name=self.job.metadata.name,
|
|
1310
|
+
project_id=self.project_id,
|
|
1311
|
+
)
|
|
1357
1312
|
return k8s.V1Job.to_dict(self.job)
|
|
1358
1313
|
|
|
1359
1314
|
|
|
@@ -1426,6 +1381,13 @@ class GKEResumeJobOperator(GKEOperatorMixin, GoogleCloudBaseOperator):
|
|
|
1426
1381
|
self.name,
|
|
1427
1382
|
self.cluster_name,
|
|
1428
1383
|
)
|
|
1429
|
-
KubernetesEngineJobLink.persist(
|
|
1384
|
+
KubernetesEngineJobLink.persist(
|
|
1385
|
+
context=context,
|
|
1386
|
+
location=self.location,
|
|
1387
|
+
cluster_name=self.cluster_name,
|
|
1388
|
+
namespace=self.job.metadata.namespace,
|
|
1389
|
+
job_name=self.job.metadata.name,
|
|
1390
|
+
project_id=self.project_id,
|
|
1391
|
+
)
|
|
1430
1392
|
|
|
1431
1393
|
return k8s.V1Job.to_dict(self.job)
|
|
@@ -21,12 +21,12 @@ from __future__ import annotations
|
|
|
21
21
|
|
|
22
22
|
from typing import TYPE_CHECKING
|
|
23
23
|
|
|
24
|
-
from airflow.
|
|
24
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
25
25
|
from airflow.providers.google.cloud.hooks.looker import LookerHook
|
|
26
26
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
27
27
|
|
|
28
28
|
if TYPE_CHECKING:
|
|
29
|
-
from airflow.
|
|
29
|
+
from airflow.providers.common.compat.sdk import Context
|
|
30
30
|
|
|
31
31
|
|
|
32
32
|
class LookerStartPdtBuildOperator(GoogleCloudBaseOperator):
|
|
@@ -21,13 +21,13 @@ from __future__ import annotations
|
|
|
21
21
|
|
|
22
22
|
from collections.abc import Sequence
|
|
23
23
|
from functools import cached_property
|
|
24
|
-
from typing import TYPE_CHECKING
|
|
24
|
+
from typing import TYPE_CHECKING, Any
|
|
25
25
|
|
|
26
26
|
from google.api_core.exceptions import AlreadyExists, NotFound
|
|
27
27
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
28
28
|
from google.cloud.managedkafka_v1 import types
|
|
29
29
|
|
|
30
|
-
from airflow.
|
|
30
|
+
from airflow.providers.common.compat.sdk import AirflowException
|
|
31
31
|
from airflow.providers.google.cloud.hooks.managed_kafka import ManagedKafkaHook
|
|
32
32
|
from airflow.providers.google.cloud.links.managed_kafka import (
|
|
33
33
|
ApacheKafkaClusterLink,
|
|
@@ -41,7 +41,7 @@ if TYPE_CHECKING:
|
|
|
41
41
|
from google.api_core.retry import Retry
|
|
42
42
|
from google.protobuf.field_mask_pb2 import FieldMask
|
|
43
43
|
|
|
44
|
-
from airflow.
|
|
44
|
+
from airflow.providers.common.compat.sdk import Context
|
|
45
45
|
|
|
46
46
|
|
|
47
47
|
class ManagedKafkaBaseOperator(GoogleCloudBaseOperator):
|
|
@@ -150,9 +150,17 @@ class ManagedKafkaCreateClusterOperator(ManagedKafkaBaseOperator):
|
|
|
150
150
|
self.cluster_id = cluster_id
|
|
151
151
|
self.request_id = request_id
|
|
152
152
|
|
|
153
|
+
@property
|
|
154
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
155
|
+
return {
|
|
156
|
+
"location": self.location,
|
|
157
|
+
"cluster_id": self.cluster_id,
|
|
158
|
+
"project_id": self.project_id,
|
|
159
|
+
}
|
|
160
|
+
|
|
153
161
|
def execute(self, context: Context):
|
|
154
162
|
self.log.info("Creating an Apache Kafka cluster.")
|
|
155
|
-
ApacheKafkaClusterLink.persist(context=context
|
|
163
|
+
ApacheKafkaClusterLink.persist(context=context)
|
|
156
164
|
try:
|
|
157
165
|
operation = self.hook.create_cluster(
|
|
158
166
|
project_id=self.project_id,
|
|
@@ -227,8 +235,14 @@ class ManagedKafkaListClustersOperator(ManagedKafkaBaseOperator):
|
|
|
227
235
|
self.filter = filter
|
|
228
236
|
self.order_by = order_by
|
|
229
237
|
|
|
238
|
+
@property
|
|
239
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
240
|
+
return {
|
|
241
|
+
"project_id": self.project_id,
|
|
242
|
+
}
|
|
243
|
+
|
|
230
244
|
def execute(self, context: Context):
|
|
231
|
-
ApacheKafkaClusterListLink.persist(context=context
|
|
245
|
+
ApacheKafkaClusterListLink.persist(context=context)
|
|
232
246
|
self.log.info("Listing Clusters from location %s.", self.location)
|
|
233
247
|
try:
|
|
234
248
|
cluster_list_pager = self.hook.list_clusters(
|
|
@@ -242,8 +256,7 @@ class ManagedKafkaListClustersOperator(ManagedKafkaBaseOperator):
|
|
|
242
256
|
timeout=self.timeout,
|
|
243
257
|
metadata=self.metadata,
|
|
244
258
|
)
|
|
245
|
-
|
|
246
|
-
context=context,
|
|
259
|
+
context["ti"].xcom_push(
|
|
247
260
|
key="cluster_page",
|
|
248
261
|
value=types.ListClustersResponse.to_dict(cluster_list_pager._response),
|
|
249
262
|
)
|
|
@@ -285,12 +298,16 @@ class ManagedKafkaGetClusterOperator(ManagedKafkaBaseOperator):
|
|
|
285
298
|
super().__init__(*args, **kwargs)
|
|
286
299
|
self.cluster_id = cluster_id
|
|
287
300
|
|
|
301
|
+
@property
|
|
302
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
303
|
+
return {
|
|
304
|
+
"location": self.location,
|
|
305
|
+
"cluster_id": self.cluster_id,
|
|
306
|
+
"project_id": self.project_id,
|
|
307
|
+
}
|
|
308
|
+
|
|
288
309
|
def execute(self, context: Context):
|
|
289
|
-
ApacheKafkaClusterLink.persist(
|
|
290
|
-
context=context,
|
|
291
|
-
task_instance=self,
|
|
292
|
-
cluster_id=self.cluster_id,
|
|
293
|
-
)
|
|
310
|
+
ApacheKafkaClusterLink.persist(context=context)
|
|
294
311
|
self.log.info("Getting Cluster: %s", self.cluster_id)
|
|
295
312
|
try:
|
|
296
313
|
cluster = self.hook.get_cluster(
|
|
@@ -362,12 +379,16 @@ class ManagedKafkaUpdateClusterOperator(ManagedKafkaBaseOperator):
|
|
|
362
379
|
self.update_mask = update_mask
|
|
363
380
|
self.request_id = request_id
|
|
364
381
|
|
|
382
|
+
@property
|
|
383
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
384
|
+
return {
|
|
385
|
+
"location": self.location,
|
|
386
|
+
"cluster_id": self.cluster_id,
|
|
387
|
+
"project_id": self.project_id,
|
|
388
|
+
}
|
|
389
|
+
|
|
365
390
|
def execute(self, context: Context):
|
|
366
|
-
ApacheKafkaClusterLink.persist(
|
|
367
|
-
context=context,
|
|
368
|
-
task_instance=self,
|
|
369
|
-
cluster_id=self.cluster_id,
|
|
370
|
-
)
|
|
391
|
+
ApacheKafkaClusterLink.persist(context=context)
|
|
371
392
|
self.log.info("Updating an Apache Kafka cluster.")
|
|
372
393
|
try:
|
|
373
394
|
operation = self.hook.update_cluster(
|
|
@@ -497,14 +518,18 @@ class ManagedKafkaCreateTopicOperator(ManagedKafkaBaseOperator):
|
|
|
497
518
|
self.topic_id = topic_id
|
|
498
519
|
self.topic = topic
|
|
499
520
|
|
|
521
|
+
@property
|
|
522
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
523
|
+
return {
|
|
524
|
+
"location": self.location,
|
|
525
|
+
"cluster_id": self.cluster_id,
|
|
526
|
+
"topic_id": self.topic_id,
|
|
527
|
+
"project_id": self.project_id,
|
|
528
|
+
}
|
|
529
|
+
|
|
500
530
|
def execute(self, context: Context):
|
|
501
531
|
self.log.info("Creating an Apache Kafka topic.")
|
|
502
|
-
ApacheKafkaTopicLink.persist(
|
|
503
|
-
context=context,
|
|
504
|
-
task_instance=self,
|
|
505
|
-
cluster_id=self.cluster_id,
|
|
506
|
-
topic_id=self.topic_id,
|
|
507
|
-
)
|
|
532
|
+
ApacheKafkaTopicLink.persist(context=context)
|
|
508
533
|
try:
|
|
509
534
|
topic_obj = self.hook.create_topic(
|
|
510
535
|
project_id=self.project_id,
|
|
@@ -574,8 +599,16 @@ class ManagedKafkaListTopicsOperator(ManagedKafkaBaseOperator):
|
|
|
574
599
|
self.page_size = page_size
|
|
575
600
|
self.page_token = page_token
|
|
576
601
|
|
|
602
|
+
@property
|
|
603
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
604
|
+
return {
|
|
605
|
+
"location": self.location,
|
|
606
|
+
"cluster_id": self.cluster_id,
|
|
607
|
+
"project_id": self.project_id,
|
|
608
|
+
}
|
|
609
|
+
|
|
577
610
|
def execute(self, context: Context):
|
|
578
|
-
ApacheKafkaClusterLink.persist(context=context
|
|
611
|
+
ApacheKafkaClusterLink.persist(context=context)
|
|
579
612
|
self.log.info("Listing Topics for cluster %s.", self.cluster_id)
|
|
580
613
|
try:
|
|
581
614
|
topic_list_pager = self.hook.list_topics(
|
|
@@ -588,8 +621,7 @@ class ManagedKafkaListTopicsOperator(ManagedKafkaBaseOperator):
|
|
|
588
621
|
timeout=self.timeout,
|
|
589
622
|
metadata=self.metadata,
|
|
590
623
|
)
|
|
591
|
-
|
|
592
|
-
context=context,
|
|
624
|
+
context["ti"].xcom_push(
|
|
593
625
|
key="topic_page",
|
|
594
626
|
value=types.ListTopicsResponse.to_dict(topic_list_pager._response),
|
|
595
627
|
)
|
|
@@ -636,13 +668,17 @@ class ManagedKafkaGetTopicOperator(ManagedKafkaBaseOperator):
|
|
|
636
668
|
self.cluster_id = cluster_id
|
|
637
669
|
self.topic_id = topic_id
|
|
638
670
|
|
|
671
|
+
@property
|
|
672
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
673
|
+
return {
|
|
674
|
+
"location": self.location,
|
|
675
|
+
"cluster_id": self.cluster_id,
|
|
676
|
+
"topic_id": self.topic_id,
|
|
677
|
+
"project_id": self.project_id,
|
|
678
|
+
}
|
|
679
|
+
|
|
639
680
|
def execute(self, context: Context):
|
|
640
|
-
ApacheKafkaTopicLink.persist(
|
|
641
|
-
context=context,
|
|
642
|
-
task_instance=self,
|
|
643
|
-
cluster_id=self.cluster_id,
|
|
644
|
-
topic_id=self.topic_id,
|
|
645
|
-
)
|
|
681
|
+
ApacheKafkaTopicLink.persist(context=context)
|
|
646
682
|
self.log.info("Getting Topic: %s", self.topic_id)
|
|
647
683
|
try:
|
|
648
684
|
topic = self.hook.get_topic(
|
|
@@ -707,13 +743,17 @@ class ManagedKafkaUpdateTopicOperator(ManagedKafkaBaseOperator):
|
|
|
707
743
|
self.topic = topic
|
|
708
744
|
self.update_mask = update_mask
|
|
709
745
|
|
|
746
|
+
@property
|
|
747
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
748
|
+
return {
|
|
749
|
+
"location": self.location,
|
|
750
|
+
"cluster_id": self.cluster_id,
|
|
751
|
+
"topic_id": self.topic_id,
|
|
752
|
+
"project_id": self.project_id,
|
|
753
|
+
}
|
|
754
|
+
|
|
710
755
|
def execute(self, context: Context):
|
|
711
|
-
ApacheKafkaTopicLink.persist(
|
|
712
|
-
context=context,
|
|
713
|
-
task_instance=self,
|
|
714
|
-
cluster_id=self.cluster_id,
|
|
715
|
-
topic_id=self.topic_id,
|
|
716
|
-
)
|
|
756
|
+
ApacheKafkaTopicLink.persist(context=context)
|
|
717
757
|
self.log.info("Updating an Apache Kafka topic.")
|
|
718
758
|
try:
|
|
719
759
|
topic_obj = self.hook.update_topic(
|
|
@@ -833,8 +873,16 @@ class ManagedKafkaListConsumerGroupsOperator(ManagedKafkaBaseOperator):
|
|
|
833
873
|
self.page_size = page_size
|
|
834
874
|
self.page_token = page_token
|
|
835
875
|
|
|
876
|
+
@property
|
|
877
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
878
|
+
return {
|
|
879
|
+
"location": self.location,
|
|
880
|
+
"cluster_id": self.cluster_id,
|
|
881
|
+
"project_id": self.project_id,
|
|
882
|
+
}
|
|
883
|
+
|
|
836
884
|
def execute(self, context: Context):
|
|
837
|
-
ApacheKafkaClusterLink.persist(context=context
|
|
885
|
+
ApacheKafkaClusterLink.persist(context=context)
|
|
838
886
|
self.log.info("Listing Consumer Groups for cluster %s.", self.cluster_id)
|
|
839
887
|
try:
|
|
840
888
|
consumer_group_list_pager = self.hook.list_consumer_groups(
|
|
@@ -847,8 +895,7 @@ class ManagedKafkaListConsumerGroupsOperator(ManagedKafkaBaseOperator):
|
|
|
847
895
|
timeout=self.timeout,
|
|
848
896
|
metadata=self.metadata,
|
|
849
897
|
)
|
|
850
|
-
|
|
851
|
-
context=context,
|
|
898
|
+
context["ti"].xcom_push(
|
|
852
899
|
key="consumer_group_page",
|
|
853
900
|
value=types.ListConsumerGroupsResponse.to_dict(consumer_group_list_pager._response),
|
|
854
901
|
)
|
|
@@ -895,13 +942,17 @@ class ManagedKafkaGetConsumerGroupOperator(ManagedKafkaBaseOperator):
|
|
|
895
942
|
self.cluster_id = cluster_id
|
|
896
943
|
self.consumer_group_id = consumer_group_id
|
|
897
944
|
|
|
945
|
+
@property
|
|
946
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
947
|
+
return {
|
|
948
|
+
"location": self.location,
|
|
949
|
+
"cluster_id": self.cluster_id,
|
|
950
|
+
"consumer_group_id": self.consumer_group_id,
|
|
951
|
+
"project_id": self.project_id,
|
|
952
|
+
}
|
|
953
|
+
|
|
898
954
|
def execute(self, context: Context):
|
|
899
|
-
ApacheKafkaConsumerGroupLink.persist(
|
|
900
|
-
context=context,
|
|
901
|
-
task_instance=self,
|
|
902
|
-
cluster_id=self.cluster_id,
|
|
903
|
-
consumer_group_id=self.consumer_group_id,
|
|
904
|
-
)
|
|
955
|
+
ApacheKafkaConsumerGroupLink.persist(context=context)
|
|
905
956
|
self.log.info("Getting Consumer Group: %s", self.consumer_group_id)
|
|
906
957
|
try:
|
|
907
958
|
consumer_group = self.hook.get_consumer_group(
|
|
@@ -971,13 +1022,17 @@ class ManagedKafkaUpdateConsumerGroupOperator(ManagedKafkaBaseOperator):
|
|
|
971
1022
|
self.consumer_group = consumer_group
|
|
972
1023
|
self.update_mask = update_mask
|
|
973
1024
|
|
|
1025
|
+
@property
|
|
1026
|
+
def extra_links_params(self) -> dict[str, Any]:
|
|
1027
|
+
return {
|
|
1028
|
+
"location": self.location,
|
|
1029
|
+
"cluster_id": self.cluster_id,
|
|
1030
|
+
"consumer_group_id": self.consumer_group_id,
|
|
1031
|
+
"project_id": self.project_id,
|
|
1032
|
+
}
|
|
1033
|
+
|
|
974
1034
|
def execute(self, context: Context):
|
|
975
|
-
ApacheKafkaConsumerGroupLink.persist(
|
|
976
|
-
context=context,
|
|
977
|
-
task_instance=self,
|
|
978
|
-
cluster_id=self.cluster_id,
|
|
979
|
-
consumer_group_id=self.consumer_group_id,
|
|
980
|
-
)
|
|
1035
|
+
ApacheKafkaConsumerGroupLink.persist(context=context)
|
|
981
1036
|
self.log.info("Updating an Apache Kafka consumer group.")
|
|
982
1037
|
try:
|
|
983
1038
|
consumer_group_obj = self.hook.update_consumer_group(
|
|
@@ -32,7 +32,7 @@ if TYPE_CHECKING:
|
|
|
32
32
|
from google.api_core.retry import Retry
|
|
33
33
|
from google.cloud.language_v1.types import Document, EncodingType
|
|
34
34
|
|
|
35
|
-
from airflow.
|
|
35
|
+
from airflow.providers.common.compat.sdk import Context
|
|
36
36
|
|
|
37
37
|
|
|
38
38
|
MetaData = Sequence[tuple[str, str]]
|