apache-airflow-providers-google 14.0.0__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
- airflow/providers/google/3rd-party-licenses/NOTICE +5 -0
- airflow/providers/google/__init__.py +3 -3
- airflow/providers/google/_vendor/__init__.py +0 -0
- airflow/providers/google/_vendor/json_merge_patch.py +91 -0
- airflow/providers/google/ads/hooks/ads.py +52 -43
- airflow/providers/google/ads/operators/ads.py +2 -2
- airflow/providers/google/ads/transfers/ads_to_gcs.py +3 -19
- airflow/providers/google/assets/gcs.py +1 -11
- airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
- airflow/providers/google/cloud/bundles/gcs.py +161 -0
- airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
- airflow/providers/google/cloud/hooks/bigquery.py +195 -318
- airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
- airflow/providers/google/cloud/hooks/bigtable.py +3 -2
- airflow/providers/google/cloud/hooks/cloud_batch.py +8 -9
- airflow/providers/google/cloud/hooks/cloud_build.py +6 -65
- airflow/providers/google/cloud/hooks/cloud_composer.py +292 -24
- airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
- airflow/providers/google/cloud/hooks/cloud_run.py +20 -11
- airflow/providers/google/cloud/hooks/cloud_sql.py +136 -64
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +35 -15
- airflow/providers/google/cloud/hooks/compute.py +7 -6
- airflow/providers/google/cloud/hooks/compute_ssh.py +7 -4
- airflow/providers/google/cloud/hooks/datacatalog.py +12 -3
- airflow/providers/google/cloud/hooks/dataflow.py +87 -242
- airflow/providers/google/cloud/hooks/dataform.py +9 -14
- airflow/providers/google/cloud/hooks/datafusion.py +7 -9
- airflow/providers/google/cloud/hooks/dataplex.py +13 -12
- airflow/providers/google/cloud/hooks/dataprep.py +2 -2
- airflow/providers/google/cloud/hooks/dataproc.py +76 -74
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/hooks/dlp.py +5 -4
- airflow/providers/google/cloud/hooks/gcs.py +144 -33
- airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
- airflow/providers/google/cloud/hooks/kms.py +3 -2
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +22 -17
- airflow/providers/google/cloud/hooks/looker.py +6 -1
- airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
- airflow/providers/google/cloud/hooks/mlengine.py +7 -8
- airflow/providers/google/cloud/hooks/natural_language.py +3 -2
- airflow/providers/google/cloud/hooks/os_login.py +3 -2
- airflow/providers/google/cloud/hooks/pubsub.py +6 -6
- airflow/providers/google/cloud/hooks/secret_manager.py +105 -12
- airflow/providers/google/cloud/hooks/spanner.py +75 -10
- airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
- airflow/providers/google/cloud/hooks/stackdriver.py +18 -18
- airflow/providers/google/cloud/hooks/tasks.py +4 -3
- airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
- airflow/providers/google/cloud/hooks/translate.py +8 -17
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +8 -222
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +9 -15
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +33 -283
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +311 -10
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +7 -13
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +8 -12
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
- airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
- airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
- airflow/providers/google/cloud/hooks/vision.py +7 -7
- airflow/providers/google/cloud/hooks/workflows.py +4 -3
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +77 -7
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -46
- airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -90
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +11 -89
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +11 -61
- airflow/providers/google/cloud/links/managed_kafka.py +11 -51
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +17 -187
- airflow/providers/google/cloud/links/vertex_ai.py +28 -195
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/log/gcs_task_handler.py +166 -118
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +14 -9
- airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
- airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
- airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
- airflow/providers/google/cloud/openlineage/facets.py +141 -40
- airflow/providers/google/cloud/openlineage/mixins.py +14 -13
- airflow/providers/google/cloud/openlineage/utils.py +19 -3
- airflow/providers/google/cloud/operators/alloy_db.py +76 -61
- airflow/providers/google/cloud/operators/bigquery.py +104 -667
- airflow/providers/google/cloud/operators/bigquery_dts.py +12 -12
- airflow/providers/google/cloud/operators/bigtable.py +38 -7
- airflow/providers/google/cloud/operators/cloud_base.py +22 -1
- airflow/providers/google/cloud/operators/cloud_batch.py +18 -18
- airflow/providers/google/cloud/operators/cloud_build.py +80 -36
- airflow/providers/google/cloud/operators/cloud_composer.py +157 -71
- airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
- airflow/providers/google/cloud/operators/cloud_memorystore.py +74 -46
- airflow/providers/google/cloud/operators/cloud_run.py +39 -20
- airflow/providers/google/cloud/operators/cloud_sql.py +46 -61
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +92 -14
- airflow/providers/google/cloud/operators/compute.py +18 -50
- airflow/providers/google/cloud/operators/datacatalog.py +167 -29
- airflow/providers/google/cloud/operators/dataflow.py +38 -15
- airflow/providers/google/cloud/operators/dataform.py +19 -7
- airflow/providers/google/cloud/operators/datafusion.py +43 -43
- airflow/providers/google/cloud/operators/dataplex.py +212 -126
- airflow/providers/google/cloud/operators/dataprep.py +1 -5
- airflow/providers/google/cloud/operators/dataproc.py +134 -207
- airflow/providers/google/cloud/operators/dataproc_metastore.py +102 -84
- airflow/providers/google/cloud/operators/datastore.py +22 -6
- airflow/providers/google/cloud/operators/dlp.py +24 -45
- airflow/providers/google/cloud/operators/functions.py +21 -14
- airflow/providers/google/cloud/operators/gcs.py +15 -12
- airflow/providers/google/cloud/operators/gen_ai.py +389 -0
- airflow/providers/google/cloud/operators/kubernetes_engine.py +115 -106
- airflow/providers/google/cloud/operators/looker.py +1 -1
- airflow/providers/google/cloud/operators/managed_kafka.py +362 -40
- airflow/providers/google/cloud/operators/natural_language.py +5 -3
- airflow/providers/google/cloud/operators/pubsub.py +69 -21
- airflow/providers/google/cloud/operators/spanner.py +53 -45
- airflow/providers/google/cloud/operators/speech_to_text.py +5 -4
- airflow/providers/google/cloud/operators/stackdriver.py +5 -11
- airflow/providers/google/cloud/operators/tasks.py +6 -15
- airflow/providers/google/cloud/operators/text_to_speech.py +4 -3
- airflow/providers/google/cloud/operators/translate.py +46 -20
- airflow/providers/google/cloud/operators/translate_speech.py +4 -3
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +44 -34
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +34 -12
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +62 -53
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +75 -11
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +48 -12
- airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -116
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +16 -12
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +62 -14
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +35 -10
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
- airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
- airflow/providers/google/cloud/operators/vision.py +7 -5
- airflow/providers/google/cloud/operators/workflows.py +24 -19
- airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
- airflow/providers/google/cloud/sensors/bigquery.py +2 -2
- airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -4
- airflow/providers/google/cloud/sensors/bigtable.py +14 -6
- airflow/providers/google/cloud/sensors/cloud_composer.py +535 -33
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -5
- airflow/providers/google/cloud/sensors/dataflow.py +27 -10
- airflow/providers/google/cloud/sensors/dataform.py +2 -2
- airflow/providers/google/cloud/sensors/datafusion.py +4 -4
- airflow/providers/google/cloud/sensors/dataplex.py +7 -5
- airflow/providers/google/cloud/sensors/dataprep.py +2 -2
- airflow/providers/google/cloud/sensors/dataproc.py +10 -9
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +4 -3
- airflow/providers/google/cloud/sensors/gcs.py +22 -21
- airflow/providers/google/cloud/sensors/looker.py +5 -5
- airflow/providers/google/cloud/sensors/pubsub.py +20 -20
- airflow/providers/google/cloud/sensors/tasks.py +2 -2
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -2
- airflow/providers/google/cloud/sensors/workflows.py +6 -4
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +14 -13
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
- airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +18 -22
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -5
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +45 -38
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/gcs_to_local.py +5 -3
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +10 -4
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
- airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
- airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +44 -12
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +12 -6
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +36 -14
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
- airflow/providers/google/cloud/triggers/bigquery.py +75 -34
- airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +3 -2
- airflow/providers/google/cloud/triggers/cloud_composer.py +303 -47
- airflow/providers/google/cloud/triggers/cloud_run.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +96 -5
- airflow/providers/google/cloud/triggers/dataflow.py +125 -2
- airflow/providers/google/cloud/triggers/datafusion.py +1 -1
- airflow/providers/google/cloud/triggers/dataplex.py +16 -3
- airflow/providers/google/cloud/triggers/dataproc.py +124 -53
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +46 -28
- airflow/providers/google/cloud/triggers/mlengine.py +1 -1
- airflow/providers/google/cloud/triggers/pubsub.py +17 -20
- airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
- airflow/providers/google/cloud/utils/bigquery.py +5 -7
- airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
- airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
- airflow/providers/google/cloud/utils/field_validator.py +1 -2
- airflow/providers/google/cloud/utils/validators.py +43 -0
- airflow/providers/google/common/auth_backend/google_openid.py +26 -9
- airflow/providers/google/common/consts.py +2 -1
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +40 -43
- airflow/providers/google/common/hooks/operation_helpers.py +78 -0
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/common/utils/get_secret.py +31 -0
- airflow/providers/google/common/utils/id_token_credentials.py +4 -5
- airflow/providers/google/firebase/operators/firestore.py +2 -2
- airflow/providers/google/get_provider_info.py +61 -216
- airflow/providers/google/go_module_utils.py +35 -3
- airflow/providers/google/leveldb/hooks/leveldb.py +30 -6
- airflow/providers/google/leveldb/operators/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
- airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
- airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
- airflow/providers/google/marketing_platform/links/analytics_admin.py +4 -5
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -6
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +5 -5
- airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
- airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
- airflow/providers/google/marketing_platform/sensors/display_video.py +4 -64
- airflow/providers/google/suite/hooks/calendar.py +1 -1
- airflow/providers/google/suite/hooks/drive.py +2 -2
- airflow/providers/google/suite/hooks/sheets.py +15 -1
- airflow/providers/google/suite/operators/sheets.py +8 -3
- airflow/providers/google/suite/sensors/drive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
- airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
- airflow/providers/google/version_compat.py +15 -1
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/METADATA +117 -72
- apache_airflow_providers_google-19.1.0rc1.dist-info/RECORD +331 -0
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/WHEEL +1 -1
- apache_airflow_providers_google-19.1.0rc1.dist-info/licenses/NOTICE +5 -0
- airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
- airflow/providers/google/cloud/hooks/automl.py +0 -679
- airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
- airflow/providers/google/cloud/links/automl.py +0 -193
- airflow/providers/google/cloud/operators/automl.py +0 -1360
- airflow/providers/google/cloud/operators/life_sciences.py +0 -119
- airflow/providers/google/cloud/operators/mlengine.py +0 -1515
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +0 -273
- apache_airflow_providers_google-14.0.0.dist-info/RECORD +0 -318
- /airflow/providers/google/cloud/{example_dags → bundles}/__init__.py +0 -0
- {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/entry_points.txt +0 -0
- {airflow/providers/google → apache_airflow_providers_google-19.1.0rc1.dist-info/licenses}/LICENSE +0 -0
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
from __future__ import annotations
|
|
21
21
|
|
|
22
22
|
from collections.abc import Sequence
|
|
23
|
+
from functools import cached_property
|
|
23
24
|
from typing import TYPE_CHECKING
|
|
24
25
|
|
|
25
26
|
from airflow.exceptions import AirflowException
|
|
@@ -29,7 +30,8 @@ from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseO
|
|
|
29
30
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
30
31
|
|
|
31
32
|
if TYPE_CHECKING:
|
|
32
|
-
from airflow.
|
|
33
|
+
from airflow.providers.common.compat.sdk import Context
|
|
34
|
+
from airflow.providers.openlineage.extractors import OperatorLineage
|
|
33
35
|
|
|
34
36
|
|
|
35
37
|
class SpannerDeployInstanceOperator(GoogleCloudBaseOperator):
|
|
@@ -122,7 +124,6 @@ class SpannerDeployInstanceOperator(GoogleCloudBaseOperator):
|
|
|
122
124
|
)
|
|
123
125
|
SpannerInstanceLink.persist(
|
|
124
126
|
context=context,
|
|
125
|
-
task_instance=self,
|
|
126
127
|
instance_id=self.instance_id,
|
|
127
128
|
project_id=self.project_id or hook.project_id,
|
|
128
129
|
)
|
|
@@ -188,13 +189,12 @@ class SpannerDeleteInstanceOperator(GoogleCloudBaseOperator):
|
|
|
188
189
|
)
|
|
189
190
|
if hook.get_instance(project_id=self.project_id, instance_id=self.instance_id):
|
|
190
191
|
return hook.delete_instance(project_id=self.project_id, instance_id=self.instance_id)
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
return True
|
|
192
|
+
self.log.info(
|
|
193
|
+
"Instance '%s' does not exist in project '%s'. Aborting delete.",
|
|
194
|
+
self.instance_id,
|
|
195
|
+
self.project_id,
|
|
196
|
+
)
|
|
197
|
+
return True
|
|
198
198
|
|
|
199
199
|
|
|
200
200
|
class SpannerQueryDatabaseInstanceOperator(GoogleCloudBaseOperator):
|
|
@@ -256,6 +256,13 @@ class SpannerQueryDatabaseInstanceOperator(GoogleCloudBaseOperator):
|
|
|
256
256
|
self.impersonation_chain = impersonation_chain
|
|
257
257
|
super().__init__(**kwargs)
|
|
258
258
|
|
|
259
|
+
@cached_property
|
|
260
|
+
def hook(self) -> SpannerHook:
|
|
261
|
+
return SpannerHook(
|
|
262
|
+
gcp_conn_id=self.gcp_conn_id,
|
|
263
|
+
impersonation_chain=self.impersonation_chain,
|
|
264
|
+
)
|
|
265
|
+
|
|
259
266
|
def _validate_inputs(self) -> None:
|
|
260
267
|
if self.project_id == "":
|
|
261
268
|
raise AirflowException("The required parameter 'project_id' is empty")
|
|
@@ -267,10 +274,6 @@ class SpannerQueryDatabaseInstanceOperator(GoogleCloudBaseOperator):
|
|
|
267
274
|
raise AirflowException("The required parameter 'query' is empty")
|
|
268
275
|
|
|
269
276
|
def execute(self, context: Context):
|
|
270
|
-
hook = SpannerHook(
|
|
271
|
-
gcp_conn_id=self.gcp_conn_id,
|
|
272
|
-
impersonation_chain=self.impersonation_chain,
|
|
273
|
-
)
|
|
274
277
|
if isinstance(self.query, str):
|
|
275
278
|
queries = [x.strip() for x in self.query.split(";")]
|
|
276
279
|
self.sanitize_queries(queries)
|
|
@@ -282,8 +285,8 @@ class SpannerQueryDatabaseInstanceOperator(GoogleCloudBaseOperator):
|
|
|
282
285
|
self.instance_id,
|
|
283
286
|
self.database_id,
|
|
284
287
|
)
|
|
285
|
-
self.log.info(queries)
|
|
286
|
-
hook.execute_dml(
|
|
288
|
+
self.log.info("Executing queries: %s", queries)
|
|
289
|
+
result_rows_count_per_query = self.hook.execute_dml(
|
|
287
290
|
project_id=self.project_id,
|
|
288
291
|
instance_id=self.instance_id,
|
|
289
292
|
database_id=self.database_id,
|
|
@@ -291,11 +294,11 @@ class SpannerQueryDatabaseInstanceOperator(GoogleCloudBaseOperator):
|
|
|
291
294
|
)
|
|
292
295
|
SpannerDatabaseLink.persist(
|
|
293
296
|
context=context,
|
|
294
|
-
task_instance=self,
|
|
295
297
|
instance_id=self.instance_id,
|
|
296
298
|
database_id=self.database_id,
|
|
297
|
-
project_id=self.project_id or hook.project_id,
|
|
299
|
+
project_id=self.project_id or self.hook.project_id,
|
|
298
300
|
)
|
|
301
|
+
return result_rows_count_per_query
|
|
299
302
|
|
|
300
303
|
@staticmethod
|
|
301
304
|
def sanitize_queries(queries: list[str]) -> None:
|
|
@@ -307,6 +310,17 @@ class SpannerQueryDatabaseInstanceOperator(GoogleCloudBaseOperator):
|
|
|
307
310
|
if queries and queries[-1] == "":
|
|
308
311
|
queries.pop()
|
|
309
312
|
|
|
313
|
+
def get_openlineage_facets_on_complete(self, task_instance) -> OperatorLineage | None:
|
|
314
|
+
"""Build a generic OpenLineage facet, aligned with SQL-based operators."""
|
|
315
|
+
from airflow.providers.common.compat.openlineage.utils.sql import get_openlineage_facets_with_sql
|
|
316
|
+
|
|
317
|
+
return get_openlineage_facets_with_sql(
|
|
318
|
+
hook=self.hook,
|
|
319
|
+
sql=self.query,
|
|
320
|
+
conn_id=self.gcp_conn_id,
|
|
321
|
+
database=self.database_id,
|
|
322
|
+
)
|
|
323
|
+
|
|
310
324
|
|
|
311
325
|
class SpannerDeployDatabaseInstanceOperator(GoogleCloudBaseOperator):
|
|
312
326
|
"""
|
|
@@ -381,7 +395,6 @@ class SpannerDeployDatabaseInstanceOperator(GoogleCloudBaseOperator):
|
|
|
381
395
|
)
|
|
382
396
|
SpannerDatabaseLink.persist(
|
|
383
397
|
context=context,
|
|
384
|
-
task_instance=self,
|
|
385
398
|
instance_id=self.instance_id,
|
|
386
399
|
database_id=self.database_id,
|
|
387
400
|
project_id=self.project_id or hook.project_id,
|
|
@@ -401,14 +414,12 @@ class SpannerDeployDatabaseInstanceOperator(GoogleCloudBaseOperator):
|
|
|
401
414
|
database_id=self.database_id,
|
|
402
415
|
ddl_statements=self.ddl_statements,
|
|
403
416
|
)
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
self.instance_id,
|
|
411
|
-
)
|
|
417
|
+
self.log.info(
|
|
418
|
+
"The database '%s' in project '%s' and instance '%s' already exists. Nothing to do. Exiting.",
|
|
419
|
+
self.database_id,
|
|
420
|
+
self.project_id,
|
|
421
|
+
self.instance_id,
|
|
422
|
+
)
|
|
412
423
|
return True
|
|
413
424
|
|
|
414
425
|
|
|
@@ -497,21 +508,19 @@ class SpannerUpdateDatabaseInstanceOperator(GoogleCloudBaseOperator):
|
|
|
497
508
|
f"and instance '{self.instance_id}' is missing. "
|
|
498
509
|
f"Create the database first before you can update it."
|
|
499
510
|
)
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
operation_id=self.operation_id,
|
|
514
|
-
)
|
|
511
|
+
SpannerDatabaseLink.persist(
|
|
512
|
+
context=context,
|
|
513
|
+
instance_id=self.instance_id,
|
|
514
|
+
database_id=self.database_id,
|
|
515
|
+
project_id=self.project_id or hook.project_id,
|
|
516
|
+
)
|
|
517
|
+
return hook.update_database(
|
|
518
|
+
project_id=self.project_id,
|
|
519
|
+
instance_id=self.instance_id,
|
|
520
|
+
database_id=self.database_id,
|
|
521
|
+
ddl_statements=self.ddl_statements,
|
|
522
|
+
operation_id=self.operation_id,
|
|
523
|
+
)
|
|
515
524
|
|
|
516
525
|
|
|
517
526
|
class SpannerDeleteDatabaseInstanceOperator(GoogleCloudBaseOperator):
|
|
@@ -590,7 +599,6 @@ class SpannerDeleteDatabaseInstanceOperator(GoogleCloudBaseOperator):
|
|
|
590
599
|
self.instance_id,
|
|
591
600
|
)
|
|
592
601
|
return True
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
)
|
|
602
|
+
return hook.delete_database(
|
|
603
|
+
project_id=self.project_id, instance_id=self.instance_id, database_id=self.database_id
|
|
604
|
+
)
|
|
@@ -22,19 +22,21 @@ from __future__ import annotations
|
|
|
22
22
|
from collections.abc import Sequence
|
|
23
23
|
from typing import TYPE_CHECKING
|
|
24
24
|
|
|
25
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
26
|
+
from google.protobuf.json_format import MessageToDict
|
|
27
|
+
|
|
25
28
|
from airflow.exceptions import AirflowException
|
|
26
29
|
from airflow.providers.google.cloud.hooks.speech_to_text import CloudSpeechToTextHook, RecognitionAudio
|
|
27
30
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
28
31
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
29
32
|
from airflow.providers.google.common.links.storage import FileDetailsLink
|
|
30
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
31
|
-
from google.protobuf.json_format import MessageToDict
|
|
32
33
|
|
|
33
34
|
if TYPE_CHECKING:
|
|
34
|
-
from airflow.utils.context import Context
|
|
35
35
|
from google.api_core.retry import Retry
|
|
36
36
|
from google.cloud.speech_v1.types import RecognitionConfig
|
|
37
37
|
|
|
38
|
+
from airflow.providers.common.compat.sdk import Context
|
|
39
|
+
|
|
38
40
|
|
|
39
41
|
class CloudSpeechToTextRecognizeSpeechOperator(GoogleCloudBaseOperator):
|
|
40
42
|
"""
|
|
@@ -115,7 +117,6 @@ class CloudSpeechToTextRecognizeSpeechOperator(GoogleCloudBaseOperator):
|
|
|
115
117
|
if self.audio.uri:
|
|
116
118
|
FileDetailsLink.persist(
|
|
117
119
|
context=context,
|
|
118
|
-
task_instance=self,
|
|
119
120
|
# Slice from: "gs://{BUCKET_NAME}/{FILE_NAME}" to: "{BUCKET_NAME}/{FILE_NAME}"
|
|
120
121
|
uri=self.audio.uri[5:],
|
|
121
122
|
project_id=self.project_id or hook.project_id,
|
|
@@ -20,6 +20,9 @@ from __future__ import annotations
|
|
|
20
20
|
from collections.abc import Sequence
|
|
21
21
|
from typing import TYPE_CHECKING
|
|
22
22
|
|
|
23
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
24
|
+
from google.cloud.monitoring_v3 import AlertPolicy, NotificationChannel
|
|
25
|
+
|
|
23
26
|
from airflow.providers.google.cloud.hooks.stackdriver import StackdriverHook
|
|
24
27
|
from airflow.providers.google.cloud.links.stackdriver import (
|
|
25
28
|
StackdriverNotificationsLink,
|
|
@@ -27,13 +30,12 @@ from airflow.providers.google.cloud.links.stackdriver import (
|
|
|
27
30
|
)
|
|
28
31
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
29
32
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
30
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
31
|
-
from google.cloud.monitoring_v3 import AlertPolicy, NotificationChannel
|
|
32
33
|
|
|
33
34
|
if TYPE_CHECKING:
|
|
34
|
-
from airflow.utils.context import Context
|
|
35
35
|
from google.api_core.retry import Retry
|
|
36
36
|
|
|
37
|
+
from airflow.providers.common.compat.sdk import Context
|
|
38
|
+
|
|
37
39
|
|
|
38
40
|
class StackdriverListAlertPoliciesOperator(GoogleCloudBaseOperator):
|
|
39
41
|
"""
|
|
@@ -143,7 +145,6 @@ class StackdriverListAlertPoliciesOperator(GoogleCloudBaseOperator):
|
|
|
143
145
|
)
|
|
144
146
|
StackdriverPoliciesLink.persist(
|
|
145
147
|
context=context,
|
|
146
|
-
operator_instance=self,
|
|
147
148
|
project_id=self.project_id or self.hook.project_id,
|
|
148
149
|
)
|
|
149
150
|
return [AlertPolicy.to_dict(policy) for policy in result]
|
|
@@ -226,7 +227,6 @@ class StackdriverEnableAlertPoliciesOperator(GoogleCloudBaseOperator):
|
|
|
226
227
|
)
|
|
227
228
|
StackdriverPoliciesLink.persist(
|
|
228
229
|
context=context,
|
|
229
|
-
operator_instance=self,
|
|
230
230
|
project_id=self.project_id or self.hook.project_id,
|
|
231
231
|
)
|
|
232
232
|
|
|
@@ -309,7 +309,6 @@ class StackdriverDisableAlertPoliciesOperator(GoogleCloudBaseOperator):
|
|
|
309
309
|
)
|
|
310
310
|
StackdriverPoliciesLink.persist(
|
|
311
311
|
context=context,
|
|
312
|
-
operator_instance=self,
|
|
313
312
|
project_id=self.project_id or self.hook.project_id,
|
|
314
313
|
)
|
|
315
314
|
|
|
@@ -392,7 +391,6 @@ class StackdriverUpsertAlertOperator(GoogleCloudBaseOperator):
|
|
|
392
391
|
)
|
|
393
392
|
StackdriverPoliciesLink.persist(
|
|
394
393
|
context=context,
|
|
395
|
-
operator_instance=self,
|
|
396
394
|
project_id=self.project_id or self.hook.project_id,
|
|
397
395
|
)
|
|
398
396
|
|
|
@@ -578,7 +576,6 @@ class StackdriverListNotificationChannelsOperator(GoogleCloudBaseOperator):
|
|
|
578
576
|
)
|
|
579
577
|
StackdriverNotificationsLink.persist(
|
|
580
578
|
context=context,
|
|
581
|
-
operator_instance=self,
|
|
582
579
|
project_id=self.project_id or self.hook.project_id,
|
|
583
580
|
)
|
|
584
581
|
return [NotificationChannel.to_dict(channel) for channel in channels]
|
|
@@ -664,7 +661,6 @@ class StackdriverEnableNotificationChannelsOperator(GoogleCloudBaseOperator):
|
|
|
664
661
|
)
|
|
665
662
|
StackdriverNotificationsLink.persist(
|
|
666
663
|
context=context,
|
|
667
|
-
operator_instance=self,
|
|
668
664
|
project_id=self.project_id or self.hook.project_id,
|
|
669
665
|
)
|
|
670
666
|
|
|
@@ -749,7 +745,6 @@ class StackdriverDisableNotificationChannelsOperator(GoogleCloudBaseOperator):
|
|
|
749
745
|
)
|
|
750
746
|
StackdriverNotificationsLink.persist(
|
|
751
747
|
context=context,
|
|
752
|
-
operator_instance=self,
|
|
753
748
|
project_id=self.project_id or self.hook.project_id,
|
|
754
749
|
)
|
|
755
750
|
|
|
@@ -836,7 +831,6 @@ class StackdriverUpsertNotificationChannelOperator(GoogleCloudBaseOperator):
|
|
|
836
831
|
)
|
|
837
832
|
StackdriverNotificationsLink.persist(
|
|
838
833
|
context=context,
|
|
839
|
-
operator_instance=self,
|
|
840
834
|
project_id=self.project_id or self.hook.project_id,
|
|
841
835
|
)
|
|
842
836
|
|
|
@@ -22,19 +22,21 @@ from __future__ import annotations
|
|
|
22
22
|
from collections.abc import Sequence
|
|
23
23
|
from typing import TYPE_CHECKING
|
|
24
24
|
|
|
25
|
+
from google.api_core.exceptions import AlreadyExists
|
|
26
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
27
|
+
from google.cloud.tasks_v2.types import Queue, Task
|
|
28
|
+
|
|
25
29
|
from airflow.providers.google.cloud.hooks.tasks import CloudTasksHook
|
|
26
30
|
from airflow.providers.google.cloud.links.cloud_tasks import CloudTasksLink, CloudTasksQueueLink
|
|
27
31
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
28
32
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
29
|
-
from google.api_core.exceptions import AlreadyExists
|
|
30
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
31
|
-
from google.cloud.tasks_v2.types import Queue, Task
|
|
32
33
|
|
|
33
34
|
if TYPE_CHECKING:
|
|
34
|
-
from airflow.utils.context import Context
|
|
35
35
|
from google.api_core.retry import Retry
|
|
36
36
|
from google.protobuf.field_mask_pb2 import FieldMask
|
|
37
37
|
|
|
38
|
+
from airflow.providers.common.compat.sdk import Context
|
|
39
|
+
|
|
38
40
|
|
|
39
41
|
MetaData = Sequence[tuple[str, str]]
|
|
40
42
|
|
|
@@ -135,7 +137,6 @@ class CloudTasksQueueCreateOperator(GoogleCloudBaseOperator):
|
|
|
135
137
|
metadata=self.metadata,
|
|
136
138
|
)
|
|
137
139
|
CloudTasksQueueLink.persist(
|
|
138
|
-
operator_instance=self,
|
|
139
140
|
context=context,
|
|
140
141
|
queue_name=queue.name,
|
|
141
142
|
)
|
|
@@ -234,7 +235,6 @@ class CloudTasksQueueUpdateOperator(GoogleCloudBaseOperator):
|
|
|
234
235
|
metadata=self.metadata,
|
|
235
236
|
)
|
|
236
237
|
CloudTasksQueueLink.persist(
|
|
237
|
-
operator_instance=self,
|
|
238
238
|
context=context,
|
|
239
239
|
queue_name=queue.name,
|
|
240
240
|
)
|
|
@@ -317,7 +317,6 @@ class CloudTasksQueueGetOperator(GoogleCloudBaseOperator):
|
|
|
317
317
|
metadata=self.metadata,
|
|
318
318
|
)
|
|
319
319
|
CloudTasksQueueLink.persist(
|
|
320
|
-
operator_instance=self,
|
|
321
320
|
context=context,
|
|
322
321
|
queue_name=queue.name,
|
|
323
322
|
)
|
|
@@ -404,7 +403,6 @@ class CloudTasksQueuesListOperator(GoogleCloudBaseOperator):
|
|
|
404
403
|
metadata=self.metadata,
|
|
405
404
|
)
|
|
406
405
|
CloudTasksLink.persist(
|
|
407
|
-
operator_instance=self,
|
|
408
406
|
context=context,
|
|
409
407
|
project_id=self.project_id or hook.project_id,
|
|
410
408
|
)
|
|
@@ -562,7 +560,6 @@ class CloudTasksQueuePurgeOperator(GoogleCloudBaseOperator):
|
|
|
562
560
|
metadata=self.metadata,
|
|
563
561
|
)
|
|
564
562
|
CloudTasksQueueLink.persist(
|
|
565
|
-
operator_instance=self,
|
|
566
563
|
context=context,
|
|
567
564
|
queue_name=queue.name,
|
|
568
565
|
)
|
|
@@ -645,7 +642,6 @@ class CloudTasksQueuePauseOperator(GoogleCloudBaseOperator):
|
|
|
645
642
|
metadata=self.metadata,
|
|
646
643
|
)
|
|
647
644
|
CloudTasksQueueLink.persist(
|
|
648
|
-
operator_instance=self,
|
|
649
645
|
context=context,
|
|
650
646
|
queue_name=queue.name,
|
|
651
647
|
)
|
|
@@ -728,7 +724,6 @@ class CloudTasksQueueResumeOperator(GoogleCloudBaseOperator):
|
|
|
728
724
|
metadata=self.metadata,
|
|
729
725
|
)
|
|
730
726
|
CloudTasksQueueLink.persist(
|
|
731
|
-
operator_instance=self,
|
|
732
727
|
context=context,
|
|
733
728
|
queue_name=queue.name,
|
|
734
729
|
)
|
|
@@ -828,7 +823,6 @@ class CloudTasksTaskCreateOperator(GoogleCloudBaseOperator):
|
|
|
828
823
|
metadata=self.metadata,
|
|
829
824
|
)
|
|
830
825
|
CloudTasksQueueLink.persist(
|
|
831
|
-
operator_instance=self,
|
|
832
826
|
context=context,
|
|
833
827
|
queue_name=task.name,
|
|
834
828
|
)
|
|
@@ -921,7 +915,6 @@ class CloudTasksTaskGetOperator(GoogleCloudBaseOperator):
|
|
|
921
915
|
metadata=self.metadata,
|
|
922
916
|
)
|
|
923
917
|
CloudTasksQueueLink.persist(
|
|
924
|
-
operator_instance=self,
|
|
925
918
|
context=context,
|
|
926
919
|
queue_name=task.name,
|
|
927
920
|
)
|
|
@@ -1014,7 +1007,6 @@ class CloudTasksTasksListOperator(GoogleCloudBaseOperator):
|
|
|
1014
1007
|
metadata=self.metadata,
|
|
1015
1008
|
)
|
|
1016
1009
|
CloudTasksQueueLink.persist(
|
|
1017
|
-
operator_instance=self,
|
|
1018
1010
|
context=context,
|
|
1019
1011
|
queue_name=f"projects/{self.project_id or hook.project_id}/"
|
|
1020
1012
|
f"locations/{self.location}/queues/{self.queue_name}",
|
|
@@ -1188,7 +1180,6 @@ class CloudTasksTaskRunOperator(GoogleCloudBaseOperator):
|
|
|
1188
1180
|
metadata=self.metadata,
|
|
1189
1181
|
)
|
|
1190
1182
|
CloudTasksQueueLink.persist(
|
|
1191
|
-
operator_instance=self,
|
|
1192
1183
|
context=context,
|
|
1193
1184
|
queue_name=task.name,
|
|
1194
1185
|
)
|
|
@@ -23,19 +23,21 @@ from collections.abc import Sequence
|
|
|
23
23
|
from tempfile import NamedTemporaryFile
|
|
24
24
|
from typing import TYPE_CHECKING
|
|
25
25
|
|
|
26
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
27
|
+
|
|
26
28
|
from airflow.exceptions import AirflowException
|
|
27
29
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
|
28
30
|
from airflow.providers.google.cloud.hooks.text_to_speech import CloudTextToSpeechHook
|
|
29
31
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
30
32
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
31
33
|
from airflow.providers.google.common.links.storage import FileDetailsLink
|
|
32
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
33
34
|
|
|
34
35
|
if TYPE_CHECKING:
|
|
35
|
-
from airflow.utils.context import Context
|
|
36
36
|
from google.api_core.retry import Retry
|
|
37
37
|
from google.cloud.texttospeech_v1.types import AudioConfig, SynthesisInput, VoiceSelectionParams
|
|
38
38
|
|
|
39
|
+
from airflow.providers.common.compat.sdk import Context
|
|
40
|
+
|
|
39
41
|
|
|
40
42
|
class CloudTextToSpeechSynthesizeOperator(GoogleCloudBaseOperator):
|
|
41
43
|
"""
|
|
@@ -148,7 +150,6 @@ class CloudTextToSpeechSynthesizeOperator(GoogleCloudBaseOperator):
|
|
|
148
150
|
)
|
|
149
151
|
FileDetailsLink.persist(
|
|
150
152
|
context=context,
|
|
151
|
-
task_instance=self,
|
|
152
153
|
uri=f"{self.target_bucket_name}/{self.target_filename}",
|
|
153
154
|
project_id=cloud_storage_hook.project_id,
|
|
154
155
|
)
|
|
@@ -22,6 +22,9 @@ from __future__ import annotations
|
|
|
22
22
|
from collections.abc import MutableMapping, MutableSequence, Sequence
|
|
23
23
|
from typing import TYPE_CHECKING, cast
|
|
24
24
|
|
|
25
|
+
from google.api_core.exceptions import GoogleAPICallError
|
|
26
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
27
|
+
|
|
25
28
|
from airflow.exceptions import AirflowException
|
|
26
29
|
from airflow.providers.google.cloud.hooks.translate import CloudTranslateHook, TranslateHook
|
|
27
30
|
from airflow.providers.google.cloud.links.translate import (
|
|
@@ -34,12 +37,10 @@ from airflow.providers.google.cloud.links.translate import (
|
|
|
34
37
|
TranslationNativeDatasetLink,
|
|
35
38
|
)
|
|
36
39
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
|
40
|
+
from airflow.providers.google.cloud.operators.vertex_ai.dataset import DatasetImportDataResultsCheckHelper
|
|
37
41
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
|
38
|
-
from google.api_core.exceptions import GoogleAPICallError
|
|
39
|
-
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
40
42
|
|
|
41
43
|
if TYPE_CHECKING:
|
|
42
|
-
from airflow.utils.context import Context
|
|
43
44
|
from google.api_core.retry import Retry
|
|
44
45
|
from google.cloud.translate_v3.types import (
|
|
45
46
|
BatchDocumentInputConfig,
|
|
@@ -55,6 +56,8 @@ if TYPE_CHECKING:
|
|
|
55
56
|
)
|
|
56
57
|
from google.cloud.translate_v3.types.translation_service import Glossary, GlossaryInputConfig
|
|
57
58
|
|
|
59
|
+
from airflow.providers.common.compat.sdk import Context
|
|
60
|
+
|
|
58
61
|
|
|
59
62
|
class CloudTranslateTextOperator(GoogleCloudBaseOperator):
|
|
60
63
|
"""
|
|
@@ -392,7 +395,6 @@ class TranslateTextBatchOperator(GoogleCloudBaseOperator):
|
|
|
392
395
|
self.log.info("Translate text batch job started.")
|
|
393
396
|
TranslateTextBatchLink.persist(
|
|
394
397
|
context=context,
|
|
395
|
-
task_instance=self,
|
|
396
398
|
project_id=self.project_id or hook.project_id,
|
|
397
399
|
output_config=self.output_config,
|
|
398
400
|
)
|
|
@@ -478,15 +480,15 @@ class TranslateCreateDatasetOperator(GoogleCloudBaseOperator):
|
|
|
478
480
|
result = hook.wait_for_operation_result(result_operation)
|
|
479
481
|
result = type(result).to_dict(result)
|
|
480
482
|
dataset_id = hook.extract_object_id(result)
|
|
481
|
-
|
|
483
|
+
context["ti"].xcom_push(key="dataset_id", value=dataset_id)
|
|
482
484
|
self.log.info("Dataset creation complete. The dataset_id: %s.", dataset_id)
|
|
483
485
|
|
|
484
486
|
project_id = self.project_id or hook.project_id
|
|
485
487
|
TranslationNativeDatasetLink.persist(
|
|
486
488
|
context=context,
|
|
487
|
-
task_instance=self,
|
|
488
489
|
dataset_id=dataset_id,
|
|
489
490
|
project_id=project_id,
|
|
491
|
+
location=self.location,
|
|
490
492
|
)
|
|
491
493
|
return result
|
|
492
494
|
|
|
@@ -554,7 +556,6 @@ class TranslateDatasetsListOperator(GoogleCloudBaseOperator):
|
|
|
554
556
|
project_id = self.project_id or hook.project_id
|
|
555
557
|
TranslationDatasetsListLink.persist(
|
|
556
558
|
context=context,
|
|
557
|
-
task_instance=self,
|
|
558
559
|
project_id=project_id,
|
|
559
560
|
)
|
|
560
561
|
self.log.info("Requesting datasets list")
|
|
@@ -575,7 +576,7 @@ class TranslateDatasetsListOperator(GoogleCloudBaseOperator):
|
|
|
575
576
|
return result_ids
|
|
576
577
|
|
|
577
578
|
|
|
578
|
-
class TranslateImportDataOperator(GoogleCloudBaseOperator):
|
|
579
|
+
class TranslateImportDataOperator(GoogleCloudBaseOperator, DatasetImportDataResultsCheckHelper):
|
|
579
580
|
"""
|
|
580
581
|
Import data to the translation dataset.
|
|
581
582
|
|
|
@@ -602,6 +603,7 @@ class TranslateImportDataOperator(GoogleCloudBaseOperator):
|
|
|
602
603
|
If set as a sequence, the identities from the list must grant
|
|
603
604
|
Service Account Token Creator IAM role to the directly preceding identity, with first
|
|
604
605
|
account from the list granting this role to the originating account (templated).
|
|
606
|
+
:param raise_for_empty_result: Raise an error if no additional data has been populated after the import.
|
|
605
607
|
"""
|
|
606
608
|
|
|
607
609
|
template_fields: Sequence[str] = (
|
|
@@ -627,6 +629,7 @@ class TranslateImportDataOperator(GoogleCloudBaseOperator):
|
|
|
627
629
|
retry: Retry | _MethodDefault = DEFAULT,
|
|
628
630
|
gcp_conn_id: str = "google_cloud_default",
|
|
629
631
|
impersonation_chain: str | Sequence[str] | None = None,
|
|
632
|
+
raise_for_empty_result: bool = False,
|
|
630
633
|
**kwargs,
|
|
631
634
|
) -> None:
|
|
632
635
|
super().__init__(**kwargs)
|
|
@@ -639,9 +642,21 @@ class TranslateImportDataOperator(GoogleCloudBaseOperator):
|
|
|
639
642
|
self.retry = retry
|
|
640
643
|
self.gcp_conn_id = gcp_conn_id
|
|
641
644
|
self.impersonation_chain = impersonation_chain
|
|
645
|
+
self.raise_for_empty_result = raise_for_empty_result
|
|
642
646
|
|
|
643
647
|
def execute(self, context: Context):
|
|
644
648
|
hook = TranslateHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
|
|
649
|
+
initial_dataset_size = self._get_number_of_ds_items(
|
|
650
|
+
dataset=hook.get_dataset(
|
|
651
|
+
dataset_id=self.dataset_id,
|
|
652
|
+
project_id=self.project_id,
|
|
653
|
+
location=self.location,
|
|
654
|
+
retry=self.retry,
|
|
655
|
+
timeout=self.timeout,
|
|
656
|
+
metadata=self.metadata,
|
|
657
|
+
),
|
|
658
|
+
total_key_name="example_count",
|
|
659
|
+
)
|
|
645
660
|
self.log.info("Importing data to dataset...")
|
|
646
661
|
operation = hook.import_dataset_data(
|
|
647
662
|
dataset_id=self.dataset_id,
|
|
@@ -655,12 +670,27 @@ class TranslateImportDataOperator(GoogleCloudBaseOperator):
|
|
|
655
670
|
project_id = self.project_id or hook.project_id
|
|
656
671
|
TranslationNativeDatasetLink.persist(
|
|
657
672
|
context=context,
|
|
658
|
-
task_instance=self,
|
|
659
673
|
dataset_id=self.dataset_id,
|
|
660
674
|
project_id=project_id,
|
|
675
|
+
location=self.location,
|
|
661
676
|
)
|
|
662
677
|
hook.wait_for_operation_done(operation=operation, timeout=self.timeout)
|
|
678
|
+
|
|
679
|
+
result_dataset_size = self._get_number_of_ds_items(
|
|
680
|
+
dataset=hook.get_dataset(
|
|
681
|
+
dataset_id=self.dataset_id,
|
|
682
|
+
project_id=self.project_id,
|
|
683
|
+
location=self.location,
|
|
684
|
+
retry=self.retry,
|
|
685
|
+
timeout=self.timeout,
|
|
686
|
+
metadata=self.metadata,
|
|
687
|
+
),
|
|
688
|
+
total_key_name="example_count",
|
|
689
|
+
)
|
|
690
|
+
if self.raise_for_empty_result:
|
|
691
|
+
self._raise_for_empty_import_result(self.dataset_id, initial_dataset_size, result_dataset_size)
|
|
663
692
|
self.log.info("Importing data finished!")
|
|
693
|
+
return {"total_imported": int(result_dataset_size) - int(initial_dataset_size)}
|
|
664
694
|
|
|
665
695
|
|
|
666
696
|
class TranslateDeleteDatasetOperator(GoogleCloudBaseOperator):
|
|
@@ -819,16 +849,16 @@ class TranslateCreateModelOperator(GoogleCloudBaseOperator):
|
|
|
819
849
|
result = hook.wait_for_operation_result(operation=result_operation)
|
|
820
850
|
result = type(result).to_dict(result)
|
|
821
851
|
model_id = hook.extract_object_id(result)
|
|
822
|
-
|
|
852
|
+
context["ti"].xcom_push(key="model_id", value=model_id)
|
|
823
853
|
self.log.info("Model creation complete. The model_id: %s.", model_id)
|
|
824
854
|
|
|
825
855
|
project_id = self.project_id or hook.project_id
|
|
826
856
|
TranslationModelLink.persist(
|
|
827
857
|
context=context,
|
|
828
|
-
task_instance=self,
|
|
829
858
|
dataset_id=self.dataset_id,
|
|
830
859
|
model_id=model_id,
|
|
831
860
|
project_id=project_id,
|
|
861
|
+
location=self.location,
|
|
832
862
|
)
|
|
833
863
|
return result
|
|
834
864
|
|
|
@@ -896,7 +926,6 @@ class TranslateModelsListOperator(GoogleCloudBaseOperator):
|
|
|
896
926
|
project_id = self.project_id or hook.project_id
|
|
897
927
|
TranslationModelsListLink.persist(
|
|
898
928
|
context=context,
|
|
899
|
-
task_instance=self,
|
|
900
929
|
project_id=project_id,
|
|
901
930
|
)
|
|
902
931
|
self.log.info("Requesting models list")
|
|
@@ -1139,11 +1168,10 @@ class TranslateDocumentOperator(GoogleCloudBaseOperator):
|
|
|
1139
1168
|
if self.document_output_config:
|
|
1140
1169
|
TranslateResultByOutputConfigLink.persist(
|
|
1141
1170
|
context=context,
|
|
1142
|
-
task_instance=self,
|
|
1143
1171
|
project_id=self.project_id or hook.project_id,
|
|
1144
1172
|
output_config=self.document_output_config,
|
|
1145
1173
|
)
|
|
1146
|
-
return cast(dict, type(doc_translation_result).to_dict(doc_translation_result))
|
|
1174
|
+
return cast("dict", type(doc_translation_result).to_dict(doc_translation_result))
|
|
1147
1175
|
|
|
1148
1176
|
|
|
1149
1177
|
class TranslateDocumentBatchOperator(GoogleCloudBaseOperator):
|
|
@@ -1302,13 +1330,12 @@ class TranslateDocumentBatchOperator(GoogleCloudBaseOperator):
|
|
|
1302
1330
|
self.log.info("Batch document translation job started.")
|
|
1303
1331
|
TranslateResultByOutputConfigLink.persist(
|
|
1304
1332
|
context=context,
|
|
1305
|
-
task_instance=self,
|
|
1306
1333
|
project_id=self.project_id or hook.project_id,
|
|
1307
1334
|
output_config=self.output_config,
|
|
1308
1335
|
)
|
|
1309
1336
|
result = hook.wait_for_operation_result(batch_document_translate_operation)
|
|
1310
1337
|
self.log.info("Batch document translation job finished")
|
|
1311
|
-
return cast(dict, type(result).to_dict(result))
|
|
1338
|
+
return cast("dict", type(result).to_dict(result))
|
|
1312
1339
|
|
|
1313
1340
|
|
|
1314
1341
|
class TranslateCreateGlossaryOperator(GoogleCloudBaseOperator):
|
|
@@ -1409,7 +1436,7 @@ class TranslateCreateGlossaryOperator(GoogleCloudBaseOperator):
|
|
|
1409
1436
|
result = type(result).to_dict(result)
|
|
1410
1437
|
|
|
1411
1438
|
glossary_id = hook.extract_object_id(result)
|
|
1412
|
-
|
|
1439
|
+
context["ti"].xcom_push(key="glossary_id", value=glossary_id)
|
|
1413
1440
|
self.log.info("Glossary creation complete. The glossary_id: %s.", glossary_id)
|
|
1414
1441
|
return result
|
|
1415
1442
|
|
|
@@ -1608,7 +1635,6 @@ class TranslateListGlossariesOperator(GoogleCloudBaseOperator):
|
|
|
1608
1635
|
project_id = self.project_id or hook.project_id
|
|
1609
1636
|
TranslationGlossariesListLink.persist(
|
|
1610
1637
|
context=context,
|
|
1611
|
-
task_instance=self,
|
|
1612
1638
|
project_id=project_id,
|
|
1613
1639
|
)
|
|
1614
1640
|
self.log.info("Requesting glossaries list")
|
|
@@ -1628,8 +1654,8 @@ class TranslateListGlossariesOperator(GoogleCloudBaseOperator):
|
|
|
1628
1654
|
raise AirflowException(e)
|
|
1629
1655
|
|
|
1630
1656
|
result_ids = []
|
|
1631
|
-
for
|
|
1632
|
-
glossary_item = type(
|
|
1657
|
+
for glossary_item_raw in results_pager:
|
|
1658
|
+
glossary_item = type(glossary_item_raw).to_dict(glossary_item_raw)
|
|
1633
1659
|
glossary_id = hook.extract_object_id(glossary_item)
|
|
1634
1660
|
result_ids.append(glossary_id)
|
|
1635
1661
|
self.log.info("Fetching the glossaries list complete. Glossary id-s: %s", result_ids)
|