apache-airflow-providers-google 14.0.0__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (278) hide show
  1. airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
  2. airflow/providers/google/3rd-party-licenses/NOTICE +5 -0
  3. airflow/providers/google/__init__.py +3 -3
  4. airflow/providers/google/_vendor/__init__.py +0 -0
  5. airflow/providers/google/_vendor/json_merge_patch.py +91 -0
  6. airflow/providers/google/ads/hooks/ads.py +52 -43
  7. airflow/providers/google/ads/operators/ads.py +2 -2
  8. airflow/providers/google/ads/transfers/ads_to_gcs.py +3 -19
  9. airflow/providers/google/assets/gcs.py +1 -11
  10. airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
  11. airflow/providers/google/cloud/bundles/gcs.py +161 -0
  12. airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
  13. airflow/providers/google/cloud/hooks/bigquery.py +195 -318
  14. airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
  15. airflow/providers/google/cloud/hooks/bigtable.py +3 -2
  16. airflow/providers/google/cloud/hooks/cloud_batch.py +8 -9
  17. airflow/providers/google/cloud/hooks/cloud_build.py +6 -65
  18. airflow/providers/google/cloud/hooks/cloud_composer.py +292 -24
  19. airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
  20. airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
  21. airflow/providers/google/cloud/hooks/cloud_run.py +20 -11
  22. airflow/providers/google/cloud/hooks/cloud_sql.py +136 -64
  23. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +35 -15
  24. airflow/providers/google/cloud/hooks/compute.py +7 -6
  25. airflow/providers/google/cloud/hooks/compute_ssh.py +7 -4
  26. airflow/providers/google/cloud/hooks/datacatalog.py +12 -3
  27. airflow/providers/google/cloud/hooks/dataflow.py +87 -242
  28. airflow/providers/google/cloud/hooks/dataform.py +9 -14
  29. airflow/providers/google/cloud/hooks/datafusion.py +7 -9
  30. airflow/providers/google/cloud/hooks/dataplex.py +13 -12
  31. airflow/providers/google/cloud/hooks/dataprep.py +2 -2
  32. airflow/providers/google/cloud/hooks/dataproc.py +76 -74
  33. airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
  34. airflow/providers/google/cloud/hooks/dlp.py +5 -4
  35. airflow/providers/google/cloud/hooks/gcs.py +144 -33
  36. airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
  37. airflow/providers/google/cloud/hooks/kms.py +3 -2
  38. airflow/providers/google/cloud/hooks/kubernetes_engine.py +22 -17
  39. airflow/providers/google/cloud/hooks/looker.py +6 -1
  40. airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
  41. airflow/providers/google/cloud/hooks/mlengine.py +7 -8
  42. airflow/providers/google/cloud/hooks/natural_language.py +3 -2
  43. airflow/providers/google/cloud/hooks/os_login.py +3 -2
  44. airflow/providers/google/cloud/hooks/pubsub.py +6 -6
  45. airflow/providers/google/cloud/hooks/secret_manager.py +105 -12
  46. airflow/providers/google/cloud/hooks/spanner.py +75 -10
  47. airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
  48. airflow/providers/google/cloud/hooks/stackdriver.py +18 -18
  49. airflow/providers/google/cloud/hooks/tasks.py +4 -3
  50. airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
  51. airflow/providers/google/cloud/hooks/translate.py +8 -17
  52. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +8 -222
  53. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +9 -15
  54. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +33 -283
  55. airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
  56. airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +6 -12
  57. airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
  58. airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +311 -10
  59. airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
  60. airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +7 -13
  61. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +8 -12
  62. airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
  63. airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
  64. airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
  65. airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
  66. airflow/providers/google/cloud/hooks/vision.py +7 -7
  67. airflow/providers/google/cloud/hooks/workflows.py +4 -3
  68. airflow/providers/google/cloud/links/alloy_db.py +0 -46
  69. airflow/providers/google/cloud/links/base.py +77 -7
  70. airflow/providers/google/cloud/links/bigquery.py +0 -47
  71. airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
  72. airflow/providers/google/cloud/links/bigtable.py +0 -48
  73. airflow/providers/google/cloud/links/cloud_build.py +0 -73
  74. airflow/providers/google/cloud/links/cloud_functions.py +0 -33
  75. airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
  76. airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
  77. airflow/providers/google/cloud/links/cloud_sql.py +0 -33
  78. airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -46
  79. airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
  80. airflow/providers/google/cloud/links/compute.py +0 -58
  81. airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
  82. airflow/providers/google/cloud/links/datacatalog.py +23 -54
  83. airflow/providers/google/cloud/links/dataflow.py +0 -34
  84. airflow/providers/google/cloud/links/dataform.py +0 -64
  85. airflow/providers/google/cloud/links/datafusion.py +1 -90
  86. airflow/providers/google/cloud/links/dataplex.py +0 -154
  87. airflow/providers/google/cloud/links/dataprep.py +0 -24
  88. airflow/providers/google/cloud/links/dataproc.py +11 -89
  89. airflow/providers/google/cloud/links/datastore.py +0 -31
  90. airflow/providers/google/cloud/links/kubernetes_engine.py +11 -61
  91. airflow/providers/google/cloud/links/managed_kafka.py +11 -51
  92. airflow/providers/google/cloud/links/mlengine.py +0 -70
  93. airflow/providers/google/cloud/links/pubsub.py +0 -32
  94. airflow/providers/google/cloud/links/spanner.py +0 -33
  95. airflow/providers/google/cloud/links/stackdriver.py +0 -30
  96. airflow/providers/google/cloud/links/translate.py +17 -187
  97. airflow/providers/google/cloud/links/vertex_ai.py +28 -195
  98. airflow/providers/google/cloud/links/workflows.py +0 -52
  99. airflow/providers/google/cloud/log/gcs_task_handler.py +166 -118
  100. airflow/providers/google/cloud/log/stackdriver_task_handler.py +14 -9
  101. airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
  102. airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
  103. airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
  104. airflow/providers/google/cloud/openlineage/facets.py +141 -40
  105. airflow/providers/google/cloud/openlineage/mixins.py +14 -13
  106. airflow/providers/google/cloud/openlineage/utils.py +19 -3
  107. airflow/providers/google/cloud/operators/alloy_db.py +76 -61
  108. airflow/providers/google/cloud/operators/bigquery.py +104 -667
  109. airflow/providers/google/cloud/operators/bigquery_dts.py +12 -12
  110. airflow/providers/google/cloud/operators/bigtable.py +38 -7
  111. airflow/providers/google/cloud/operators/cloud_base.py +22 -1
  112. airflow/providers/google/cloud/operators/cloud_batch.py +18 -18
  113. airflow/providers/google/cloud/operators/cloud_build.py +80 -36
  114. airflow/providers/google/cloud/operators/cloud_composer.py +157 -71
  115. airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
  116. airflow/providers/google/cloud/operators/cloud_memorystore.py +74 -46
  117. airflow/providers/google/cloud/operators/cloud_run.py +39 -20
  118. airflow/providers/google/cloud/operators/cloud_sql.py +46 -61
  119. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +92 -14
  120. airflow/providers/google/cloud/operators/compute.py +18 -50
  121. airflow/providers/google/cloud/operators/datacatalog.py +167 -29
  122. airflow/providers/google/cloud/operators/dataflow.py +38 -15
  123. airflow/providers/google/cloud/operators/dataform.py +19 -7
  124. airflow/providers/google/cloud/operators/datafusion.py +43 -43
  125. airflow/providers/google/cloud/operators/dataplex.py +212 -126
  126. airflow/providers/google/cloud/operators/dataprep.py +1 -5
  127. airflow/providers/google/cloud/operators/dataproc.py +134 -207
  128. airflow/providers/google/cloud/operators/dataproc_metastore.py +102 -84
  129. airflow/providers/google/cloud/operators/datastore.py +22 -6
  130. airflow/providers/google/cloud/operators/dlp.py +24 -45
  131. airflow/providers/google/cloud/operators/functions.py +21 -14
  132. airflow/providers/google/cloud/operators/gcs.py +15 -12
  133. airflow/providers/google/cloud/operators/gen_ai.py +389 -0
  134. airflow/providers/google/cloud/operators/kubernetes_engine.py +115 -106
  135. airflow/providers/google/cloud/operators/looker.py +1 -1
  136. airflow/providers/google/cloud/operators/managed_kafka.py +362 -40
  137. airflow/providers/google/cloud/operators/natural_language.py +5 -3
  138. airflow/providers/google/cloud/operators/pubsub.py +69 -21
  139. airflow/providers/google/cloud/operators/spanner.py +53 -45
  140. airflow/providers/google/cloud/operators/speech_to_text.py +5 -4
  141. airflow/providers/google/cloud/operators/stackdriver.py +5 -11
  142. airflow/providers/google/cloud/operators/tasks.py +6 -15
  143. airflow/providers/google/cloud/operators/text_to_speech.py +4 -3
  144. airflow/providers/google/cloud/operators/translate.py +46 -20
  145. airflow/providers/google/cloud/operators/translate_speech.py +4 -3
  146. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +44 -34
  147. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +34 -12
  148. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +62 -53
  149. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +75 -11
  150. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +48 -12
  151. airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
  152. airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
  153. airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -116
  154. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +16 -12
  155. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +62 -14
  156. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +35 -10
  157. airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
  158. airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
  159. airflow/providers/google/cloud/operators/vision.py +7 -5
  160. airflow/providers/google/cloud/operators/workflows.py +24 -19
  161. airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
  162. airflow/providers/google/cloud/sensors/bigquery.py +2 -2
  163. airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -4
  164. airflow/providers/google/cloud/sensors/bigtable.py +14 -6
  165. airflow/providers/google/cloud/sensors/cloud_composer.py +535 -33
  166. airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -5
  167. airflow/providers/google/cloud/sensors/dataflow.py +27 -10
  168. airflow/providers/google/cloud/sensors/dataform.py +2 -2
  169. airflow/providers/google/cloud/sensors/datafusion.py +4 -4
  170. airflow/providers/google/cloud/sensors/dataplex.py +7 -5
  171. airflow/providers/google/cloud/sensors/dataprep.py +2 -2
  172. airflow/providers/google/cloud/sensors/dataproc.py +10 -9
  173. airflow/providers/google/cloud/sensors/dataproc_metastore.py +4 -3
  174. airflow/providers/google/cloud/sensors/gcs.py +22 -21
  175. airflow/providers/google/cloud/sensors/looker.py +5 -5
  176. airflow/providers/google/cloud/sensors/pubsub.py +20 -20
  177. airflow/providers/google/cloud/sensors/tasks.py +2 -2
  178. airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -2
  179. airflow/providers/google/cloud/sensors/workflows.py +6 -4
  180. airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
  181. airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
  182. airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -2
  183. airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
  184. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +14 -13
  185. airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
  186. airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
  187. airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
  188. airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
  189. airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
  190. airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +18 -22
  191. airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -5
  192. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +45 -38
  193. airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -2
  194. airflow/providers/google/cloud/transfers/gcs_to_local.py +5 -3
  195. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +10 -4
  196. airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
  197. airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
  198. airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
  199. airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
  200. airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
  201. airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
  202. airflow/providers/google/cloud/transfers/postgres_to_gcs.py +44 -12
  203. airflow/providers/google/cloud/transfers/s3_to_gcs.py +12 -6
  204. airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
  205. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +36 -14
  206. airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
  207. airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
  208. airflow/providers/google/cloud/triggers/bigquery.py +75 -34
  209. airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
  210. airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
  211. airflow/providers/google/cloud/triggers/cloud_build.py +3 -2
  212. airflow/providers/google/cloud/triggers/cloud_composer.py +303 -47
  213. airflow/providers/google/cloud/triggers/cloud_run.py +2 -2
  214. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +96 -5
  215. airflow/providers/google/cloud/triggers/dataflow.py +125 -2
  216. airflow/providers/google/cloud/triggers/datafusion.py +1 -1
  217. airflow/providers/google/cloud/triggers/dataplex.py +16 -3
  218. airflow/providers/google/cloud/triggers/dataproc.py +124 -53
  219. airflow/providers/google/cloud/triggers/kubernetes_engine.py +46 -28
  220. airflow/providers/google/cloud/triggers/mlengine.py +1 -1
  221. airflow/providers/google/cloud/triggers/pubsub.py +17 -20
  222. airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
  223. airflow/providers/google/cloud/utils/bigquery.py +5 -7
  224. airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
  225. airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
  226. airflow/providers/google/cloud/utils/dataform.py +1 -1
  227. airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
  228. airflow/providers/google/cloud/utils/field_validator.py +1 -2
  229. airflow/providers/google/cloud/utils/validators.py +43 -0
  230. airflow/providers/google/common/auth_backend/google_openid.py +26 -9
  231. airflow/providers/google/common/consts.py +2 -1
  232. airflow/providers/google/common/deprecated.py +2 -1
  233. airflow/providers/google/common/hooks/base_google.py +40 -43
  234. airflow/providers/google/common/hooks/operation_helpers.py +78 -0
  235. airflow/providers/google/common/links/storage.py +0 -22
  236. airflow/providers/google/common/utils/get_secret.py +31 -0
  237. airflow/providers/google/common/utils/id_token_credentials.py +4 -5
  238. airflow/providers/google/firebase/operators/firestore.py +2 -2
  239. airflow/providers/google/get_provider_info.py +61 -216
  240. airflow/providers/google/go_module_utils.py +35 -3
  241. airflow/providers/google/leveldb/hooks/leveldb.py +30 -6
  242. airflow/providers/google/leveldb/operators/leveldb.py +2 -2
  243. airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
  244. airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
  245. airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
  246. airflow/providers/google/marketing_platform/links/analytics_admin.py +4 -5
  247. airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -6
  248. airflow/providers/google/marketing_platform/operators/campaign_manager.py +5 -5
  249. airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
  250. airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
  251. airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
  252. airflow/providers/google/marketing_platform/sensors/display_video.py +4 -64
  253. airflow/providers/google/suite/hooks/calendar.py +1 -1
  254. airflow/providers/google/suite/hooks/drive.py +2 -2
  255. airflow/providers/google/suite/hooks/sheets.py +15 -1
  256. airflow/providers/google/suite/operators/sheets.py +8 -3
  257. airflow/providers/google/suite/sensors/drive.py +2 -2
  258. airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -2
  259. airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
  260. airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
  261. airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
  262. airflow/providers/google/version_compat.py +15 -1
  263. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/METADATA +117 -72
  264. apache_airflow_providers_google-19.1.0rc1.dist-info/RECORD +331 -0
  265. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/WHEEL +1 -1
  266. apache_airflow_providers_google-19.1.0rc1.dist-info/licenses/NOTICE +5 -0
  267. airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
  268. airflow/providers/google/cloud/hooks/automl.py +0 -679
  269. airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
  270. airflow/providers/google/cloud/links/automl.py +0 -193
  271. airflow/providers/google/cloud/operators/automl.py +0 -1360
  272. airflow/providers/google/cloud/operators/life_sciences.py +0 -119
  273. airflow/providers/google/cloud/operators/mlengine.py +0 -1515
  274. airflow/providers/google/cloud/utils/mlengine_operator_utils.py +0 -273
  275. apache_airflow_providers_google-14.0.0.dist-info/RECORD +0 -318
  276. /airflow/providers/google/cloud/{example_dags → bundles}/__init__.py +0 -0
  277. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/entry_points.txt +0 -0
  278. {airflow/providers/google → apache_airflow_providers_google-19.1.0rc1.dist-info/licenses}/LICENSE +0 -0
@@ -1,1360 +0,0 @@
1
- #
2
- # Licensed to the Apache Software Foundation (ASF) under one
3
- # or more contributor license agreements. See the NOTICE file
4
- # distributed with this work for additional information
5
- # regarding copyright ownership. The ASF licenses this file
6
- # to you under the Apache License, Version 2.0 (the
7
- # "License"); you may not use this file except in compliance
8
- # with the License. You may obtain a copy of the License at
9
- #
10
- # http://www.apache.org/licenses/LICENSE-2.0
11
- #
12
- # Unless required by applicable law or agreed to in writing,
13
- # software distributed under the License is distributed on an
14
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
- # KIND, either express or implied. See the License for the
16
- # specific language governing permissions and limitations
17
- # under the License.
18
- """This module contains Google AutoML operators."""
19
-
20
- from __future__ import annotations
21
-
22
- import ast
23
- from collections.abc import Sequence
24
- from functools import cached_property
25
- from typing import TYPE_CHECKING, cast
26
-
27
- from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
28
- from airflow.providers.google.cloud.hooks.automl import CloudAutoMLHook
29
- from airflow.providers.google.cloud.hooks.vertex_ai.prediction_service import PredictionServiceHook
30
- from airflow.providers.google.cloud.links.translate import (
31
- TranslationDatasetListLink,
32
- TranslationLegacyDatasetLink,
33
- TranslationLegacyModelLink,
34
- TranslationLegacyModelPredictLink,
35
- TranslationLegacyModelTrainLink,
36
- )
37
- from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
38
- from airflow.providers.google.common.deprecated import deprecated
39
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
40
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
41
- from google.cloud.automl_v1beta1 import (
42
- ColumnSpec,
43
- Dataset,
44
- Model,
45
- PredictResponse,
46
- TableSpec,
47
- )
48
-
49
- if TYPE_CHECKING:
50
- from airflow.utils.context import Context
51
- from google.api_core.retry import Retry
52
-
53
- MetaData = Sequence[tuple[str, str]]
54
-
55
-
56
- @deprecated(
57
- planned_removal_date="September 30, 2025",
58
- use_instead="airflow.providers.google.cloud.operators.vertex_ai.auto_ml.CreateAutoMLTabularTrainingJobOperator, "
59
- "airflow.providers.google.cloud.operators.vertex_ai.auto_ml.CreateAutoMLVideoTrainingJobOperator, "
60
- "airflow.providers.google.cloud.operators.vertex_ai.auto_ml.CreateAutoMLImageTrainingJobOperator, "
61
- "airflow.providers.google.cloud.operators.vertex_ai.generative_model.SupervisedFineTuningTrainOperator, "
62
- "airflow.providers.google.cloud.operators.translate.TranslateCreateModelOperator",
63
- category=AirflowProviderDeprecationWarning,
64
- )
65
- class AutoMLTrainModelOperator(GoogleCloudBaseOperator):
66
- """
67
- Creates Google Cloud AutoML model.
68
-
69
- .. warning::
70
- AutoMLTrainModelOperator for tables, video intelligence, vision and natural language has been deprecated
71
- and no longer available. Please use
72
- :class:`airflow.providers.google.cloud.operators.vertex_ai.auto_ml.CreateAutoMLTabularTrainingJobOperator`,
73
- :class:`airflow.providers.google.cloud.operators.vertex_ai.auto_ml.CreateAutoMLVideoTrainingJobOperator`,
74
- :class:`airflow.providers.google.cloud.operators.vertex_ai.auto_ml.CreateAutoMLImageTrainingJobOperator`,
75
- :class:`airflow.providers.google.cloud.operators.vertex_ai.generative_model.SupervisedFineTuningTrainOperator`,
76
- :class:`airflow.providers.google.cloud.operators.translate.TranslateCreateModelOperator`.
77
- instead.
78
-
79
- .. seealso::
80
- For more information on how to use this operator, take a look at the guide:
81
- :ref:`howto/operator:AutoMLTrainModelOperator`
82
-
83
- :param model: Model definition.
84
- :param project_id: ID of the Google Cloud project where model will be created if None then
85
- default project_id is used.
86
- :param location: The location of the project.
87
- :param retry: A retry object used to retry requests. If `None` is specified, requests will not be
88
- retried.
89
- :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
90
- `retry` is specified, the timeout applies to each individual attempt.
91
- :param metadata: Additional metadata that is provided to the method.
92
- :param gcp_conn_id: The connection ID to use to connect to Google Cloud.
93
- :param impersonation_chain: Optional service account to impersonate using short-term
94
- credentials, or chained list of accounts required to get the access_token
95
- of the last account in the list, which will be impersonated in the request.
96
- If set as a string, the account must grant the originating account
97
- the Service Account Token Creator IAM role.
98
- If set as a sequence, the identities from the list must grant
99
- Service Account Token Creator IAM role to the directly preceding identity, with first
100
- account from the list granting this role to the originating account (templated).
101
- """
102
-
103
- template_fields: Sequence[str] = (
104
- "model",
105
- "location",
106
- "project_id",
107
- "impersonation_chain",
108
- )
109
- operator_extra_links = (
110
- TranslationLegacyModelTrainLink(),
111
- TranslationLegacyModelLink(),
112
- )
113
-
114
- def __init__(
115
- self,
116
- *,
117
- model: dict,
118
- location: str,
119
- project_id: str = PROVIDE_PROJECT_ID,
120
- metadata: MetaData = (),
121
- timeout: float | None = None,
122
- retry: Retry | _MethodDefault = DEFAULT,
123
- gcp_conn_id: str = "google_cloud_default",
124
- impersonation_chain: str | Sequence[str] | None = None,
125
- **kwargs,
126
- ) -> None:
127
- super().__init__(**kwargs)
128
- self.model = model
129
- self.location = location
130
- self.project_id = project_id
131
- self.metadata = metadata
132
- self.timeout = timeout
133
- self.retry = retry
134
- self.gcp_conn_id = gcp_conn_id
135
- self.impersonation_chain = impersonation_chain
136
-
137
- def execute(self, context: Context):
138
- hook = CloudAutoMLHook(
139
- gcp_conn_id=self.gcp_conn_id,
140
- impersonation_chain=self.impersonation_chain,
141
- )
142
- self.log.info("Creating model %s...", self.model["display_name"])
143
- operation = hook.create_model(
144
- model=self.model,
145
- location=self.location,
146
- project_id=self.project_id,
147
- retry=self.retry,
148
- timeout=self.timeout,
149
- metadata=self.metadata,
150
- )
151
- project_id = self.project_id or hook.project_id
152
- if project_id:
153
- TranslationLegacyModelTrainLink.persist(
154
- context=context, task_instance=self, project_id=project_id
155
- )
156
- operation_result = hook.wait_for_operation(timeout=self.timeout, operation=operation)
157
- result = Model.to_dict(operation_result)
158
- model_id = hook.extract_object_id(result)
159
- self.log.info("Model is created, model_id: %s", model_id)
160
-
161
- self.xcom_push(context, key="model_id", value=model_id)
162
- if project_id:
163
- TranslationLegacyModelLink.persist(
164
- context=context,
165
- task_instance=self,
166
- dataset_id=self.model["dataset_id"] or "-",
167
- model_id=model_id,
168
- project_id=project_id,
169
- )
170
- return result
171
-
172
-
173
- @deprecated(
174
- planned_removal_date="September 30, 2025",
175
- use_instead="airflow.providers.google.cloud.operators.translate.TranslateTextOperator",
176
- category=AirflowProviderDeprecationWarning,
177
- )
178
- class AutoMLPredictOperator(GoogleCloudBaseOperator):
179
- """
180
- Runs prediction operation on Google Cloud AutoML.
181
-
182
- .. warning::
183
- AutoMLPredictOperator for text, image, and video prediction has been deprecated.
184
- Please use endpoint_id param instead of model_id param.
185
-
186
- .. seealso::
187
- For more information on how to use this operator, take a look at the guide:
188
- :ref:`howto/operator:AutoMLPredictOperator`
189
-
190
- :param model_id: Name of the model requested to serve the batch prediction.
191
- :param endpoint_id: Name of the endpoint used for the prediction.
192
- :param payload: Name of the model used for the prediction.
193
- :param project_id: ID of the Google Cloud project where model is located if None then
194
- default project_id is used.
195
- :param location: The location of the project.
196
- :param operation_params: Additional domain-specific parameters for the predictions.
197
- :param retry: A retry object used to retry requests. If `None` is specified, requests will not be
198
- retried.
199
- :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
200
- `retry` is specified, the timeout applies to each individual attempt.
201
- :param metadata: Additional metadata that is provided to the method.
202
- :param gcp_conn_id: The connection ID to use to connect to Google Cloud.
203
- :param impersonation_chain: Optional service account to impersonate using short-term
204
- credentials, or chained list of accounts required to get the access_token
205
- of the last account in the list, which will be impersonated in the request.
206
- If set as a string, the account must grant the originating account
207
- the Service Account Token Creator IAM role.
208
- If set as a sequence, the identities from the list must grant
209
- Service Account Token Creator IAM role to the directly preceding identity, with first
210
- account from the list granting this role to the originating account (templated).
211
- """
212
-
213
- template_fields: Sequence[str] = (
214
- "model_id",
215
- "location",
216
- "project_id",
217
- "impersonation_chain",
218
- )
219
- operator_extra_links = (TranslationLegacyModelPredictLink(),)
220
-
221
- def __init__(
222
- self,
223
- *,
224
- model_id: str | None = None,
225
- endpoint_id: str | None = None,
226
- location: str,
227
- payload: dict,
228
- operation_params: dict[str, str] | None = None,
229
- instances: list[str] | None = None,
230
- project_id: str = PROVIDE_PROJECT_ID,
231
- metadata: MetaData = (),
232
- timeout: float | None = None,
233
- retry: Retry | _MethodDefault = DEFAULT,
234
- gcp_conn_id: str = "google_cloud_default",
235
- impersonation_chain: str | Sequence[str] | None = None,
236
- **kwargs,
237
- ) -> None:
238
- super().__init__(**kwargs)
239
-
240
- self.model_id = model_id
241
- self.endpoint_id = endpoint_id
242
- self.operation_params = operation_params # type: ignore
243
- self.instances = instances
244
- self.location = location
245
- self.project_id = project_id
246
- self.metadata = metadata
247
- self.timeout = timeout
248
- self.retry = retry
249
- self.payload = payload
250
- self.gcp_conn_id = gcp_conn_id
251
- self.impersonation_chain = impersonation_chain
252
-
253
- @cached_property
254
- def hook(self) -> CloudAutoMLHook | PredictionServiceHook:
255
- if self.model_id:
256
- return CloudAutoMLHook(
257
- gcp_conn_id=self.gcp_conn_id,
258
- impersonation_chain=self.impersonation_chain,
259
- )
260
- else: # endpoint_id defined
261
- return PredictionServiceHook(
262
- gcp_conn_id=self.gcp_conn_id,
263
- impersonation_chain=self.impersonation_chain,
264
- )
265
-
266
- @cached_property
267
- def model(self) -> Model | None:
268
- if self.model_id:
269
- hook = cast(CloudAutoMLHook, self.hook)
270
- return hook.get_model(
271
- model_id=self.model_id,
272
- location=self.location,
273
- project_id=self.project_id,
274
- retry=self.retry,
275
- timeout=self.timeout,
276
- metadata=self.metadata,
277
- )
278
- return None
279
-
280
- def execute(self, context: Context):
281
- if self.model_id is None and self.endpoint_id is None:
282
- raise AirflowException("You must specify model_id or endpoint_id!")
283
-
284
- hook = self.hook
285
- if self.model_id:
286
- result = hook.predict(
287
- model_id=self.model_id,
288
- payload=self.payload,
289
- location=self.location,
290
- project_id=self.project_id,
291
- params=self.operation_params,
292
- retry=self.retry,
293
- timeout=self.timeout,
294
- metadata=self.metadata,
295
- )
296
- else: # self.endpoint_id is defined
297
- result = hook.predict(
298
- endpoint_id=self.endpoint_id,
299
- instances=self.instances,
300
- payload=self.payload,
301
- location=self.location,
302
- project_id=self.project_id,
303
- parameters=self.operation_params,
304
- retry=self.retry,
305
- timeout=self.timeout,
306
- metadata=self.metadata,
307
- )
308
-
309
- project_id = self.project_id or hook.project_id
310
- dataset_id: str | None = self.model.dataset_id if self.model else None
311
- if project_id and self.model_id and dataset_id:
312
- TranslationLegacyModelPredictLink.persist(
313
- context=context,
314
- task_instance=self,
315
- model_id=self.model_id,
316
- dataset_id=dataset_id,
317
- project_id=project_id,
318
- )
319
- return PredictResponse.to_dict(result)
320
-
321
-
322
- @deprecated(
323
- planned_removal_date="September 30, 2025",
324
- use_instead="airflow.providers.google.cloud.operators.vertex_ai.dataset.CreateDatasetOperator, "
325
- "airflow.providers.google.cloud.operators.translate.TranslateCreateDatasetOperator",
326
- category=AirflowProviderDeprecationWarning,
327
- )
328
- class AutoMLCreateDatasetOperator(GoogleCloudBaseOperator):
329
- """
330
- Creates a Google Cloud AutoML dataset.
331
-
332
- AutoMLCreateDatasetOperator for tables, video intelligence, vision and natural language has been
333
- deprecated and no longer available. Please use
334
- :class:`airflow.providers.google.cloud.operators.vertex_ai.dataset.CreateDatasetOperator`,
335
- :class:`airflow.providers.google.cloud.operators.translate.TranslateCreateDatasetOperator` instead.
336
-
337
- .. seealso::
338
- For more information on how to use this operator, take a look at the guide:
339
- :ref:`howto/operator:AutoMLCreateDatasetOperator`
340
-
341
- :param dataset: The dataset to create. If a dict is provided, it must be of the
342
- same form as the protobuf message Dataset.
343
- :param project_id: ID of the Google Cloud project where dataset is located if None then
344
- default project_id is used.
345
- :param location: The location of the project.
346
- :param params: Additional domain-specific parameters for the predictions.
347
- :param retry: A retry object used to retry requests. If `None` is specified, requests will not be
348
- retried.
349
- :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
350
- `retry` is specified, the timeout applies to each individual attempt.
351
- :param metadata: Additional metadata that is provided to the method.
352
- :param gcp_conn_id: The connection ID to use to connect to Google Cloud.
353
- :param impersonation_chain: Optional service account to impersonate using short-term
354
- credentials, or chained list of accounts required to get the access_token
355
- of the last account in the list, which will be impersonated in the request.
356
- If set as a string, the account must grant the originating account
357
- the Service Account Token Creator IAM role.
358
- If set as a sequence, the identities from the list must grant
359
- Service Account Token Creator IAM role to the directly preceding identity, with first
360
- account from the list granting this role to the originating account (templated).
361
- """
362
-
363
- template_fields: Sequence[str] = (
364
- "dataset",
365
- "location",
366
- "project_id",
367
- "impersonation_chain",
368
- )
369
- operator_extra_links = (TranslationLegacyDatasetLink(),)
370
-
371
- def __init__(
372
- self,
373
- *,
374
- dataset: dict,
375
- location: str,
376
- project_id: str = PROVIDE_PROJECT_ID,
377
- metadata: MetaData = (),
378
- timeout: float | None = None,
379
- retry: Retry | _MethodDefault = DEFAULT,
380
- gcp_conn_id: str = "google_cloud_default",
381
- impersonation_chain: str | Sequence[str] | None = None,
382
- **kwargs,
383
- ) -> None:
384
- super().__init__(**kwargs)
385
-
386
- self.dataset = dataset
387
- self.location = location
388
- self.project_id = project_id
389
- self.metadata = metadata
390
- self.timeout = timeout
391
- self.retry = retry
392
- self.gcp_conn_id = gcp_conn_id
393
- self.impersonation_chain = impersonation_chain
394
-
395
- def execute(self, context: Context):
396
- hook = CloudAutoMLHook(
397
- gcp_conn_id=self.gcp_conn_id,
398
- impersonation_chain=self.impersonation_chain,
399
- )
400
- self.log.info("Creating dataset %s...", self.dataset)
401
- result = hook.create_dataset(
402
- dataset=self.dataset,
403
- location=self.location,
404
- project_id=self.project_id,
405
- retry=self.retry,
406
- timeout=self.timeout,
407
- metadata=self.metadata,
408
- )
409
- result = Dataset.to_dict(result)
410
- dataset_id = hook.extract_object_id(result)
411
- self.log.info("Creating completed. Dataset id: %s", dataset_id)
412
-
413
- self.xcom_push(context, key="dataset_id", value=dataset_id)
414
- project_id = self.project_id or hook.project_id
415
- if project_id:
416
- TranslationLegacyDatasetLink.persist(
417
- context=context,
418
- task_instance=self,
419
- dataset_id=dataset_id,
420
- project_id=project_id,
421
- )
422
- return result
423
-
424
-
425
- @deprecated(
426
- planned_removal_date="September 30, 2025",
427
- use_instead="airflow.providers.google.cloud.operators.vertex_ai.dataset.ImportDataOperator, "
428
- "airflow.providers.google.cloud.operators.translate.TranslateImportDataOperator",
429
- category=AirflowProviderDeprecationWarning,
430
- )
431
- class AutoMLImportDataOperator(GoogleCloudBaseOperator):
432
- """
433
- Imports data to a Google Cloud AutoML dataset.
434
-
435
- .. warning::
436
- AutoMLImportDataOperator for tables, video intelligence, vision and natural language has been deprecated
437
- and no longer available. Please use
438
- :class:`airflow.providers.google.cloud.operators.vertex_ai.dataset.ImportDataOperator` instead.
439
-
440
- .. seealso::
441
- For more information on how to use this operator, take a look at the guide:
442
- :ref:`howto/operator:AutoMLImportDataOperator`
443
-
444
- :param dataset_id: ID of dataset to be updated.
445
- :param input_config: The desired input location and its domain specific semantics, if any.
446
- If a dict is provided, it must be of the same form as the protobuf message InputConfig.
447
- :param project_id: ID of the Google Cloud project where dataset is located if None then
448
- default project_id is used.
449
- :param location: The location of the project.
450
- :param params: Additional domain-specific parameters for the predictions.
451
- :param retry: A retry object used to retry requests. If `None` is specified, requests will not be
452
- retried.
453
- :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
454
- `retry` is specified, the timeout applies to each individual attempt.
455
- :param metadata: Additional metadata that is provided to the method.
456
- :param gcp_conn_id: The connection ID to use to connect to Google Cloud.
457
- :param impersonation_chain: Optional service account to impersonate using short-term
458
- credentials, or chained list of accounts required to get the access_token
459
- of the last account in the list, which will be impersonated in the request.
460
- If set as a string, the account must grant the originating account
461
- the Service Account Token Creator IAM role.
462
- If set as a sequence, the identities from the list must grant
463
- Service Account Token Creator IAM role to the directly preceding identity, with first
464
- account from the list granting this role to the originating account (templated).
465
- """
466
-
467
- template_fields: Sequence[str] = (
468
- "dataset_id",
469
- "input_config",
470
- "location",
471
- "project_id",
472
- "impersonation_chain",
473
- )
474
- operator_extra_links = (TranslationLegacyDatasetLink(),)
475
-
476
- def __init__(
477
- self,
478
- *,
479
- dataset_id: str,
480
- location: str,
481
- input_config: dict,
482
- project_id: str = PROVIDE_PROJECT_ID,
483
- metadata: MetaData = (),
484
- timeout: float | None = None,
485
- retry: Retry | _MethodDefault = DEFAULT,
486
- gcp_conn_id: str = "google_cloud_default",
487
- impersonation_chain: str | Sequence[str] | None = None,
488
- **kwargs,
489
- ) -> None:
490
- super().__init__(**kwargs)
491
-
492
- self.dataset_id = dataset_id
493
- self.input_config = input_config
494
- self.location = location
495
- self.project_id = project_id
496
- self.metadata = metadata
497
- self.timeout = timeout
498
- self.retry = retry
499
- self.gcp_conn_id = gcp_conn_id
500
- self.impersonation_chain = impersonation_chain
501
-
502
- def execute(self, context: Context):
503
- hook = CloudAutoMLHook(
504
- gcp_conn_id=self.gcp_conn_id,
505
- impersonation_chain=self.impersonation_chain,
506
- )
507
- hook.get_dataset(
508
- dataset_id=self.dataset_id,
509
- location=self.location,
510
- project_id=self.project_id,
511
- retry=self.retry,
512
- timeout=self.timeout,
513
- metadata=self.metadata,
514
- )
515
- self.log.info("Importing data to dataset...")
516
- operation = hook.import_data(
517
- dataset_id=self.dataset_id,
518
- input_config=self.input_config,
519
- location=self.location,
520
- project_id=self.project_id,
521
- retry=self.retry,
522
- timeout=self.timeout,
523
- metadata=self.metadata,
524
- )
525
- hook.wait_for_operation(timeout=self.timeout, operation=operation)
526
- self.log.info("Import is completed")
527
- project_id = self.project_id or hook.project_id
528
- if project_id:
529
- TranslationLegacyDatasetLink.persist(
530
- context=context,
531
- task_instance=self,
532
- dataset_id=self.dataset_id,
533
- project_id=project_id,
534
- )
535
-
536
-
537
- @deprecated(
538
- planned_removal_date="September 30, 2025",
539
- category=AirflowProviderDeprecationWarning,
540
- reason="Shutdown of legacy version of AutoML Tables on March 31, 2024.",
541
- )
542
- class AutoMLTablesListColumnSpecsOperator(GoogleCloudBaseOperator):
543
- """
544
- Lists column specs in a table.
545
-
546
- .. warning::
547
- Operator AutoMLTablesListColumnSpecsOperator has been deprecated due to shutdown of
548
- a legacy version of AutoML Tables on March 31, 2024. For additional information
549
- see: https://cloud.google.com/automl-tables/docs/deprecations.
550
-
551
- .. seealso::
552
- For more information on how to use this operator, take a look at the guide:
553
- :ref:`howto/operator:AutoMLTablesListColumnSpecsOperator`
554
-
555
- :param dataset_id: Name of the dataset.
556
- :param table_spec_id: table_spec_id for path builder.
557
- :param field_mask: Mask specifying which fields to read. If a dict is provided, it must be of the same
558
- form as the protobuf message `google.cloud.automl_v1beta1.types.FieldMask`
559
- :param filter_: Filter expression, see go/filtering.
560
- :param page_size: The maximum number of resources contained in the
561
- underlying API response. If page streaming is performed per
562
- resource, this parameter does not affect the return value. If page
563
- streaming is performed per page, this determines the maximum number
564
- of resources in a page.
565
- :param project_id: ID of the Google Cloud project where dataset is located if None then
566
- default project_id is used.
567
- :param location: The location of the project.
568
- :param retry: A retry object used to retry requests. If `None` is specified, requests will not be
569
- retried.
570
- :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
571
- `retry` is specified, the timeout applies to each individual attempt.
572
- :param metadata: Additional metadata that is provided to the method.
573
- :param gcp_conn_id: The connection ID to use to connect to Google Cloud.
574
- :param impersonation_chain: Optional service account to impersonate using short-term
575
- credentials, or chained list of accounts required to get the access_token
576
- of the last account in the list, which will be impersonated in the request.
577
- If set as a string, the account must grant the originating account
578
- the Service Account Token Creator IAM role.
579
- If set as a sequence, the identities from the list must grant
580
- Service Account Token Creator IAM role to the directly preceding identity, with first
581
- account from the list granting this role to the originating account (templated).
582
- """
583
-
584
- template_fields: Sequence[str] = (
585
- "dataset_id",
586
- "table_spec_id",
587
- "field_mask",
588
- "filter_",
589
- "location",
590
- "project_id",
591
- "impersonation_chain",
592
- )
593
- operator_extra_links = (TranslationLegacyDatasetLink(),)
594
-
595
- def __init__(
596
- self,
597
- *,
598
- dataset_id: str,
599
- table_spec_id: str,
600
- location: str,
601
- field_mask: dict | None = None,
602
- filter_: str | None = None,
603
- page_size: int | None = None,
604
- project_id: str = PROVIDE_PROJECT_ID,
605
- metadata: MetaData = (),
606
- timeout: float | None = None,
607
- retry: Retry | _MethodDefault = DEFAULT,
608
- gcp_conn_id: str = "google_cloud_default",
609
- impersonation_chain: str | Sequence[str] | None = None,
610
- **kwargs,
611
- ) -> None:
612
- super().__init__(**kwargs)
613
- self.dataset_id = dataset_id
614
- self.table_spec_id = table_spec_id
615
- self.field_mask = field_mask
616
- self.filter_ = filter_
617
- self.page_size = page_size
618
- self.location = location
619
- self.project_id = project_id
620
- self.metadata = metadata
621
- self.timeout = timeout
622
- self.retry = retry
623
- self.gcp_conn_id = gcp_conn_id
624
- self.impersonation_chain = impersonation_chain
625
-
626
- def execute(self, context: Context):
627
- hook = CloudAutoMLHook(
628
- gcp_conn_id=self.gcp_conn_id,
629
- impersonation_chain=self.impersonation_chain,
630
- )
631
- self.log.info("Requesting column specs.")
632
- page_iterator = hook.list_column_specs(
633
- dataset_id=self.dataset_id,
634
- table_spec_id=self.table_spec_id,
635
- field_mask=self.field_mask,
636
- filter_=self.filter_,
637
- page_size=self.page_size,
638
- location=self.location,
639
- project_id=self.project_id,
640
- retry=self.retry,
641
- timeout=self.timeout,
642
- metadata=self.metadata,
643
- )
644
- result = [ColumnSpec.to_dict(spec) for spec in page_iterator]
645
- self.log.info("Columns specs obtained.")
646
- project_id = self.project_id or hook.project_id
647
- if project_id:
648
- TranslationLegacyDatasetLink.persist(
649
- context=context,
650
- task_instance=self,
651
- dataset_id=self.dataset_id,
652
- project_id=project_id,
653
- )
654
- return result
655
-
656
-
657
- @deprecated(
658
- planned_removal_date="September 30, 2025",
659
- use_instead="airflow.providers.google.cloud.operators.vertex_ai.dataset.UpdateDatasetOperator",
660
- category=AirflowProviderDeprecationWarning,
661
- reason="Shutdown of legacy version of AutoML Tables on March 31, 2024.",
662
- )
663
- class AutoMLTablesUpdateDatasetOperator(GoogleCloudBaseOperator):
664
- """
665
- Updates a dataset.
666
-
667
- .. warning::
668
- Operator AutoMLTablesUpdateDatasetOperator has been deprecated due to shutdown of
669
- a legacy version of AutoML Tables on March 31, 2024. For additional information
670
- see: https://cloud.google.com/automl-tables/docs/deprecations.
671
- Please use :class:`airflow.providers.google.cloud.operators.vertex_ai.dataset.UpdateDatasetOperator`
672
- instead.
673
-
674
- .. seealso::
675
- For more information on how to use this operator, take a look at the guide:
676
- :ref:`howto/operator:AutoMLTablesUpdateDatasetOperator`
677
-
678
- :param dataset: The dataset which replaces the resource on the server.
679
- If a dict is provided, it must be of the same form as the protobuf message Dataset.
680
- :param update_mask: The update mask applies to the resource. If a dict is provided, it must
681
- be of the same form as the protobuf message FieldMask.
682
- :param location: The location of the project.
683
- :param params: Additional domain-specific parameters for the predictions.
684
- :param retry: A retry object used to retry requests. If `None` is specified, requests will not be
685
- retried.
686
- :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
687
- `retry` is specified, the timeout applies to each individual attempt.
688
- :param metadata: Additional metadata that is provided to the method.
689
- :param gcp_conn_id: The connection ID to use to connect to Google Cloud.
690
- :param impersonation_chain: Optional service account to impersonate using short-term
691
- credentials, or chained list of accounts required to get the access_token
692
- of the last account in the list, which will be impersonated in the request.
693
- If set as a string, the account must grant the originating account
694
- the Service Account Token Creator IAM role.
695
- If set as a sequence, the identities from the list must grant
696
- Service Account Token Creator IAM role to the directly preceding identity, with first
697
- account from the list granting this role to the originating account (templated).
698
- """
699
-
700
- template_fields: Sequence[str] = (
701
- "dataset",
702
- "update_mask",
703
- "location",
704
- "impersonation_chain",
705
- )
706
- operator_extra_links = (TranslationLegacyDatasetLink(),)
707
-
708
- def __init__(
709
- self,
710
- *,
711
- dataset: dict,
712
- location: str,
713
- update_mask: dict | None = None,
714
- metadata: MetaData = (),
715
- timeout: float | None = None,
716
- retry: Retry | _MethodDefault = DEFAULT,
717
- gcp_conn_id: str = "google_cloud_default",
718
- impersonation_chain: str | Sequence[str] | None = None,
719
- **kwargs,
720
- ) -> None:
721
- super().__init__(**kwargs)
722
-
723
- self.dataset = dataset
724
- self.update_mask = update_mask
725
- self.location = location
726
- self.metadata = metadata
727
- self.timeout = timeout
728
- self.retry = retry
729
- self.gcp_conn_id = gcp_conn_id
730
- self.impersonation_chain = impersonation_chain
731
-
732
- def execute(self, context: Context):
733
- hook = CloudAutoMLHook(
734
- gcp_conn_id=self.gcp_conn_id,
735
- impersonation_chain=self.impersonation_chain,
736
- )
737
- self.log.info("Updating AutoML dataset %s.", self.dataset["name"])
738
- result = hook.update_dataset(
739
- dataset=self.dataset,
740
- update_mask=self.update_mask,
741
- retry=self.retry,
742
- timeout=self.timeout,
743
- metadata=self.metadata,
744
- )
745
- self.log.info("Dataset updated.")
746
- project_id = hook.project_id
747
- if project_id:
748
- TranslationLegacyDatasetLink.persist(
749
- context=context,
750
- task_instance=self,
751
- dataset_id=hook.extract_object_id(self.dataset),
752
- project_id=project_id,
753
- )
754
- return Dataset.to_dict(result)
755
-
756
-
757
- @deprecated(
758
- planned_removal_date="September 30, 2025",
759
- use_instead="airflow.providers.google.cloud.operators.vertex_ai.model_service.GetModelOperator",
760
- category=AirflowProviderDeprecationWarning,
761
- )
762
- class AutoMLGetModelOperator(GoogleCloudBaseOperator):
763
- """
764
- Get Google Cloud AutoML model.
765
-
766
- .. warning::
767
- AutoMLGetModelOperator for tables, video intelligence, vision and natural language has been deprecated
768
- and no longer available. Please use
769
- :class:`airflow.providers.google.cloud.operators.vertex_ai.model_service.GetModelOperator` instead.
770
-
771
- .. seealso::
772
- For more information on how to use this operator, take a look at the guide:
773
- :ref:`howto/operator:AutoMLGetModelOperator`
774
-
775
- :param model_id: Name of the model requested to serve the prediction.
776
- :param project_id: ID of the Google Cloud project where model is located if None then
777
- default project_id is used.
778
- :param location: The location of the project.
779
- :param params: Additional domain-specific parameters for the predictions.
780
- :param retry: A retry object used to retry requests. If `None` is specified, requests will not be
781
- retried.
782
- :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
783
- `retry` is specified, the timeout applies to each individual attempt.
784
- :param metadata: Additional metadata that is provided to the method.
785
- :param gcp_conn_id: The connection ID to use to connect to Google Cloud.
786
- :param impersonation_chain: Optional service account to impersonate using short-term
787
- credentials, or chained list of accounts required to get the access_token
788
- of the last account in the list, which will be impersonated in the request.
789
- If set as a string, the account must grant the originating account
790
- the Service Account Token Creator IAM role.
791
- If set as a sequence, the identities from the list must grant
792
- Service Account Token Creator IAM role to the directly preceding identity, with first
793
- account from the list granting this role to the originating account (templated).
794
- """
795
-
796
- template_fields: Sequence[str] = (
797
- "model_id",
798
- "location",
799
- "project_id",
800
- "impersonation_chain",
801
- )
802
- operator_extra_links = (TranslationLegacyModelLink(),)
803
-
804
- def __init__(
805
- self,
806
- *,
807
- model_id: str,
808
- location: str,
809
- project_id: str = PROVIDE_PROJECT_ID,
810
- metadata: MetaData = (),
811
- timeout: float | None = None,
812
- retry: Retry | _MethodDefault = DEFAULT,
813
- gcp_conn_id: str = "google_cloud_default",
814
- impersonation_chain: str | Sequence[str] | None = None,
815
- **kwargs,
816
- ) -> None:
817
- super().__init__(**kwargs)
818
-
819
- self.model_id = model_id
820
- self.location = location
821
- self.project_id = project_id
822
- self.metadata = metadata
823
- self.timeout = timeout
824
- self.retry = retry
825
- self.gcp_conn_id = gcp_conn_id
826
- self.impersonation_chain = impersonation_chain
827
-
828
- def execute(self, context: Context):
829
- hook = CloudAutoMLHook(
830
- gcp_conn_id=self.gcp_conn_id,
831
- impersonation_chain=self.impersonation_chain,
832
- )
833
- result = hook.get_model(
834
- model_id=self.model_id,
835
- location=self.location,
836
- project_id=self.project_id,
837
- retry=self.retry,
838
- timeout=self.timeout,
839
- metadata=self.metadata,
840
- )
841
- model = Model.to_dict(result)
842
- project_id = self.project_id or hook.project_id
843
- if project_id:
844
- TranslationLegacyModelLink.persist(
845
- context=context,
846
- task_instance=self,
847
- dataset_id=model["dataset_id"],
848
- model_id=self.model_id,
849
- project_id=project_id,
850
- )
851
- return model
852
-
853
-
854
- @deprecated(
855
- planned_removal_date="September 30, 2025",
856
- use_instead="airflow.providers.google.cloud.operators.vertex_ai.model_service.DeleteModelOperator, "
857
- "airflow.providers.google.cloud.operators.translate.TranslateDeleteModelOperator",
858
- category=AirflowProviderDeprecationWarning,
859
- )
860
- class AutoMLDeleteModelOperator(GoogleCloudBaseOperator):
861
- """
862
- Delete Google Cloud AutoML model.
863
-
864
- .. warning::
865
- AutoMLDeleteModelOperator for tables, video intelligence, vision and natural language has been deprecated
866
- and no longer available. Please use
867
- :class:`airflow.providers.google.cloud.operators.vertex_ai.model_service.DeleteModelOperator` instead.
868
-
869
- .. seealso::
870
- For more information on how to use this operator, take a look at the guide:
871
- :ref:`howto/operator:AutoMLDeleteModelOperator`
872
-
873
- :param model_id: Name of the model requested to serve the prediction.
874
- :param project_id: ID of the Google Cloud project where model is located if None then
875
- default project_id is used.
876
- :param location: The location of the project.
877
- :param params: Additional domain-specific parameters for the predictions.
878
- :param retry: A retry object used to retry requests. If `None` is specified, requests will not be
879
- retried.
880
- :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
881
- `retry` is specified, the timeout applies to each individual attempt.
882
- :param metadata: Additional metadata that is provided to the method.
883
- :param gcp_conn_id: The connection ID to use to connect to Google Cloud.
884
- :param impersonation_chain: Optional service account to impersonate using short-term
885
- credentials, or chained list of accounts required to get the access_token
886
- of the last account in the list, which will be impersonated in the request.
887
- If set as a string, the account must grant the originating account
888
- the Service Account Token Creator IAM role.
889
- If set as a sequence, the identities from the list must grant
890
- Service Account Token Creator IAM role to the directly preceding identity, with first
891
- account from the list granting this role to the originating account (templated).
892
- """
893
-
894
- template_fields: Sequence[str] = (
895
- "model_id",
896
- "location",
897
- "project_id",
898
- "impersonation_chain",
899
- )
900
-
901
- def __init__(
902
- self,
903
- *,
904
- model_id: str,
905
- location: str,
906
- project_id: str = PROVIDE_PROJECT_ID,
907
- metadata: MetaData = (),
908
- timeout: float | None = None,
909
- retry: Retry | _MethodDefault = DEFAULT,
910
- gcp_conn_id: str = "google_cloud_default",
911
- impersonation_chain: str | Sequence[str] | None = None,
912
- **kwargs,
913
- ) -> None:
914
- super().__init__(**kwargs)
915
-
916
- self.model_id = model_id
917
- self.location = location
918
- self.project_id = project_id
919
- self.metadata = metadata
920
- self.timeout = timeout
921
- self.retry = retry
922
- self.gcp_conn_id = gcp_conn_id
923
- self.impersonation_chain = impersonation_chain
924
-
925
- def execute(self, context: Context):
926
- hook = CloudAutoMLHook(
927
- gcp_conn_id=self.gcp_conn_id,
928
- impersonation_chain=self.impersonation_chain,
929
- )
930
- hook.get_model(
931
- model_id=self.model_id,
932
- location=self.location,
933
- project_id=self.project_id,
934
- retry=self.retry,
935
- timeout=self.timeout,
936
- metadata=self.metadata,
937
- )
938
- operation = hook.delete_model(
939
- model_id=self.model_id,
940
- location=self.location,
941
- project_id=self.project_id,
942
- retry=self.retry,
943
- timeout=self.timeout,
944
- metadata=self.metadata,
945
- )
946
- hook.wait_for_operation(timeout=self.timeout, operation=operation)
947
- self.log.info("Deletion is completed")
948
-
949
-
950
- @deprecated(
951
- planned_removal_date="September 30, 2025",
952
- use_instead="airflow.providers.google.cloud.operators.vertex_ai.endpoint_service.DeployModelOperator",
953
- category=AirflowProviderDeprecationWarning,
954
- )
955
- class AutoMLDeployModelOperator(GoogleCloudBaseOperator):
956
- """
957
- Deploys a model; if a model is already deployed, deploying it with the same parameters has no effect.
958
-
959
- Deploying with different parameters (as e.g. changing node_number) will
960
- reset the deployment state without pausing the model_id's availability.
961
-
962
- Only applicable for Text Classification, Image Object Detection and Tables; all other
963
- domains manage deployment automatically.
964
-
965
- .. warning::
966
- Operator AutoMLDeployModelOperator has been deprecated due to shutdown of a legacy version
967
- of AutoML Natural Language, Vision, Video Intelligence on March 31, 2024.
968
- For additional information see: https://cloud.google.com/vision/automl/docs/deprecations .
969
- Please use :class:`airflow.providers.google.cloud.operators.vertex_ai.endpoint_service.DeployModelOperator`
970
- instead.
971
-
972
- .. seealso::
973
- For more information on how to use this operator, take a look at the guide:
974
- :ref:`howto/operator:AutoMLDeployModelOperator`
975
-
976
- :param model_id: Name of the model to be deployed.
977
- :param image_detection_metadata: Model deployment metadata specific to Image Object Detection.
978
- If a dict is provided, it must be of the same form as the protobuf message
979
- ImageObjectDetectionModelDeploymentMetadata
980
- :param project_id: ID of the Google Cloud project where model is located if None then
981
- default project_id is used.
982
- :param location: The location of the project.
983
- :param params: Additional domain-specific parameters for the predictions.
984
- :param retry: A retry object used to retry requests. If `None` is specified, requests will not be
985
- retried.
986
- :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
987
- `retry` is specified, the timeout applies to each individual attempt.
988
- :param metadata: Additional metadata that is provided to the method.
989
- :param gcp_conn_id: The connection ID to use to connect to Google Cloud.
990
- :param impersonation_chain: Optional service account to impersonate using short-term
991
- credentials, or chained list of accounts required to get the access_token
992
- of the last account in the list, which will be impersonated in the request.
993
- If set as a string, the account must grant the originating account
994
- the Service Account Token Creator IAM role.
995
- If set as a sequence, the identities from the list must grant
996
- Service Account Token Creator IAM role to the directly preceding identity, with first
997
- account from the list granting this role to the originating account (templated).
998
- """
999
-
1000
- template_fields: Sequence[str] = (
1001
- "model_id",
1002
- "location",
1003
- "project_id",
1004
- "impersonation_chain",
1005
- )
1006
-
1007
- def __init__(
1008
- self,
1009
- *,
1010
- model_id: str,
1011
- location: str,
1012
- project_id: str = PROVIDE_PROJECT_ID,
1013
- image_detection_metadata: dict | None = None,
1014
- metadata: Sequence[tuple[str, str]] = (),
1015
- timeout: float | None = None,
1016
- retry: Retry | _MethodDefault = DEFAULT,
1017
- gcp_conn_id: str = "google_cloud_default",
1018
- impersonation_chain: str | Sequence[str] | None = None,
1019
- **kwargs,
1020
- ) -> None:
1021
- super().__init__(**kwargs)
1022
-
1023
- self.model_id = model_id
1024
- self.image_detection_metadata = image_detection_metadata
1025
- self.location = location
1026
- self.project_id = project_id
1027
- self.metadata = metadata
1028
- self.timeout = timeout
1029
- self.retry = retry
1030
- self.gcp_conn_id = gcp_conn_id
1031
- self.impersonation_chain = impersonation_chain
1032
-
1033
- def execute(self, context: Context):
1034
- hook = CloudAutoMLHook(
1035
- gcp_conn_id=self.gcp_conn_id,
1036
- impersonation_chain=self.impersonation_chain,
1037
- )
1038
- self.log.info("Deploying model_id %s", self.model_id)
1039
- operation = hook.deploy_model(
1040
- model_id=self.model_id,
1041
- location=self.location,
1042
- project_id=self.project_id,
1043
- image_detection_metadata=self.image_detection_metadata,
1044
- retry=self.retry,
1045
- timeout=self.timeout,
1046
- metadata=self.metadata,
1047
- )
1048
- hook.wait_for_operation(timeout=self.timeout, operation=operation)
1049
- self.log.info("Model was deployed successfully.")
1050
-
1051
-
1052
- @deprecated(
1053
- planned_removal_date="September 30, 2025",
1054
- category=AirflowProviderDeprecationWarning,
1055
- reason="Shutdown of legacy version of AutoML Tables on March 31, 2024.",
1056
- )
1057
- class AutoMLTablesListTableSpecsOperator(GoogleCloudBaseOperator):
1058
- """
1059
- Lists table specs in a dataset.
1060
-
1061
- .. warning::
1062
- Operator AutoMLTablesListTableSpecsOperator has been deprecated due to shutdown of
1063
- a legacy version of AutoML Tables on March 31, 2024. For additional information
1064
- see: https://cloud.google.com/automl-tables/docs/deprecations.
1065
-
1066
- .. seealso::
1067
- For more information on how to use this operator, take a look at the guide:
1068
- :ref:`howto/operator:AutoMLTablesListTableSpecsOperator`
1069
-
1070
- :param dataset_id: Name of the dataset.
1071
- :param filter_: Filter expression, see go/filtering.
1072
- :param page_size: The maximum number of resources contained in the
1073
- underlying API response. If page streaming is performed per
1074
- resource, this parameter does not affect the return value. If page
1075
- streaming is performed per-page, this determines the maximum number
1076
- of resources in a page.
1077
- :param project_id: ID of the Google Cloud project if None then
1078
- default project_id is used.
1079
- :param location: The location of the project.
1080
- :param retry: A retry object used to retry requests. If `None` is specified, requests will not be
1081
- retried.
1082
- :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
1083
- `retry` is specified, the timeout applies to each individual attempt.
1084
- :param metadata: Additional metadata that is provided to the method.
1085
- :param gcp_conn_id: The connection ID to use to connect to Google Cloud.
1086
- :param impersonation_chain: Optional service account to impersonate using short-term
1087
- credentials, or chained list of accounts required to get the access_token
1088
- of the last account in the list, which will be impersonated in the request.
1089
- If set as a string, the account must grant the originating account
1090
- the Service Account Token Creator IAM role.
1091
- If set as a sequence, the identities from the list must grant
1092
- Service Account Token Creator IAM role to the directly preceding identity, with first
1093
- account from the list granting this role to the originating account (templated).
1094
- """
1095
-
1096
- template_fields: Sequence[str] = (
1097
- "dataset_id",
1098
- "filter_",
1099
- "location",
1100
- "project_id",
1101
- "impersonation_chain",
1102
- )
1103
- operator_extra_links = (TranslationLegacyDatasetLink(),)
1104
-
1105
- def __init__(
1106
- self,
1107
- *,
1108
- dataset_id: str,
1109
- location: str,
1110
- page_size: int | None = None,
1111
- filter_: str | None = None,
1112
- project_id: str = PROVIDE_PROJECT_ID,
1113
- metadata: MetaData = (),
1114
- timeout: float | None = None,
1115
- retry: Retry | _MethodDefault = DEFAULT,
1116
- gcp_conn_id: str = "google_cloud_default",
1117
- impersonation_chain: str | Sequence[str] | None = None,
1118
- **kwargs,
1119
- ) -> None:
1120
- super().__init__(**kwargs)
1121
- self.dataset_id = dataset_id
1122
- self.filter_ = filter_
1123
- self.page_size = page_size
1124
- self.location = location
1125
- self.project_id = project_id
1126
- self.metadata = metadata
1127
- self.timeout = timeout
1128
- self.retry = retry
1129
- self.gcp_conn_id = gcp_conn_id
1130
- self.impersonation_chain = impersonation_chain
1131
-
1132
- def execute(self, context: Context):
1133
- hook = CloudAutoMLHook(
1134
- gcp_conn_id=self.gcp_conn_id,
1135
- impersonation_chain=self.impersonation_chain,
1136
- )
1137
- self.log.info("Requesting table specs for %s.", self.dataset_id)
1138
- page_iterator = hook.list_table_specs(
1139
- dataset_id=self.dataset_id,
1140
- filter_=self.filter_,
1141
- page_size=self.page_size,
1142
- location=self.location,
1143
- project_id=self.project_id,
1144
- retry=self.retry,
1145
- timeout=self.timeout,
1146
- metadata=self.metadata,
1147
- )
1148
- result = [TableSpec.to_dict(spec) for spec in page_iterator]
1149
- self.log.info(result)
1150
- self.log.info("Table specs obtained.")
1151
- project_id = self.project_id or hook.project_id
1152
- if project_id:
1153
- TranslationLegacyDatasetLink.persist(
1154
- context=context,
1155
- task_instance=self,
1156
- dataset_id=self.dataset_id,
1157
- project_id=project_id,
1158
- )
1159
- return result
1160
-
1161
-
1162
- @deprecated(
1163
- planned_removal_date="September 30, 2025",
1164
- use_instead="airflow.providers.google.cloud.operators.vertex_ai.dataset.ListDatasetsOperator, "
1165
- "airflow.providers.google.cloud.operators.translate.TranslateDatasetsListOperator",
1166
- category=AirflowProviderDeprecationWarning,
1167
- )
1168
- class AutoMLListDatasetOperator(GoogleCloudBaseOperator):
1169
- """
1170
- Lists AutoML Datasets in project.
1171
-
1172
- .. warning::
1173
- AutoMLListDatasetOperator for tables, video intelligence, vision and natural language has been deprecated
1174
- and no longer available. Please use
1175
- :class:`airflow.providers.google.cloud.operators.vertex_ai.dataset.ListDatasetsOperator` instead.
1176
-
1177
- .. seealso::
1178
- For more information on how to use this operator, take a look at the guide:
1179
- :ref:`howto/operator:AutoMLListDatasetOperator`
1180
-
1181
- :param project_id: ID of the Google Cloud project where datasets are located if None then
1182
- default project_id is used.
1183
- :param location: The location of the project.
1184
- :param retry: A retry object used to retry requests. If `None` is specified, requests will not be
1185
- retried.
1186
- :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
1187
- `retry` is specified, the timeout applies to each individual attempt.
1188
- :param metadata: Additional metadata that is provided to the method.
1189
- :param gcp_conn_id: The connection ID to use to connect to Google Cloud.
1190
- :param impersonation_chain: Optional service account to impersonate using short-term
1191
- credentials, or chained list of accounts required to get the access_token
1192
- of the last account in the list, which will be impersonated in the request.
1193
- If set as a string, the account must grant the originating account
1194
- the Service Account Token Creator IAM role.
1195
- If set as a sequence, the identities from the list must grant
1196
- Service Account Token Creator IAM role to the directly preceding identity, with first
1197
- account from the list granting this role to the originating account (templated).
1198
- """
1199
-
1200
- template_fields: Sequence[str] = (
1201
- "location",
1202
- "project_id",
1203
- "impersonation_chain",
1204
- )
1205
- operator_extra_links = (TranslationDatasetListLink(),)
1206
-
1207
- def __init__(
1208
- self,
1209
- *,
1210
- location: str,
1211
- project_id: str = PROVIDE_PROJECT_ID,
1212
- metadata: MetaData = (),
1213
- timeout: float | None = None,
1214
- retry: Retry | _MethodDefault = DEFAULT,
1215
- gcp_conn_id: str = "google_cloud_default",
1216
- impersonation_chain: str | Sequence[str] | None = None,
1217
- **kwargs,
1218
- ) -> None:
1219
- super().__init__(**kwargs)
1220
- self.location = location
1221
- self.project_id = project_id
1222
- self.metadata = metadata
1223
- self.timeout = timeout
1224
- self.retry = retry
1225
- self.gcp_conn_id = gcp_conn_id
1226
- self.impersonation_chain = impersonation_chain
1227
-
1228
- def execute(self, context: Context):
1229
- hook = CloudAutoMLHook(
1230
- gcp_conn_id=self.gcp_conn_id,
1231
- impersonation_chain=self.impersonation_chain,
1232
- )
1233
- self.log.info("Requesting datasets")
1234
- page_iterator = hook.list_datasets(
1235
- location=self.location,
1236
- project_id=self.project_id,
1237
- retry=self.retry,
1238
- timeout=self.timeout,
1239
- metadata=self.metadata,
1240
- )
1241
- result = []
1242
- for dataset in page_iterator:
1243
- result.append(Dataset.to_dict(dataset))
1244
- self.log.info("Datasets obtained.")
1245
-
1246
- self.xcom_push(
1247
- context,
1248
- key="dataset_id_list",
1249
- value=[hook.extract_object_id(d) for d in result],
1250
- )
1251
- project_id = self.project_id or hook.project_id
1252
- if project_id:
1253
- TranslationDatasetListLink.persist(context=context, task_instance=self, project_id=project_id)
1254
- return result
1255
-
1256
-
1257
- @deprecated(
1258
- planned_removal_date="September 30, 2025",
1259
- use_instead="airflow.providers.google.cloud.operators.vertex_ai.dataset.ListDatasetsOperator, "
1260
- "airflow.providers.google.cloud.operators.translate.TranslateDatasetsListOperator",
1261
- category=AirflowProviderDeprecationWarning,
1262
- )
1263
- class AutoMLDeleteDatasetOperator(GoogleCloudBaseOperator):
1264
- """
1265
- Deletes a dataset and all of its contents.
1266
-
1267
- AutoMLDeleteDatasetOperator for tables, video intelligence, vision and natural language has been
1268
- deprecated and no longer available. Please use
1269
- :class:`airflow.providers.google.cloud.operators.vertex_ai.dataset.DeleteDatasetOperator` instead.
1270
-
1271
- .. seealso::
1272
- For more information on how to use this operator, take a look at the guide:
1273
- :ref:`howto/operator:AutoMLDeleteDatasetOperator`
1274
-
1275
- :param dataset_id: Name of the dataset_id, list of dataset_id or string of dataset_id
1276
- coma separated to be deleted.
1277
- :param project_id: ID of the Google Cloud project where dataset is located if None then
1278
- default project_id is used.
1279
- :param location: The location of the project.
1280
- :param retry: A retry object used to retry requests. If `None` is specified, requests will not be
1281
- retried.
1282
- :param timeout: The amount of time, in seconds, to wait for the request to complete. Note that if
1283
- `retry` is specified, the timeout applies to each individual attempt.
1284
- :param metadata: Additional metadata that is provided to the method.
1285
- :param gcp_conn_id: The connection ID to use to connect to Google Cloud.
1286
- :param impersonation_chain: Optional service account to impersonate using short-term
1287
- credentials, or chained list of accounts required to get the access_token
1288
- of the last account in the list, which will be impersonated in the request.
1289
- If set as a string, the account must grant the originating account
1290
- the Service Account Token Creator IAM role.
1291
- If set as a sequence, the identities from the list must grant
1292
- Service Account Token Creator IAM role to the directly preceding identity, with first
1293
- account from the list granting this role to the originating account (templated).
1294
- """
1295
-
1296
- template_fields: Sequence[str] = (
1297
- "dataset_id",
1298
- "location",
1299
- "project_id",
1300
- "impersonation_chain",
1301
- )
1302
-
1303
- def __init__(
1304
- self,
1305
- *,
1306
- dataset_id: str | list[str],
1307
- location: str,
1308
- project_id: str = PROVIDE_PROJECT_ID,
1309
- metadata: MetaData = (),
1310
- timeout: float | None = None,
1311
- retry: Retry | _MethodDefault = DEFAULT,
1312
- gcp_conn_id: str = "google_cloud_default",
1313
- impersonation_chain: str | Sequence[str] | None = None,
1314
- **kwargs,
1315
- ) -> None:
1316
- super().__init__(**kwargs)
1317
-
1318
- self.dataset_id = dataset_id
1319
- self.location = location
1320
- self.project_id = project_id
1321
- self.metadata = metadata
1322
- self.timeout = timeout
1323
- self.retry = retry
1324
- self.gcp_conn_id = gcp_conn_id
1325
- self.impersonation_chain = impersonation_chain
1326
-
1327
- @staticmethod
1328
- def _parse_dataset_id(dataset_id: str | list[str]) -> list[str]:
1329
- if not isinstance(dataset_id, str):
1330
- return dataset_id
1331
- try:
1332
- return ast.literal_eval(dataset_id)
1333
- except (SyntaxError, ValueError):
1334
- return dataset_id.split(",")
1335
-
1336
- def execute(self, context: Context):
1337
- hook = CloudAutoMLHook(
1338
- gcp_conn_id=self.gcp_conn_id,
1339
- impersonation_chain=self.impersonation_chain,
1340
- )
1341
- hook.get_dataset(
1342
- dataset_id=self.dataset_id,
1343
- location=self.location,
1344
- project_id=self.project_id,
1345
- retry=self.retry,
1346
- timeout=self.timeout,
1347
- metadata=self.metadata,
1348
- )
1349
- dataset_id_list = self._parse_dataset_id(self.dataset_id)
1350
- for dataset_id in dataset_id_list:
1351
- self.log.info("Deleting dataset %s", dataset_id)
1352
- hook.delete_dataset(
1353
- dataset_id=dataset_id,
1354
- location=self.location,
1355
- project_id=self.project_id,
1356
- retry=self.retry,
1357
- timeout=self.timeout,
1358
- metadata=self.metadata,
1359
- )
1360
- self.log.info("Dataset deleted.")