apache-airflow-providers-google 12.0.0rc2__py3-none-any.whl → 14.0.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. airflow/providers/google/LICENSE +0 -52
  2. airflow/providers/google/__init__.py +1 -1
  3. airflow/providers/google/ads/hooks/ads.py +27 -13
  4. airflow/providers/google/ads/transfers/ads_to_gcs.py +18 -4
  5. airflow/providers/google/assets/bigquery.py +17 -0
  6. airflow/providers/google/cloud/_internal_client/secret_manager_client.py +2 -3
  7. airflow/providers/google/cloud/hooks/alloy_db.py +736 -8
  8. airflow/providers/google/cloud/hooks/automl.py +10 -4
  9. airflow/providers/google/cloud/hooks/bigquery.py +125 -22
  10. airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
  11. airflow/providers/google/cloud/hooks/bigtable.py +2 -3
  12. airflow/providers/google/cloud/hooks/cloud_batch.py +3 -4
  13. airflow/providers/google/cloud/hooks/cloud_build.py +4 -5
  14. airflow/providers/google/cloud/hooks/cloud_composer.py +3 -4
  15. airflow/providers/google/cloud/hooks/cloud_memorystore.py +3 -4
  16. airflow/providers/google/cloud/hooks/cloud_run.py +3 -4
  17. airflow/providers/google/cloud/hooks/cloud_sql.py +7 -3
  18. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +119 -7
  19. airflow/providers/google/cloud/hooks/compute.py +3 -3
  20. airflow/providers/google/cloud/hooks/datacatalog.py +3 -4
  21. airflow/providers/google/cloud/hooks/dataflow.py +12 -12
  22. airflow/providers/google/cloud/hooks/dataform.py +2 -3
  23. airflow/providers/google/cloud/hooks/datafusion.py +2 -2
  24. airflow/providers/google/cloud/hooks/dataplex.py +1032 -11
  25. airflow/providers/google/cloud/hooks/dataproc.py +4 -5
  26. airflow/providers/google/cloud/hooks/dataproc_metastore.py +3 -4
  27. airflow/providers/google/cloud/hooks/dlp.py +3 -4
  28. airflow/providers/google/cloud/hooks/gcs.py +7 -6
  29. airflow/providers/google/cloud/hooks/kms.py +2 -3
  30. airflow/providers/google/cloud/hooks/kubernetes_engine.py +8 -8
  31. airflow/providers/google/cloud/hooks/life_sciences.py +1 -1
  32. airflow/providers/google/cloud/hooks/managed_kafka.py +482 -0
  33. airflow/providers/google/cloud/hooks/natural_language.py +2 -3
  34. airflow/providers/google/cloud/hooks/os_login.py +2 -3
  35. airflow/providers/google/cloud/hooks/pubsub.py +6 -6
  36. airflow/providers/google/cloud/hooks/secret_manager.py +2 -3
  37. airflow/providers/google/cloud/hooks/spanner.py +2 -2
  38. airflow/providers/google/cloud/hooks/speech_to_text.py +2 -3
  39. airflow/providers/google/cloud/hooks/stackdriver.py +4 -4
  40. airflow/providers/google/cloud/hooks/tasks.py +3 -4
  41. airflow/providers/google/cloud/hooks/text_to_speech.py +2 -3
  42. airflow/providers/google/cloud/hooks/translate.py +236 -5
  43. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +9 -4
  44. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +3 -4
  45. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +4 -5
  46. airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +3 -4
  47. airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +2 -3
  48. airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +3 -4
  49. airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +1 -181
  50. airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +3 -4
  51. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +2 -3
  52. airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +3 -4
  53. airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +2 -3
  54. airflow/providers/google/cloud/hooks/video_intelligence.py +2 -3
  55. airflow/providers/google/cloud/hooks/vision.py +3 -4
  56. airflow/providers/google/cloud/hooks/workflows.py +2 -3
  57. airflow/providers/google/cloud/links/alloy_db.py +46 -0
  58. airflow/providers/google/cloud/links/bigquery.py +25 -0
  59. airflow/providers/google/cloud/links/dataplex.py +172 -2
  60. airflow/providers/google/cloud/links/kubernetes_engine.py +1 -2
  61. airflow/providers/google/cloud/links/managed_kafka.py +104 -0
  62. airflow/providers/google/cloud/links/translate.py +28 -0
  63. airflow/providers/google/cloud/log/gcs_task_handler.py +3 -3
  64. airflow/providers/google/cloud/log/stackdriver_task_handler.py +11 -10
  65. airflow/providers/google/cloud/openlineage/facets.py +67 -0
  66. airflow/providers/google/cloud/openlineage/mixins.py +438 -173
  67. airflow/providers/google/cloud/openlineage/utils.py +394 -61
  68. airflow/providers/google/cloud/operators/alloy_db.py +980 -69
  69. airflow/providers/google/cloud/operators/automl.py +83 -245
  70. airflow/providers/google/cloud/operators/bigquery.py +377 -74
  71. airflow/providers/google/cloud/operators/bigquery_dts.py +126 -13
  72. airflow/providers/google/cloud/operators/bigtable.py +1 -3
  73. airflow/providers/google/cloud/operators/cloud_base.py +1 -2
  74. airflow/providers/google/cloud/operators/cloud_batch.py +2 -4
  75. airflow/providers/google/cloud/operators/cloud_build.py +3 -5
  76. airflow/providers/google/cloud/operators/cloud_composer.py +5 -7
  77. airflow/providers/google/cloud/operators/cloud_memorystore.py +4 -6
  78. airflow/providers/google/cloud/operators/cloud_run.py +6 -5
  79. airflow/providers/google/cloud/operators/cloud_sql.py +20 -8
  80. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +62 -8
  81. airflow/providers/google/cloud/operators/compute.py +3 -4
  82. airflow/providers/google/cloud/operators/datacatalog.py +9 -11
  83. airflow/providers/google/cloud/operators/dataflow.py +1 -112
  84. airflow/providers/google/cloud/operators/dataform.py +3 -5
  85. airflow/providers/google/cloud/operators/datafusion.py +1 -1
  86. airflow/providers/google/cloud/operators/dataplex.py +2046 -7
  87. airflow/providers/google/cloud/operators/dataproc.py +102 -17
  88. airflow/providers/google/cloud/operators/dataproc_metastore.py +7 -9
  89. airflow/providers/google/cloud/operators/dlp.py +17 -19
  90. airflow/providers/google/cloud/operators/gcs.py +14 -17
  91. airflow/providers/google/cloud/operators/kubernetes_engine.py +2 -2
  92. airflow/providers/google/cloud/operators/managed_kafka.py +788 -0
  93. airflow/providers/google/cloud/operators/natural_language.py +3 -5
  94. airflow/providers/google/cloud/operators/pubsub.py +39 -7
  95. airflow/providers/google/cloud/operators/speech_to_text.py +3 -5
  96. airflow/providers/google/cloud/operators/stackdriver.py +3 -5
  97. airflow/providers/google/cloud/operators/tasks.py +4 -6
  98. airflow/providers/google/cloud/operators/text_to_speech.py +2 -4
  99. airflow/providers/google/cloud/operators/translate.py +414 -5
  100. airflow/providers/google/cloud/operators/translate_speech.py +2 -4
  101. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +9 -8
  102. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +4 -6
  103. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +6 -8
  104. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +4 -6
  105. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +4 -6
  106. airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +0 -322
  107. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +4 -6
  108. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +4 -6
  109. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +4 -6
  110. airflow/providers/google/cloud/operators/video_intelligence.py +3 -5
  111. airflow/providers/google/cloud/operators/vision.py +4 -6
  112. airflow/providers/google/cloud/operators/workflows.py +5 -7
  113. airflow/providers/google/cloud/secrets/secret_manager.py +1 -2
  114. airflow/providers/google/cloud/sensors/bigquery_dts.py +3 -5
  115. airflow/providers/google/cloud/sensors/bigtable.py +2 -3
  116. airflow/providers/google/cloud/sensors/cloud_composer.py +32 -8
  117. airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +39 -1
  118. airflow/providers/google/cloud/sensors/dataplex.py +4 -6
  119. airflow/providers/google/cloud/sensors/dataproc.py +2 -3
  120. airflow/providers/google/cloud/sensors/dataproc_metastore.py +1 -2
  121. airflow/providers/google/cloud/sensors/gcs.py +2 -4
  122. airflow/providers/google/cloud/sensors/pubsub.py +2 -3
  123. airflow/providers/google/cloud/sensors/workflows.py +3 -5
  124. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +5 -5
  125. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +10 -12
  126. airflow/providers/google/cloud/transfers/gcs_to_gcs.py +1 -1
  127. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +36 -4
  128. airflow/providers/google/cloud/transfers/mssql_to_gcs.py +27 -2
  129. airflow/providers/google/cloud/transfers/mysql_to_gcs.py +27 -2
  130. airflow/providers/google/cloud/transfers/postgres_to_gcs.py +27 -2
  131. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +34 -5
  132. airflow/providers/google/cloud/transfers/sql_to_gcs.py +15 -0
  133. airflow/providers/google/cloud/transfers/trino_to_gcs.py +25 -2
  134. airflow/providers/google/cloud/triggers/bigquery_dts.py +1 -2
  135. airflow/providers/google/cloud/triggers/cloud_batch.py +1 -2
  136. airflow/providers/google/cloud/triggers/cloud_build.py +1 -2
  137. airflow/providers/google/cloud/triggers/cloud_composer.py +13 -3
  138. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +102 -4
  139. airflow/providers/google/cloud/triggers/dataflow.py +2 -3
  140. airflow/providers/google/cloud/triggers/dataplex.py +1 -2
  141. airflow/providers/google/cloud/triggers/dataproc.py +2 -3
  142. airflow/providers/google/cloud/triggers/kubernetes_engine.py +1 -1
  143. airflow/providers/google/cloud/triggers/pubsub.py +1 -2
  144. airflow/providers/google/cloud/triggers/vertex_ai.py +7 -8
  145. airflow/providers/google/cloud/utils/credentials_provider.py +15 -8
  146. airflow/providers/google/cloud/utils/external_token_supplier.py +1 -0
  147. airflow/providers/google/common/auth_backend/google_openid.py +4 -4
  148. airflow/providers/google/common/consts.py +1 -2
  149. airflow/providers/google/common/hooks/base_google.py +8 -7
  150. airflow/providers/google/get_provider_info.py +186 -134
  151. airflow/providers/google/marketing_platform/hooks/analytics_admin.py +2 -3
  152. airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
  153. airflow/providers/google/marketing_platform/operators/analytics_admin.py +5 -7
  154. {apache_airflow_providers_google-12.0.0rc2.dist-info → apache_airflow_providers_google-14.0.0rc1.dist-info}/METADATA +40 -57
  155. {apache_airflow_providers_google-12.0.0rc2.dist-info → apache_airflow_providers_google-14.0.0rc1.dist-info}/RECORD +157 -159
  156. airflow/providers/google/cloud/example_dags/example_facebook_ads_to_gcs.py +0 -141
  157. airflow/providers/google/cloud/example_dags/example_looker.py +0 -64
  158. airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py +0 -194
  159. airflow/providers/google/cloud/example_dags/example_salesforce_to_gcs.py +0 -129
  160. airflow/providers/google/marketing_platform/example_dags/__init__.py +0 -16
  161. airflow/providers/google/marketing_platform/example_dags/example_display_video.py +0 -213
  162. {apache_airflow_providers_google-12.0.0rc2.dist-info → apache_airflow_providers_google-14.0.0rc1.dist-info}/WHEEL +0 -0
  163. {apache_airflow_providers_google-12.0.0rc2.dist-info → apache_airflow_providers_google-14.0.0rc1.dist-info}/entry_points.txt +0 -0
@@ -19,7 +19,8 @@
19
19
  from __future__ import annotations
20
20
 
21
21
  import time
22
- from collections.abc import Sequence
22
+ from collections.abc import MutableSequence, Sequence
23
+ from functools import cached_property
23
24
  from typing import TYPE_CHECKING, Any
24
25
 
25
26
  from airflow.exceptions import AirflowException
@@ -27,26 +28,50 @@ from airflow.providers.google.cloud.triggers.dataplex import (
27
28
  DataplexDataProfileJobTrigger,
28
29
  DataplexDataQualityJobTrigger,
29
30
  )
31
+ from google.protobuf.json_format import MessageToDict
30
32
 
31
33
  if TYPE_CHECKING:
32
- from google.protobuf.field_mask_pb2 import FieldMask
33
-
34
34
  from airflow.utils.context import Context
35
+ from google.protobuf.field_mask_pb2 import FieldMask
35
36
 
36
- from google.api_core.exceptions import AlreadyExists, GoogleAPICallError
37
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
38
- from google.api_core.retry import Retry, exponential_sleep_generator
39
- from google.cloud.dataplex_v1.types import Asset, DataScan, DataScanJob, Lake, Task, Zone
40
37
  from googleapiclient.errors import HttpError
41
38
 
42
39
  from airflow.configuration import conf
43
40
  from airflow.providers.google.cloud.hooks.dataplex import AirflowDataQualityScanException, DataplexHook
44
41
  from airflow.providers.google.cloud.links.dataplex import (
42
+ DataplexCatalogAspectTypeLink,
43
+ DataplexCatalogAspectTypesLink,
44
+ DataplexCatalogEntryGroupLink,
45
+ DataplexCatalogEntryGroupsLink,
46
+ DataplexCatalogEntryLink,
47
+ DataplexCatalogEntryTypeLink,
48
+ DataplexCatalogEntryTypesLink,
45
49
  DataplexLakeLink,
46
50
  DataplexTaskLink,
47
51
  DataplexTasksLink,
48
52
  )
49
53
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
54
+ from google.api_core.exceptions import AlreadyExists, GoogleAPICallError, NotFound
55
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
56
+ from google.api_core.retry import Retry, exponential_sleep_generator
57
+ from google.cloud.dataplex_v1.types import (
58
+ AspectType,
59
+ Asset,
60
+ DataScan,
61
+ DataScanJob,
62
+ Entry,
63
+ EntryGroup,
64
+ EntryType,
65
+ EntryView,
66
+ Lake,
67
+ ListAspectTypesResponse,
68
+ ListEntriesResponse,
69
+ ListEntryGroupsResponse,
70
+ ListEntryTypesResponse,
71
+ SearchEntriesResponse,
72
+ Task,
73
+ Zone,
74
+ )
50
75
 
51
76
 
52
77
  class DataplexCreateTaskOperator(GoogleCloudBaseOperator):
@@ -2093,3 +2118,2017 @@ class DataplexDeleteAssetOperator(GoogleCloudBaseOperator):
2093
2118
  )
2094
2119
  hook.wait_for_operation(timeout=self.timeout, operation=operation)
2095
2120
  self.log.info("Dataplex asset %s deleted successfully!", self.asset_id)
2121
+
2122
+
2123
+ class DataplexCatalogBaseOperator(GoogleCloudBaseOperator):
2124
+ """
2125
+ Base class for all Dataplex Catalog operators.
2126
+
2127
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
2128
+ :param location: Required. The ID of the Google Cloud region where the service is used.
2129
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
2130
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
2131
+ be retried.
2132
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
2133
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
2134
+ :param metadata: Optional. Additional metadata that is provided to the method.
2135
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
2136
+ credentials, or chained list of accounts required to get the access_token
2137
+ of the last account in the list, which will be impersonated in the request.
2138
+ If set as a string, the account must grant the originating account
2139
+ the Service Account Token Creator IAM role.
2140
+ If set as a sequence, the identities from the list must grant
2141
+ Service Account Token Creator IAM role to the directly preceding identity, with first
2142
+ account from the list granting this role to the originating account (templated).
2143
+ """
2144
+
2145
+ template_fields: Sequence[str] = (
2146
+ "project_id",
2147
+ "location",
2148
+ "gcp_conn_id",
2149
+ "impersonation_chain",
2150
+ )
2151
+
2152
+ def __init__(
2153
+ self,
2154
+ project_id: str,
2155
+ location: str,
2156
+ gcp_conn_id: str = "google_cloud_default",
2157
+ retry: Retry | _MethodDefault = DEFAULT,
2158
+ timeout: float | None = None,
2159
+ metadata: Sequence[tuple[str, str]] = (),
2160
+ impersonation_chain: str | Sequence[str] | None = None,
2161
+ *args,
2162
+ **kwargs,
2163
+ ):
2164
+ super().__init__(*args, **kwargs)
2165
+ self.project_id = project_id
2166
+ self.location = location
2167
+ self.gcp_conn_id = gcp_conn_id
2168
+ self.impersonation_chain = impersonation_chain
2169
+ self.retry = retry
2170
+ self.timeout = timeout
2171
+ self.metadata = metadata
2172
+
2173
+ @cached_property
2174
+ def hook(self) -> DataplexHook:
2175
+ return DataplexHook(
2176
+ gcp_conn_id=self.gcp_conn_id,
2177
+ impersonation_chain=self.impersonation_chain,
2178
+ )
2179
+
2180
+
2181
+ class DataplexCatalogCreateEntryGroupOperator(DataplexCatalogBaseOperator):
2182
+ """
2183
+ Create an EntryGroup resource.
2184
+
2185
+ .. seealso::
2186
+ For more information on how to use this operator, take a look at the guide:
2187
+ :ref:`howto/operator:DataplexCatalogCreateEntryGroupOperator`
2188
+
2189
+ :param entry_group_id: Required. EntryGroup identifier.
2190
+ :param entry_group_configuration: Required. EntryGroup configuration.
2191
+ For more details please see API documentation:
2192
+ https://cloud.google.com/dataplex/docs/reference/rest/v1/projects.locations.entryGroups#EntryGroup
2193
+ :param validate_request: Optional. If set, performs request validation, but does not actually
2194
+ execute the request.
2195
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
2196
+ :param location: Required. The ID of the Google Cloud region where the service is used.
2197
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
2198
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
2199
+ be retried.
2200
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
2201
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
2202
+ :param metadata: Optional. Additional metadata that is provided to the method.
2203
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
2204
+ credentials, or chained list of accounts required to get the access_token
2205
+ of the last account in the list, which will be impersonated in the request.
2206
+ If set as a string, the account must grant the originating account
2207
+ the Service Account Token Creator IAM role.
2208
+ If set as a sequence, the identities from the list must grant
2209
+ Service Account Token Creator IAM role to the directly preceding identity, with first
2210
+ account from the list granting this role to the originating account (templated).
2211
+ """
2212
+
2213
+ template_fields: Sequence[str] = tuple(
2214
+ {"entry_group_id", "entry_group_configuration"} | set(DataplexCatalogBaseOperator.template_fields)
2215
+ )
2216
+ operator_extra_links = (DataplexCatalogEntryGroupLink(),)
2217
+
2218
+ def __init__(
2219
+ self,
2220
+ entry_group_id: str,
2221
+ entry_group_configuration: EntryGroup | dict,
2222
+ validate_request: bool = False,
2223
+ *args,
2224
+ **kwargs,
2225
+ ) -> None:
2226
+ super().__init__(*args, **kwargs)
2227
+ self.entry_group_id = entry_group_id
2228
+ self.entry_group_configuration = entry_group_configuration
2229
+ self.validate_request = validate_request
2230
+
2231
+ def execute(self, context: Context):
2232
+ DataplexCatalogEntryGroupLink.persist(
2233
+ context=context,
2234
+ task_instance=self,
2235
+ )
2236
+
2237
+ if self.validate_request:
2238
+ self.log.info("Validating a Create Dataplex Catalog EntryGroup request.")
2239
+ else:
2240
+ self.log.info("Creating a Dataplex Catalog EntryGroup.")
2241
+
2242
+ try:
2243
+ operation = self.hook.create_entry_group(
2244
+ entry_group_id=self.entry_group_id,
2245
+ entry_group_configuration=self.entry_group_configuration,
2246
+ location=self.location,
2247
+ project_id=self.project_id,
2248
+ validate_only=self.validate_request,
2249
+ retry=self.retry,
2250
+ timeout=self.timeout,
2251
+ metadata=self.metadata,
2252
+ )
2253
+ entry_group = self.hook.wait_for_operation(timeout=self.timeout, operation=operation)
2254
+ except AlreadyExists:
2255
+ entry_group = self.hook.get_entry_group(
2256
+ entry_group_id=self.entry_group_id,
2257
+ location=self.location,
2258
+ project_id=self.project_id,
2259
+ )
2260
+ self.log.info(
2261
+ "Dataplex Catalog EntryGroup %s already exists.",
2262
+ self.entry_group_id,
2263
+ )
2264
+ result = EntryGroup.to_dict(entry_group)
2265
+ return result
2266
+ except Exception as ex:
2267
+ raise AirflowException(ex)
2268
+ else:
2269
+ result = EntryGroup.to_dict(entry_group) if not self.validate_request else None
2270
+
2271
+ if not self.validate_request:
2272
+ self.log.info("Dataplex Catalog EntryGroup %s was successfully created.", self.entry_group_id)
2273
+ return result
2274
+
2275
+
2276
+ class DataplexCatalogGetEntryGroupOperator(DataplexCatalogBaseOperator):
2277
+ """
2278
+ Get an EntryGroup resource.
2279
+
2280
+ .. seealso::
2281
+ For more information on how to use this operator, take a look at the guide:
2282
+ :ref:`howto/operator:DataplexCatalogGetEntryGroupOperator`
2283
+
2284
+ :param entry_group_id: Required. EntryGroup identifier.
2285
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
2286
+ :param location: Required. The ID of the Google Cloud region where the service is used.
2287
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
2288
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
2289
+ be retried.
2290
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
2291
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
2292
+ :param metadata: Optional. Additional metadata that is provided to the method.
2293
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
2294
+ credentials, or chained list of accounts required to get the access_token
2295
+ of the last account in the list, which will be impersonated in the request.
2296
+ If set as a string, the account must grant the originating account
2297
+ the Service Account Token Creator IAM role.
2298
+ If set as a sequence, the identities from the list must grant
2299
+ Service Account Token Creator IAM role to the directly preceding identity, with first
2300
+ account from the list granting this role to the originating account (templated).
2301
+ """
2302
+
2303
+ template_fields: Sequence[str] = tuple(
2304
+ {"entry_group_id"} | set(DataplexCatalogBaseOperator.template_fields)
2305
+ )
2306
+ operator_extra_links = (DataplexCatalogEntryGroupLink(),)
2307
+
2308
+ def __init__(
2309
+ self,
2310
+ entry_group_id: str,
2311
+ *args,
2312
+ **kwargs,
2313
+ ) -> None:
2314
+ super().__init__(*args, **kwargs)
2315
+ self.entry_group_id = entry_group_id
2316
+
2317
+ def execute(self, context: Context):
2318
+ DataplexCatalogEntryGroupLink.persist(
2319
+ context=context,
2320
+ task_instance=self,
2321
+ )
2322
+ self.log.info(
2323
+ "Retrieving Dataplex Catalog EntryGroup %s.",
2324
+ self.entry_group_id,
2325
+ )
2326
+ try:
2327
+ entry_group = self.hook.get_entry_group(
2328
+ entry_group_id=self.entry_group_id,
2329
+ location=self.location,
2330
+ project_id=self.project_id,
2331
+ retry=self.retry,
2332
+ timeout=self.timeout,
2333
+ metadata=self.metadata,
2334
+ )
2335
+ except NotFound:
2336
+ self.log.info(
2337
+ "Dataplex Catalog EntryGroup %s not found.",
2338
+ self.entry_group_id,
2339
+ )
2340
+ raise AirflowException(NotFound)
2341
+ except Exception as ex:
2342
+ raise AirflowException(ex)
2343
+
2344
+ return EntryGroup.to_dict(entry_group)
2345
+
2346
+
2347
+ class DataplexCatalogDeleteEntryGroupOperator(DataplexCatalogBaseOperator):
2348
+ """
2349
+ Delete an EntryGroup resource.
2350
+
2351
+ .. seealso::
2352
+ For more information on how to use this operator, take a look at the guide:
2353
+ :ref:`howto/operator:DataplexCatalogDeleteEntryGroupOperator`
2354
+
2355
+ :param entry_group_id: Required. EntryGroup identifier.
2356
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
2357
+ :param location: Required. The ID of the Google Cloud region where the service is used.
2358
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
2359
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
2360
+ be retried.
2361
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
2362
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
2363
+ :param metadata: Optional. Additional metadata that is provided to the method.
2364
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
2365
+ credentials, or chained list of accounts required to get the access_token
2366
+ of the last account in the list, which will be impersonated in the request.
2367
+ If set as a string, the account must grant the originating account
2368
+ the Service Account Token Creator IAM role.
2369
+ If set as a sequence, the identities from the list must grant
2370
+ Service Account Token Creator IAM role to the directly preceding identity, with first
2371
+ account from the list granting this role to the originating account (templated).
2372
+ """
2373
+
2374
+ template_fields: Sequence[str] = tuple(
2375
+ {"entry_group_id"} | set(DataplexCatalogBaseOperator.template_fields)
2376
+ )
2377
+
2378
+ def __init__(
2379
+ self,
2380
+ entry_group_id: str,
2381
+ *args,
2382
+ **kwargs,
2383
+ ) -> None:
2384
+ super().__init__(*args, **kwargs)
2385
+ self.entry_group_id = entry_group_id
2386
+
2387
+ def execute(self, context: Context):
2388
+ self.log.info(
2389
+ "Deleting Dataplex Catalog EntryGroup %s.",
2390
+ self.entry_group_id,
2391
+ )
2392
+ try:
2393
+ operation = self.hook.delete_entry_group(
2394
+ entry_group_id=self.entry_group_id,
2395
+ location=self.location,
2396
+ project_id=self.project_id,
2397
+ retry=self.retry,
2398
+ timeout=self.timeout,
2399
+ metadata=self.metadata,
2400
+ )
2401
+ self.hook.wait_for_operation(timeout=self.timeout, operation=operation)
2402
+
2403
+ except NotFound:
2404
+ self.log.info(
2405
+ "Dataplex Catalog EntryGroup %s not found.",
2406
+ self.entry_group_id,
2407
+ )
2408
+ raise AirflowException(NotFound)
2409
+ except Exception as ex:
2410
+ raise AirflowException(ex)
2411
+ return None
2412
+
2413
+
2414
+ class DataplexCatalogListEntryGroupsOperator(DataplexCatalogBaseOperator):
2415
+ """
2416
+ List EntryGroup resources.
2417
+
2418
+ .. seealso::
2419
+ For more information on how to use this operator, take a look at the guide:
2420
+ :ref:`howto/operator:DataplexCatalogListEntryGroupsOperator`
2421
+
2422
+ :param filter_by: Optional. Filter to apply on the list results.
2423
+ :param order_by: Optional. Fields to order the results by.
2424
+ :param page_size: Optional. Maximum number of EntryGroups to return on the page.
2425
+ :param page_token: Optional. Token to retrieve the next page of results.
2426
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
2427
+ :param location: Required. The ID of the Google Cloud region where the service is used.
2428
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
2429
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
2430
+ be retried.
2431
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
2432
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
2433
+ :param metadata: Optional. Additional metadata that is provided to the method.
2434
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
2435
+ credentials, or chained list of accounts required to get the access_token
2436
+ of the last account in the list, which will be impersonated in the request.
2437
+ If set as a string, the account must grant the originating account
2438
+ the Service Account Token Creator IAM role.
2439
+ If set as a sequence, the identities from the list must grant
2440
+ Service Account Token Creator IAM role to the directly preceding identity, with first
2441
+ account from the list granting this role to the originating account (templated).
2442
+ """
2443
+
2444
+ template_fields: Sequence[str] = tuple(DataplexCatalogBaseOperator.template_fields)
2445
+ operator_extra_links = (DataplexCatalogEntryGroupsLink(),)
2446
+
2447
+ def __init__(
2448
+ self,
2449
+ page_size: int | None = None,
2450
+ page_token: str | None = None,
2451
+ filter_by: str | None = None,
2452
+ order_by: str | None = None,
2453
+ *args,
2454
+ **kwargs,
2455
+ ) -> None:
2456
+ super().__init__(*args, **kwargs)
2457
+ self.page_size = page_size
2458
+ self.page_token = page_token
2459
+ self.filter_by = filter_by
2460
+ self.order_by = order_by
2461
+
2462
+ def execute(self, context: Context):
2463
+ DataplexCatalogEntryGroupsLink.persist(
2464
+ context=context,
2465
+ task_instance=self,
2466
+ )
2467
+ self.log.info(
2468
+ "Listing Dataplex Catalog EntryGroup from location %s.",
2469
+ self.location,
2470
+ )
2471
+ try:
2472
+ entry_group_on_page = self.hook.list_entry_groups(
2473
+ location=self.location,
2474
+ project_id=self.project_id,
2475
+ page_size=self.page_size,
2476
+ page_token=self.page_token,
2477
+ filter_by=self.filter_by,
2478
+ order_by=self.order_by,
2479
+ retry=self.retry,
2480
+ timeout=self.timeout,
2481
+ metadata=self.metadata,
2482
+ )
2483
+ self.log.info("EntryGroup on page: %s", entry_group_on_page)
2484
+ self.xcom_push(
2485
+ context=context,
2486
+ key="entry_group_page",
2487
+ value=ListEntryGroupsResponse.to_dict(entry_group_on_page._response),
2488
+ )
2489
+ except Exception as ex:
2490
+ raise AirflowException(ex)
2491
+
2492
+ # Constructing list to return EntryGroups in readable format
2493
+ entry_groups_list = [
2494
+ MessageToDict(entry_group._pb, preserving_proto_field_name=True)
2495
+ for entry_group in next(iter(entry_group_on_page.pages)).entry_groups
2496
+ ]
2497
+ return entry_groups_list
2498
+
2499
+
2500
+ class DataplexCatalogUpdateEntryGroupOperator(DataplexCatalogBaseOperator):
2501
+ """
2502
+ Update an EntryGroup resource.
2503
+
2504
+ .. seealso::
2505
+ For more information on how to use this operator, take a look at the guide:
2506
+ :ref:`howto/operator:DataplexCatalogUpdateEntryGroupOperator`
2507
+
2508
+ :param project_id: Required. The ID of the Google Cloud project that the task belongs to.
2509
+ :param location: Required. The ID of the Google Cloud region that the task belongs to.
2510
+ :param update_mask: Optional. Names of fields whose values to overwrite on an entry group.
2511
+ If this parameter is absent or empty, all modifiable fields are overwritten. If such
2512
+ fields are non-required and omitted in the request body, their values are emptied.
2513
+ :param entry_group_id: Required. ID of the EntryGroup to update.
2514
+ :param entry_group_configuration: Required. The updated configuration body of the EntryGroup.
2515
+ For more details please see API documentation:
2516
+ https://cloud.google.com/dataplex/docs/reference/rest/v1/projects.locations.entryGroups#EntryGroup
2517
+ :param validate_only: Optional. The service validates the request without performing any mutations.
2518
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests
2519
+ will not be retried.
2520
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
2521
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
2522
+ :param metadata: Optional. Additional metadata that is provided to the method.
2523
+ :param gcp_conn_id: Optional. The connection ID to use when fetching connection info.
2524
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
2525
+ credentials, or chained list of accounts required to get the access_token
2526
+ of the last account in the list, which will be impersonated in the request.
2527
+ If set as a string, the account must grant the originating account
2528
+ the Service Account Token Creator IAM role.
2529
+ If set as a sequence, the identities from the list must grant
2530
+ Service Account Token Creator IAM role to the directly preceding identity, with first
2531
+ account from the list granting this role to the originating account (templated).
2532
+ """
2533
+
2534
+ template_fields: Sequence[str] = tuple(
2535
+ {"entry_group_id", "entry_group_configuration", "update_mask"}
2536
+ | set(DataplexCatalogBaseOperator.template_fields)
2537
+ )
2538
+ operator_extra_links = (DataplexCatalogEntryGroupLink(),)
2539
+
2540
+ def __init__(
2541
+ self,
2542
+ entry_group_id: str,
2543
+ entry_group_configuration: dict | EntryGroup,
2544
+ update_mask: list[str] | FieldMask | None = None,
2545
+ validate_request: bool | None = False,
2546
+ *args,
2547
+ **kwargs,
2548
+ ) -> None:
2549
+ super().__init__(*args, **kwargs)
2550
+ self.entry_group_id = entry_group_id
2551
+ self.entry_group_configuration = entry_group_configuration
2552
+ self.update_mask = update_mask
2553
+ self.validate_request = validate_request
2554
+
2555
+ def execute(self, context: Context):
2556
+ DataplexCatalogEntryGroupLink.persist(
2557
+ context=context,
2558
+ task_instance=self,
2559
+ )
2560
+
2561
+ if self.validate_request:
2562
+ self.log.info("Validating an Update Dataplex Catalog EntryGroup request.")
2563
+ else:
2564
+ self.log.info(
2565
+ "Updating Dataplex Catalog EntryGroup %s.",
2566
+ self.entry_group_id,
2567
+ )
2568
+ try:
2569
+ operation = self.hook.update_entry_group(
2570
+ location=self.location,
2571
+ project_id=self.project_id,
2572
+ entry_group_id=self.entry_group_id,
2573
+ entry_group_configuration=self.entry_group_configuration,
2574
+ update_mask=self.update_mask,
2575
+ validate_only=self.validate_request,
2576
+ retry=self.retry,
2577
+ timeout=self.timeout,
2578
+ metadata=self.metadata,
2579
+ )
2580
+ entry_group = self.hook.wait_for_operation(timeout=self.timeout, operation=operation)
2581
+
2582
+ except NotFound as ex:
2583
+ self.log.info("Specified EntryGroup was not found.")
2584
+ raise AirflowException(ex)
2585
+ except Exception as exc:
2586
+ raise AirflowException(exc)
2587
+ else:
2588
+ result = EntryGroup.to_dict(entry_group) if not self.validate_request else None
2589
+
2590
+ if not self.validate_request:
2591
+ self.log.info("EntryGroup %s was successfully updated.", self.entry_group_id)
2592
+ return result
2593
+
2594
+
2595
+ class DataplexCatalogCreateEntryTypeOperator(DataplexCatalogBaseOperator):
2596
+ """
2597
+ Create an EntryType resource.
2598
+
2599
+ .. seealso::
2600
+ For more information on how to use this operator, take a look at the guide:
2601
+ :ref:`howto/operator:DataplexCatalogCreateEntryTypeOperator`
2602
+
2603
+ :param entry_type_id: Required. EntryType identifier.
2604
+ :param entry_type_configuration: Required. EntryType configuration.
2605
+ For more details please see API documentation:
2606
+ https://cloud.google.com/dataplex/docs/reference/rest/v1/projects.locations.entryGroups#EntryGroup
2607
+ :param validate_request: Optional. If set, performs request validation, but does not actually
2608
+ execute the request.
2609
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
2610
+ :param location: Required. The ID of the Google Cloud region where the service is used.
2611
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
2612
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
2613
+ be retried.
2614
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
2615
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
2616
+ :param metadata: Optional. Additional metadata that is provided to the method.
2617
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
2618
+ credentials, or chained list of accounts required to get the access_token
2619
+ of the last account in the list, which will be impersonated in the request.
2620
+ If set as a string, the account must grant the originating account
2621
+ the Service Account Token Creator IAM role.
2622
+ If set as a sequence, the identities from the list must grant
2623
+ Service Account Token Creator IAM role to the directly preceding identity, with first
2624
+ account from the list granting this role to the originating account (templated).
2625
+ """
2626
+
2627
+ template_fields: Sequence[str] = tuple(
2628
+ {"entry_type_id", "entry_type_configuration"} | set(DataplexCatalogBaseOperator.template_fields)
2629
+ )
2630
+ operator_extra_links = (DataplexCatalogEntryTypeLink(),)
2631
+
2632
+ def __init__(
2633
+ self,
2634
+ entry_type_id: str,
2635
+ entry_type_configuration: EntryType | dict,
2636
+ validate_request: bool = False,
2637
+ *args,
2638
+ **kwargs,
2639
+ ) -> None:
2640
+ super().__init__(*args, **kwargs)
2641
+ self.entry_type_id = entry_type_id
2642
+ self.entry_type_configuration = entry_type_configuration
2643
+ self.validate_request = validate_request
2644
+
2645
+ def execute(self, context: Context):
2646
+ DataplexCatalogEntryTypeLink.persist(
2647
+ context=context,
2648
+ task_instance=self,
2649
+ )
2650
+
2651
+ if self.validate_request:
2652
+ self.log.info("Validating a Create Dataplex Catalog EntryType request.")
2653
+ else:
2654
+ self.log.info("Creating a Dataplex Catalog EntryType.")
2655
+
2656
+ try:
2657
+ operation = self.hook.create_entry_type(
2658
+ entry_type_id=self.entry_type_id,
2659
+ entry_type_configuration=self.entry_type_configuration,
2660
+ location=self.location,
2661
+ project_id=self.project_id,
2662
+ validate_only=self.validate_request,
2663
+ retry=self.retry,
2664
+ timeout=self.timeout,
2665
+ metadata=self.metadata,
2666
+ )
2667
+ entry_type = self.hook.wait_for_operation(timeout=self.timeout, operation=operation)
2668
+ except AlreadyExists:
2669
+ entry_type = self.hook.get_entry_type(
2670
+ entry_type_id=self.entry_type_id,
2671
+ location=self.location,
2672
+ project_id=self.project_id,
2673
+ )
2674
+ self.log.info(
2675
+ "Dataplex Catalog EntryType %s already exists.",
2676
+ self.entry_type_id,
2677
+ )
2678
+ result = EntryType.to_dict(entry_type)
2679
+ return result
2680
+ except Exception as ex:
2681
+ raise AirflowException(ex)
2682
+ else:
2683
+ result = EntryType.to_dict(entry_type) if not self.validate_request else None
2684
+
2685
+ if not self.validate_request:
2686
+ self.log.info("Dataplex Catalog EntryType %s was successfully created.", self.entry_type_id)
2687
+ return result
2688
+
2689
+
2690
+ class DataplexCatalogGetEntryTypeOperator(DataplexCatalogBaseOperator):
2691
+ """
2692
+ Get an EntryType resource.
2693
+
2694
+ .. seealso::
2695
+ For more information on how to use this operator, take a look at the guide:
2696
+ :ref:`howto/operator:DataplexCatalogGetEntryTypeOperator`
2697
+
2698
+ :param entry_type_id: Required. EntryType identifier.
2699
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
2700
+ :param location: Required. The ID of the Google Cloud region where the service is used.
2701
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
2702
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
2703
+ be retried.
2704
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
2705
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
2706
+ :param metadata: Optional. Additional metadata that is provided to the method.
2707
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
2708
+ credentials, or chained list of accounts required to get the access_token
2709
+ of the last account in the list, which will be impersonated in the request.
2710
+ If set as a string, the account must grant the originating account
2711
+ the Service Account Token Creator IAM role.
2712
+ If set as a sequence, the identities from the list must grant
2713
+ Service Account Token Creator IAM role to the directly preceding identity, with first
2714
+ account from the list granting this role to the originating account (templated).
2715
+ """
2716
+
2717
+ template_fields: Sequence[str] = tuple(
2718
+ {"entry_type_id"} | set(DataplexCatalogBaseOperator.template_fields)
2719
+ )
2720
+ operator_extra_links = (DataplexCatalogEntryTypeLink(),)
2721
+
2722
+ def __init__(
2723
+ self,
2724
+ entry_type_id: str,
2725
+ *args,
2726
+ **kwargs,
2727
+ ) -> None:
2728
+ super().__init__(*args, **kwargs)
2729
+ self.entry_type_id = entry_type_id
2730
+
2731
+ def execute(self, context: Context):
2732
+ DataplexCatalogEntryTypeLink.persist(
2733
+ context=context,
2734
+ task_instance=self,
2735
+ )
2736
+ self.log.info(
2737
+ "Retrieving Dataplex Catalog EntryType %s.",
2738
+ self.entry_type_id,
2739
+ )
2740
+ try:
2741
+ entry_type = self.hook.get_entry_type(
2742
+ entry_type_id=self.entry_type_id,
2743
+ location=self.location,
2744
+ project_id=self.project_id,
2745
+ retry=self.retry,
2746
+ timeout=self.timeout,
2747
+ metadata=self.metadata,
2748
+ )
2749
+ except NotFound:
2750
+ self.log.info(
2751
+ "Dataplex Catalog EntryType %s not found.",
2752
+ self.entry_type_id,
2753
+ )
2754
+ raise AirflowException(NotFound)
2755
+ except Exception as ex:
2756
+ raise AirflowException(ex)
2757
+
2758
+ return EntryType.to_dict(entry_type)
2759
+
2760
+
2761
+ class DataplexCatalogDeleteEntryTypeOperator(DataplexCatalogBaseOperator):
2762
+ """
2763
+ Delete an EntryType resource.
2764
+
2765
+ .. seealso::
2766
+ For more information on how to use this operator, take a look at the guide:
2767
+ :ref:`howto/operator:DataplexCatalogDeleteEntryTypeOperator`
2768
+
2769
+ :param entry_type_id: Required. EntryType identifier.
2770
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
2771
+ :param location: Required. The ID of the Google Cloud region where the service is used.
2772
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
2773
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
2774
+ be retried.
2775
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
2776
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
2777
+ :param metadata: Optional. Additional metadata that is provided to the method.
2778
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
2779
+ credentials, or chained list of accounts required to get the access_token
2780
+ of the last account in the list, which will be impersonated in the request.
2781
+ If set as a string, the account must grant the originating account
2782
+ the Service Account Token Creator IAM role.
2783
+ If set as a sequence, the identities from the list must grant
2784
+ Service Account Token Creator IAM role to the directly preceding identity, with first
2785
+ account from the list granting this role to the originating account (templated).
2786
+ """
2787
+
2788
+ template_fields: Sequence[str] = tuple(
2789
+ {"entry_type_id"} | set(DataplexCatalogBaseOperator.template_fields)
2790
+ )
2791
+
2792
+ def __init__(
2793
+ self,
2794
+ entry_type_id: str,
2795
+ *args,
2796
+ **kwargs,
2797
+ ) -> None:
2798
+ super().__init__(*args, **kwargs)
2799
+ self.entry_type_id = entry_type_id
2800
+
2801
+ def execute(self, context: Context):
2802
+ self.log.info(
2803
+ "Deleting Dataplex Catalog EntryType %s.",
2804
+ self.entry_type_id,
2805
+ )
2806
+ try:
2807
+ operation = self.hook.delete_entry_type(
2808
+ entry_type_id=self.entry_type_id,
2809
+ location=self.location,
2810
+ project_id=self.project_id,
2811
+ retry=self.retry,
2812
+ timeout=self.timeout,
2813
+ metadata=self.metadata,
2814
+ )
2815
+ self.hook.wait_for_operation(timeout=self.timeout, operation=operation)
2816
+
2817
+ except NotFound:
2818
+ self.log.info(
2819
+ "Dataplex Catalog EntryType %s not found.",
2820
+ self.entry_type_id,
2821
+ )
2822
+ raise AirflowException(NotFound)
2823
+ except Exception as ex:
2824
+ raise AirflowException(ex)
2825
+ return None
2826
+
2827
+
2828
+ class DataplexCatalogListEntryTypesOperator(DataplexCatalogBaseOperator):
2829
+ """
2830
+ List EntryType resources.
2831
+
2832
+ .. seealso::
2833
+ For more information on how to use this operator, take a look at the guide:
2834
+ :ref:`howto/operator:DataplexCatalogListEntryTypesOperator`
2835
+
2836
+ :param filter_by: Optional. Filter to apply on the list results.
2837
+ :param order_by: Optional. Fields to order the results by.
2838
+ :param page_size: Optional. Maximum number of EntryTypes to return on the page.
2839
+ :param page_token: Optional. Token to retrieve the next page of results.
2840
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
2841
+ :param location: Required. The ID of the Google Cloud region where the service is used.
2842
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
2843
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
2844
+ be retried.
2845
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
2846
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
2847
+ :param metadata: Optional. Additional metadata that is provided to the method.
2848
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
2849
+ credentials, or chained list of accounts required to get the access_token
2850
+ of the last account in the list, which will be impersonated in the request.
2851
+ If set as a string, the account must grant the originating account
2852
+ the Service Account Token Creator IAM role.
2853
+ If set as a sequence, the identities from the list must grant
2854
+ Service Account Token Creator IAM role to the directly preceding identity, with first
2855
+ account from the list granting this role to the originating account (templated).
2856
+ """
2857
+
2858
+ template_fields: Sequence[str] = tuple(DataplexCatalogBaseOperator.template_fields)
2859
+ operator_extra_links = (DataplexCatalogEntryTypesLink(),)
2860
+
2861
+ def __init__(
2862
+ self,
2863
+ page_size: int | None = None,
2864
+ page_token: str | None = None,
2865
+ filter_by: str | None = None,
2866
+ order_by: str | None = None,
2867
+ *args,
2868
+ **kwargs,
2869
+ ) -> None:
2870
+ super().__init__(*args, **kwargs)
2871
+ self.page_size = page_size
2872
+ self.page_token = page_token
2873
+ self.filter_by = filter_by
2874
+ self.order_by = order_by
2875
+
2876
+ def execute(self, context: Context):
2877
+ DataplexCatalogEntryTypesLink.persist(
2878
+ context=context,
2879
+ task_instance=self,
2880
+ )
2881
+ self.log.info(
2882
+ "Listing Dataplex Catalog EntryType from location %s.",
2883
+ self.location,
2884
+ )
2885
+ try:
2886
+ entry_type_on_page = self.hook.list_entry_types(
2887
+ location=self.location,
2888
+ project_id=self.project_id,
2889
+ page_size=self.page_size,
2890
+ page_token=self.page_token,
2891
+ filter_by=self.filter_by,
2892
+ order_by=self.order_by,
2893
+ retry=self.retry,
2894
+ timeout=self.timeout,
2895
+ metadata=self.metadata,
2896
+ )
2897
+ self.log.info("EntryType on page: %s", entry_type_on_page)
2898
+ self.xcom_push(
2899
+ context=context,
2900
+ key="entry_type_page",
2901
+ value=ListEntryTypesResponse.to_dict(entry_type_on_page._response),
2902
+ )
2903
+ except Exception as ex:
2904
+ raise AirflowException(ex)
2905
+
2906
+ # Constructing list to return EntryTypes in readable format
2907
+ entry_types_list = [
2908
+ MessageToDict(entry_type._pb, preserving_proto_field_name=True)
2909
+ for entry_type in next(iter(entry_type_on_page.pages)).entry_types
2910
+ ]
2911
+ return entry_types_list
2912
+
2913
+
2914
+ class DataplexCatalogUpdateEntryTypeOperator(DataplexCatalogBaseOperator):
2915
+ """
2916
+ Update an EntryType resource.
2917
+
2918
+ .. seealso::
2919
+ For more information on how to use this operator, take a look at the guide:
2920
+ :ref:`howto/operator:DataplexCatalogUpdateEntryTypeOperator`
2921
+
2922
+ :param project_id: Required. The ID of the Google Cloud project that the task belongs to.
2923
+ :param location: Required. The ID of the Google Cloud region that the task belongs to.
2924
+ :param update_mask: Optional. Names of fields whose values to overwrite on an entry group.
2925
+ If this parameter is absent or empty, all modifiable fields are overwritten. If such
2926
+ fields are non-required and omitted in the request body, their values are emptied.
2927
+ :param entry_type_id: Required. ID of the EntryType to update.
2928
+ :param entry_type_configuration: Required. The updated configuration body of the EntryType.
2929
+ For more details please see API documentation:
2930
+ https://cloud.google.com/dataplex/docs/reference/rest/v1/projects.locations.entryGroups#EntryGroup
2931
+ :param validate_only: Optional. The service validates the request without performing any mutations.
2932
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests
2933
+ will not be retried.
2934
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
2935
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
2936
+ :param metadata: Optional. Additional metadata that is provided to the method.
2937
+ :param gcp_conn_id: Optional. The connection ID to use when fetching connection info.
2938
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
2939
+ credentials, or chained list of accounts required to get the access_token
2940
+ of the last account in the list, which will be impersonated in the request.
2941
+ If set as a string, the account must grant the originating account
2942
+ the Service Account Token Creator IAM role.
2943
+ If set as a sequence, the identities from the list must grant
2944
+ Service Account Token Creator IAM role to the directly preceding identity, with first
2945
+ account from the list granting this role to the originating account (templated).
2946
+ """
2947
+
2948
+ template_fields: Sequence[str] = tuple(
2949
+ {"entry_type_id", "entry_type_configuration", "update_mask"}
2950
+ | set(DataplexCatalogBaseOperator.template_fields)
2951
+ )
2952
+ operator_extra_links = (DataplexCatalogEntryTypeLink(),)
2953
+
2954
+ def __init__(
2955
+ self,
2956
+ entry_type_id: str,
2957
+ entry_type_configuration: dict | EntryType,
2958
+ update_mask: list[str] | FieldMask | None = None,
2959
+ validate_request: bool | None = False,
2960
+ *args,
2961
+ **kwargs,
2962
+ ) -> None:
2963
+ super().__init__(*args, **kwargs)
2964
+ self.entry_type_id = entry_type_id
2965
+ self.entry_type_configuration = entry_type_configuration
2966
+ self.update_mask = update_mask
2967
+ self.validate_request = validate_request
2968
+
2969
+ def execute(self, context: Context):
2970
+ DataplexCatalogEntryTypeLink.persist(
2971
+ context=context,
2972
+ task_instance=self,
2973
+ )
2974
+
2975
+ if self.validate_request:
2976
+ self.log.info("Validating an Update Dataplex Catalog EntryType request.")
2977
+ else:
2978
+ self.log.info(
2979
+ "Updating Dataplex Catalog EntryType %s.",
2980
+ self.entry_type_id,
2981
+ )
2982
+ try:
2983
+ operation = self.hook.update_entry_type(
2984
+ location=self.location,
2985
+ project_id=self.project_id,
2986
+ entry_type_id=self.entry_type_id,
2987
+ entry_type_configuration=self.entry_type_configuration,
2988
+ update_mask=self.update_mask,
2989
+ validate_only=self.validate_request,
2990
+ retry=self.retry,
2991
+ timeout=self.timeout,
2992
+ metadata=self.metadata,
2993
+ )
2994
+ entry_type = self.hook.wait_for_operation(timeout=self.timeout, operation=operation)
2995
+
2996
+ except NotFound as ex:
2997
+ self.log.info("Specified EntryType was not found.")
2998
+ raise AirflowException(ex)
2999
+ except Exception as exc:
3000
+ raise AirflowException(exc)
3001
+ else:
3002
+ result = EntryType.to_dict(entry_type) if not self.validate_request else None
3003
+
3004
+ if not self.validate_request:
3005
+ self.log.info("EntryType %s was successfully updated.", self.entry_type_id)
3006
+ return result
3007
+
3008
+
3009
+ class DataplexCatalogCreateAspectTypeOperator(DataplexCatalogBaseOperator):
3010
+ """
3011
+ Create an AspectType resource.
3012
+
3013
+ .. seealso::
3014
+ For more information on how to use this operator, take a look at the guide:
3015
+ :ref:`howto/operator:DataplexCatalogCreateAspectTypeOperator`
3016
+
3017
+ :param aspect_type_id: Required. AspectType identifier.
3018
+ :param aspect_type_configuration: Required. AspectType configuration.
3019
+ For more details please see API documentation:
3020
+ https://cloud.google.com/dataplex/docs/reference/rest/v1/projects.locations.aspectTypes#AspectType
3021
+ :param validate_request: Optional. If set, performs request validation, but does not actually
3022
+ execute the request.
3023
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
3024
+ :param location: Required. The ID of the Google Cloud region where the service is used.
3025
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
3026
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
3027
+ be retried.
3028
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
3029
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
3030
+ :param metadata: Optional. Additional metadata that is provided to the method.
3031
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
3032
+ credentials, or chained list of accounts required to get the access_token
3033
+ of the last account in the list, which will be impersonated in the request.
3034
+ If set as a string, the account must grant the originating account
3035
+ the Service Account Token Creator IAM role.
3036
+ If set as a sequence, the identities from the list must grant
3037
+ Service Account Token Creator IAM role to the directly preceding identity, with first
3038
+ account from the list granting this role to the originating account (templated).
3039
+ """
3040
+
3041
+ template_fields: Sequence[str] = tuple(
3042
+ {"aspect_type_id", "aspect_type_configuration"} | set(DataplexCatalogBaseOperator.template_fields)
3043
+ )
3044
+ operator_extra_links = (DataplexCatalogAspectTypeLink(),)
3045
+
3046
+ def __init__(
3047
+ self,
3048
+ aspect_type_id: str,
3049
+ aspect_type_configuration: AspectType | dict,
3050
+ validate_request: bool = False,
3051
+ *args,
3052
+ **kwargs,
3053
+ ) -> None:
3054
+ super().__init__(*args, **kwargs)
3055
+ self.aspect_type_id = aspect_type_id
3056
+ self.aspect_type_configuration = aspect_type_configuration
3057
+ self.validate_request = validate_request
3058
+
3059
+ def execute(self, context: Context):
3060
+ DataplexCatalogAspectTypeLink.persist(
3061
+ context=context,
3062
+ task_instance=self,
3063
+ )
3064
+
3065
+ if self.validate_request:
3066
+ self.log.info("Validating a Create Dataplex Catalog AspectType request.")
3067
+ else:
3068
+ self.log.info("Creating a Dataplex Catalog AspectType.")
3069
+
3070
+ try:
3071
+ operation = self.hook.create_aspect_type(
3072
+ aspect_type_id=self.aspect_type_id,
3073
+ aspect_type_configuration=self.aspect_type_configuration,
3074
+ location=self.location,
3075
+ project_id=self.project_id,
3076
+ validate_only=self.validate_request,
3077
+ retry=self.retry,
3078
+ timeout=self.timeout,
3079
+ metadata=self.metadata,
3080
+ )
3081
+ aspect_type = self.hook.wait_for_operation(timeout=self.timeout, operation=operation)
3082
+ except AlreadyExists:
3083
+ aspect_type = self.hook.get_aspect_type(
3084
+ aspect_type_id=self.aspect_type_id,
3085
+ location=self.location,
3086
+ project_id=self.project_id,
3087
+ )
3088
+ self.log.info(
3089
+ "Dataplex Catalog AspectType %s already exists.",
3090
+ self.aspect_type_id,
3091
+ )
3092
+ result = AspectType.to_dict(aspect_type)
3093
+ return result
3094
+ except Exception as ex:
3095
+ raise AirflowException(ex)
3096
+ else:
3097
+ result = AspectType.to_dict(aspect_type) if not self.validate_request else None
3098
+
3099
+ if not self.validate_request:
3100
+ self.log.info("Dataplex Catalog AspectType %s was successfully created.", self.aspect_type_id)
3101
+ return result
3102
+
3103
+
3104
+ class DataplexCatalogGetAspectTypeOperator(DataplexCatalogBaseOperator):
3105
+ """
3106
+ Get an AspectType resource.
3107
+
3108
+ .. seealso::
3109
+ For more information on how to use this operator, take a look at the guide:
3110
+ :ref:`howto/operator:DataplexCatalogGetAspectTypeOperator`
3111
+
3112
+ :param aspect_type_id: Required. AspectType identifier.
3113
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
3114
+ :param location: Required. The ID of the Google Cloud region where the service is used.
3115
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
3116
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
3117
+ be retried.
3118
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
3119
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
3120
+ :param metadata: Optional. Additional metadata that is provided to the method.
3121
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
3122
+ credentials, or chained list of accounts required to get the access_token
3123
+ of the last account in the list, which will be impersonated in the request.
3124
+ If set as a string, the account must grant the originating account
3125
+ the Service Account Token Creator IAM role.
3126
+ If set as a sequence, the identities from the list must grant
3127
+ Service Account Token Creator IAM role to the directly preceding identity, with first
3128
+ account from the list granting this role to the originating account (templated).
3129
+ """
3130
+
3131
+ template_fields: Sequence[str] = tuple(
3132
+ {"aspect_type_id"} | set(DataplexCatalogBaseOperator.template_fields)
3133
+ )
3134
+ operator_extra_links = (DataplexCatalogAspectTypeLink(),)
3135
+
3136
+ def __init__(
3137
+ self,
3138
+ aspect_type_id: str,
3139
+ *args,
3140
+ **kwargs,
3141
+ ) -> None:
3142
+ super().__init__(*args, **kwargs)
3143
+ self.aspect_type_id = aspect_type_id
3144
+
3145
+ def execute(self, context: Context):
3146
+ DataplexCatalogAspectTypeLink.persist(
3147
+ context=context,
3148
+ task_instance=self,
3149
+ )
3150
+ self.log.info(
3151
+ "Retrieving Dataplex Catalog AspectType %s.",
3152
+ self.aspect_type_id,
3153
+ )
3154
+ try:
3155
+ aspect_type = self.hook.get_aspect_type(
3156
+ aspect_type_id=self.aspect_type_id,
3157
+ location=self.location,
3158
+ project_id=self.project_id,
3159
+ retry=self.retry,
3160
+ timeout=self.timeout,
3161
+ metadata=self.metadata,
3162
+ )
3163
+ except NotFound:
3164
+ self.log.info(
3165
+ "Dataplex Catalog AspectType %s not found.",
3166
+ self.aspect_type_id,
3167
+ )
3168
+ raise AirflowException(NotFound)
3169
+ except Exception as ex:
3170
+ raise AirflowException(ex)
3171
+
3172
+ return AspectType.to_dict(aspect_type)
3173
+
3174
+
3175
+ class DataplexCatalogListAspectTypesOperator(DataplexCatalogBaseOperator):
3176
+ """
3177
+ List AspectType resources.
3178
+
3179
+ .. seealso::
3180
+ For more information on how to use this operator, take a look at the guide:
3181
+ :ref:`howto/operator:DataplexCatalogListAspectTypesOperator`
3182
+
3183
+ :param filter_by: Optional. Filter to apply on the list results.
3184
+ :param order_by: Optional. Fields to order the results by.
3185
+ :param page_size: Optional. Maximum number of AspectTypes to return on the page.
3186
+ :param page_token: Optional. Token to retrieve the next page of results.
3187
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
3188
+ :param location: Required. The ID of the Google Cloud region where the service is used.
3189
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
3190
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
3191
+ be retried.
3192
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
3193
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
3194
+ :param metadata: Optional. Additional metadata that is provided to the method.
3195
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
3196
+ credentials, or chained list of accounts required to get the access_token
3197
+ of the last account in the list, which will be impersonated in the request.
3198
+ If set as a string, the account must grant the originating account
3199
+ the Service Account Token Creator IAM role.
3200
+ If set as a sequence, the identities from the list must grant
3201
+ Service Account Token Creator IAM role to the directly preceding identity, with first
3202
+ account from the list granting this role to the originating account (templated).
3203
+ """
3204
+
3205
+ template_fields: Sequence[str] = tuple(DataplexCatalogBaseOperator.template_fields)
3206
+ operator_extra_links = (DataplexCatalogAspectTypesLink(),)
3207
+
3208
+ def __init__(
3209
+ self,
3210
+ page_size: int | None = None,
3211
+ page_token: str | None = None,
3212
+ filter_by: str | None = None,
3213
+ order_by: str | None = None,
3214
+ *args,
3215
+ **kwargs,
3216
+ ) -> None:
3217
+ super().__init__(*args, **kwargs)
3218
+ self.page_size = page_size
3219
+ self.page_token = page_token
3220
+ self.filter_by = filter_by
3221
+ self.order_by = order_by
3222
+
3223
+ def execute(self, context: Context):
3224
+ DataplexCatalogAspectTypesLink.persist(
3225
+ context=context,
3226
+ task_instance=self,
3227
+ )
3228
+ self.log.info(
3229
+ "Listing Dataplex Catalog AspectType from location %s.",
3230
+ self.location,
3231
+ )
3232
+ try:
3233
+ aspect_type_on_page = self.hook.list_aspect_types(
3234
+ location=self.location,
3235
+ project_id=self.project_id,
3236
+ page_size=self.page_size,
3237
+ page_token=self.page_token,
3238
+ filter_by=self.filter_by,
3239
+ order_by=self.order_by,
3240
+ retry=self.retry,
3241
+ timeout=self.timeout,
3242
+ metadata=self.metadata,
3243
+ )
3244
+ self.log.info("AspectType on page: %s", aspect_type_on_page)
3245
+ self.xcom_push(
3246
+ context=context,
3247
+ key="aspect_type_page",
3248
+ value=ListAspectTypesResponse.to_dict(aspect_type_on_page._response),
3249
+ )
3250
+ except Exception as ex:
3251
+ raise AirflowException(ex)
3252
+
3253
+ # Constructing list to return AspectTypes in readable format
3254
+ aspect_types_list = [
3255
+ MessageToDict(aspect_type._pb, preserving_proto_field_name=True)
3256
+ for aspect_type in next(iter(aspect_type_on_page.pages)).aspect_types
3257
+ ]
3258
+ return aspect_types_list
3259
+
3260
+
3261
+ class DataplexCatalogUpdateAspectTypeOperator(DataplexCatalogBaseOperator):
3262
+ """
3263
+ Update an AspectType resource.
3264
+
3265
+ .. seealso::
3266
+ For more information on how to use this operator, take a look at the guide:
3267
+ :ref:`howto/operator:DataplexCatalogUpdateAspectTypeOperator`
3268
+
3269
+ :param project_id: Required. The ID of the Google Cloud project that the task belongs to.
3270
+ :param location: Required. The ID of the Google Cloud region that the task belongs to.
3271
+ :param update_mask: Optional. Names of fields whose values to overwrite on an entry group.
3272
+ If this parameter is absent or empty, all modifiable fields are overwritten. If such
3273
+ fields are non-required and omitted in the request body, their values are emptied.
3274
+ :param aspect_type_id: Required. ID of the AspectType to update.
3275
+ :param aspect_type_configuration: Required. The updated configuration body of the AspectType.
3276
+ For more details please see API documentation:
3277
+ https://cloud.google.com/dataplex/docs/reference/rest/v1/projects.locations.aspectTypes#AspectType
3278
+ :param validate_only: Optional. The service validates the request without performing any mutations.
3279
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests
3280
+ will not be retried.
3281
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
3282
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
3283
+ :param metadata: Optional. Additional metadata that is provided to the method.
3284
+ :param gcp_conn_id: Optional. The connection ID to use when fetching connection info.
3285
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
3286
+ credentials, or chained list of accounts required to get the access_token
3287
+ of the last account in the list, which will be impersonated in the request.
3288
+ If set as a string, the account must grant the originating account
3289
+ the Service Account Token Creator IAM role.
3290
+ If set as a sequence, the identities from the list must grant
3291
+ Service Account Token Creator IAM role to the directly preceding identity, with first
3292
+ account from the list granting this role to the originating account (templated).
3293
+ """
3294
+
3295
+ template_fields: Sequence[str] = tuple(
3296
+ {"aspect_type_id", "aspect_type_configuration", "update_mask"}
3297
+ | set(DataplexCatalogBaseOperator.template_fields)
3298
+ )
3299
+ operator_extra_links = (DataplexCatalogAspectTypeLink(),)
3300
+
3301
+ def __init__(
3302
+ self,
3303
+ aspect_type_id: str,
3304
+ aspect_type_configuration: dict | AspectType,
3305
+ update_mask: list[str] | FieldMask | None = None,
3306
+ validate_request: bool | None = False,
3307
+ *args,
3308
+ **kwargs,
3309
+ ) -> None:
3310
+ super().__init__(*args, **kwargs)
3311
+ self.aspect_type_id = aspect_type_id
3312
+ self.aspect_type_configuration = aspect_type_configuration
3313
+ self.update_mask = update_mask
3314
+ self.validate_request = validate_request
3315
+
3316
+ def execute(self, context: Context):
3317
+ DataplexCatalogAspectTypeLink.persist(
3318
+ context=context,
3319
+ task_instance=self,
3320
+ )
3321
+
3322
+ if self.validate_request:
3323
+ self.log.info("Validating an Update Dataplex Catalog AspectType request.")
3324
+ else:
3325
+ self.log.info(
3326
+ "Updating Dataplex Catalog AspectType %s.",
3327
+ self.aspect_type_id,
3328
+ )
3329
+ try:
3330
+ operation = self.hook.update_aspect_type(
3331
+ location=self.location,
3332
+ project_id=self.project_id,
3333
+ aspect_type_id=self.aspect_type_id,
3334
+ aspect_type_configuration=self.aspect_type_configuration,
3335
+ update_mask=self.update_mask,
3336
+ validate_only=self.validate_request,
3337
+ retry=self.retry,
3338
+ timeout=self.timeout,
3339
+ metadata=self.metadata,
3340
+ )
3341
+ aspect_type = self.hook.wait_for_operation(timeout=self.timeout, operation=operation)
3342
+
3343
+ except NotFound as ex:
3344
+ self.log.info("Specified AspectType was not found.")
3345
+ raise AirflowException(ex)
3346
+ except Exception as exc:
3347
+ raise AirflowException(exc)
3348
+ else:
3349
+ result = AspectType.to_dict(aspect_type) if not self.validate_request else None
3350
+
3351
+ if not self.validate_request:
3352
+ self.log.info("AspectType %s was successfully updated.", self.aspect_type_id)
3353
+ return result
3354
+
3355
+
3356
+ class DataplexCatalogDeleteAspectTypeOperator(DataplexCatalogBaseOperator):
3357
+ """
3358
+ Delete an AspectType resource.
3359
+
3360
+ .. seealso::
3361
+ For more information on how to use this operator, take a look at the guide:
3362
+ :ref:`howto/operator:DataplexCatalogDeleteAspectTypeOperator`
3363
+
3364
+ :param aspect_type_id: Required. AspectType identifier.
3365
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
3366
+ :param location: Required. The ID of the Google Cloud region where the service is used.
3367
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
3368
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
3369
+ be retried.
3370
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
3371
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
3372
+ :param metadata: Optional. Additional metadata that is provided to the method.
3373
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
3374
+ credentials, or chained list of accounts required to get the access_token
3375
+ of the last account in the list, which will be impersonated in the request.
3376
+ If set as a string, the account must grant the originating account
3377
+ the Service Account Token Creator IAM role.
3378
+ If set as a sequence, the identities from the list must grant
3379
+ Service Account Token Creator IAM role to the directly preceding identity, with first
3380
+ account from the list granting this role to the originating account (templated).
3381
+ """
3382
+
3383
+ template_fields: Sequence[str] = tuple(
3384
+ {"aspect_type_id"} | set(DataplexCatalogBaseOperator.template_fields)
3385
+ )
3386
+
3387
+ def __init__(
3388
+ self,
3389
+ aspect_type_id: str,
3390
+ *args,
3391
+ **kwargs,
3392
+ ) -> None:
3393
+ super().__init__(*args, **kwargs)
3394
+ self.aspect_type_id = aspect_type_id
3395
+
3396
+ def execute(self, context: Context):
3397
+ self.log.info(
3398
+ "Deleting Dataplex Catalog AspectType %s.",
3399
+ self.aspect_type_id,
3400
+ )
3401
+ try:
3402
+ operation = self.hook.delete_aspect_type(
3403
+ aspect_type_id=self.aspect_type_id,
3404
+ location=self.location,
3405
+ project_id=self.project_id,
3406
+ retry=self.retry,
3407
+ timeout=self.timeout,
3408
+ metadata=self.metadata,
3409
+ )
3410
+ self.hook.wait_for_operation(timeout=self.timeout, operation=operation)
3411
+
3412
+ except NotFound:
3413
+ self.log.info(
3414
+ "Dataplex Catalog AspectType %s not found.",
3415
+ self.aspect_type_id,
3416
+ )
3417
+ raise AirflowException(NotFound)
3418
+ except Exception as ex:
3419
+ raise AirflowException(ex)
3420
+ return None
3421
+
3422
+
3423
+ class DataplexCatalogCreateEntryOperator(DataplexCatalogBaseOperator):
3424
+ """
3425
+ Create an Entry resource.
3426
+
3427
+ .. seealso::
3428
+ For more information on how to use this operator, take a look at the guide:
3429
+ :ref:`howto/operator:DataplexCatalogCreateEntryOperator`
3430
+
3431
+ :param entry_id: Required. Entry identifier. It has to be unique within an Entry Group.
3432
+ Entries corresponding to Google Cloud resources use an Entry ID format based on `full resource
3433
+ names <https://cloud.google.com/apis/design/resource_names#full_resource_name>`__.
3434
+ The format is a full resource name of the resource without the prefix double slashes in the API
3435
+ service name part of the full resource name. This allows retrieval of entries using their associated
3436
+ resource name.
3437
+ For example, if the full resource name of a resource is
3438
+ ``//library.googleapis.com/shelves/shelf1/books/book2``, then the suggested entry_id is
3439
+ ``library.googleapis.com/shelves/shelf1/books/book2``.
3440
+ It is also suggested to follow the same convention for entries corresponding to resources from
3441
+ providers or systems other than Google Cloud.
3442
+ The maximum size of the field is 4000 characters.
3443
+ :param entry_group_id: Required. EntryGroup resource name to which created Entry will belong to.
3444
+ :param entry_configuration: Required. Entry configuration.
3445
+ For more details please see API documentation:
3446
+ https://cloud.google.com/dataplex/docs/reference/rest/v1/projects.locations.entryGroups.entries#Entry
3447
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
3448
+ :param location: Required. The ID of the Google Cloud region where the service is used.
3449
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
3450
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
3451
+ be retried.
3452
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
3453
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
3454
+ :param metadata: Optional. Additional metadata that is provided to the method.
3455
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
3456
+ credentials, or chained list of accounts required to get the access_token
3457
+ of the last account in the list, which will be impersonated in the request.
3458
+ If set as a string, the account must grant the originating account
3459
+ the Service Account Token Creator IAM role.
3460
+ If set as a sequence, the identities from the list must grant
3461
+ Service Account Token Creator IAM role to the directly preceding identity, with first
3462
+ account from the list granting this role to the originating account (templated).
3463
+ """
3464
+
3465
+ template_fields: Sequence[str] = tuple(
3466
+ {"entry_id", "entry_group_id", "entry_configuration"}
3467
+ | set(DataplexCatalogBaseOperator.template_fields)
3468
+ )
3469
+ operator_extra_links = (DataplexCatalogEntryLink(),)
3470
+
3471
+ def __init__(
3472
+ self,
3473
+ entry_id: str,
3474
+ entry_group_id: str,
3475
+ entry_configuration: Entry | dict,
3476
+ *args,
3477
+ **kwargs,
3478
+ ) -> None:
3479
+ super().__init__(*args, **kwargs)
3480
+ self.entry_id = entry_id
3481
+ self.entry_group_id = entry_group_id
3482
+ self.entry_configuration = entry_configuration
3483
+
3484
+ def _validate_fields(self, entry_configuration):
3485
+ required_fields = ["name", "entry_type"]
3486
+
3487
+ missing_fields = [field for field in required_fields if not entry_configuration.get(field)]
3488
+
3489
+ if missing_fields:
3490
+ raise AirflowException(
3491
+ f"Missing required fields in Entry configuration: {', '.join(missing_fields)}. "
3492
+ )
3493
+
3494
+ def execute(self, context: Context):
3495
+ DataplexCatalogEntryLink.persist(
3496
+ context=context,
3497
+ task_instance=self,
3498
+ )
3499
+
3500
+ self._validate_fields(self.entry_configuration)
3501
+ try:
3502
+ entry = self.hook.create_entry(
3503
+ entry_id=self.entry_id,
3504
+ entry_group_id=self.entry_group_id,
3505
+ entry_configuration=self.entry_configuration,
3506
+ location=self.location,
3507
+ project_id=self.project_id,
3508
+ retry=self.retry,
3509
+ timeout=self.timeout,
3510
+ metadata=self.metadata,
3511
+ )
3512
+ except AlreadyExists:
3513
+ entry = self.hook.get_entry(
3514
+ entry_id=self.entry_id,
3515
+ entry_group_id=self.entry_group_id,
3516
+ location=self.location,
3517
+ project_id=self.project_id,
3518
+ )
3519
+ self.log.info(
3520
+ "Dataplex Catalog Entry %s already exists.",
3521
+ self.entry_id,
3522
+ )
3523
+ result = Entry.to_dict(entry)
3524
+ return result
3525
+ except Exception as ex:
3526
+ raise AirflowException(ex)
3527
+ else:
3528
+ result = Entry.to_dict(entry)
3529
+
3530
+ self.log.info("Dataplex Catalog Entry %s was successfully created.", self.entry_id)
3531
+ return result
3532
+
3533
+
3534
+ class DataplexCatalogGetEntryOperator(DataplexCatalogBaseOperator):
3535
+ """
3536
+ Get an Entry resource.
3537
+
3538
+ .. seealso::
3539
+ For more information on how to use this operator, take a look at the guide:
3540
+ :ref:`howto/operator:DataplexCatalogGetEntryOperator`
3541
+
3542
+ :param entry_id: Required. Entry identifier. It has to be unique within an Entry Group.
3543
+ Entries corresponding to Google Cloud resources use an Entry ID format based on `full resource
3544
+ names <https://cloud.google.com/apis/design/resource_names#full_resource_name>`__.
3545
+ The format is a full resource name of the resource without the prefix double slashes in the API
3546
+ service name part of the full resource name. This allows retrieval of entries using their associated
3547
+ resource name.
3548
+ For example, if the full resource name of a resource is
3549
+ ``//library.googleapis.com/shelves/shelf1/books/book2``, then the suggested entry_id is
3550
+ ``library.googleapis.com/shelves/shelf1/books/book2``.
3551
+ It is also suggested to follow the same convention for entries corresponding to resources from
3552
+ providers or systems other than Google Cloud.
3553
+ The maximum size of the field is 4000 characters.
3554
+ :param entry_group_id: Required. EntryGroup resource name to which created Entry will belong to.
3555
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
3556
+ :param location: Required. The ID of the Google Cloud region where the service is used.
3557
+ :param view: Optional. View to control which parts of an Entry the service should return.
3558
+ :param aspect_types: Optional. Limits the aspects returned to the provided aspect types. It only works
3559
+ for CUSTOM view.
3560
+ :param paths: Optional. Limits the aspects returned to those associated with the provided paths within
3561
+ the Entry. It only works for CUSTOM view.
3562
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
3563
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
3564
+ be retried.
3565
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
3566
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
3567
+ :param metadata: Optional. Additional metadata that is provided to the method.
3568
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
3569
+ credentials, or chained list of accounts required to get the access_token
3570
+ of the last account in the list, which will be impersonated in the request.
3571
+ If set as a string, the account must grant the originating account
3572
+ the Service Account Token Creator IAM role.
3573
+ If set as a sequence, the identities from the list must grant
3574
+ Service Account Token Creator IAM role to the directly preceding identity, with first
3575
+ account from the list granting this role to the originating account (templated).
3576
+ """
3577
+
3578
+ template_fields: Sequence[str] = tuple(
3579
+ {"entry_id", "entry_group_id"} | set(DataplexCatalogBaseOperator.template_fields)
3580
+ )
3581
+ operator_extra_links = (DataplexCatalogEntryLink(),)
3582
+
3583
+ def __init__(
3584
+ self,
3585
+ entry_id: str,
3586
+ entry_group_id: str,
3587
+ view: EntryView | str | None = None,
3588
+ aspect_types: MutableSequence[str] | None = None,
3589
+ paths: MutableSequence[str] | None = None,
3590
+ *args,
3591
+ **kwargs,
3592
+ ) -> None:
3593
+ super().__init__(*args, **kwargs)
3594
+ self.entry_id = entry_id
3595
+ self.entry_group_id = entry_group_id
3596
+ self.view = view
3597
+ self.aspect_types = aspect_types
3598
+ self.paths = paths
3599
+
3600
+ def execute(self, context: Context):
3601
+ DataplexCatalogEntryLink.persist(
3602
+ context=context,
3603
+ task_instance=self,
3604
+ )
3605
+ self.log.info(
3606
+ "Retrieving Dataplex Catalog Entry %s.",
3607
+ self.entry_id,
3608
+ )
3609
+ try:
3610
+ entry = self.hook.get_entry(
3611
+ entry_id=self.entry_id,
3612
+ entry_group_id=self.entry_group_id,
3613
+ view=self.view,
3614
+ aspect_types=self.aspect_types,
3615
+ paths=self.paths,
3616
+ location=self.location,
3617
+ project_id=self.project_id,
3618
+ retry=self.retry,
3619
+ timeout=self.timeout,
3620
+ metadata=self.metadata,
3621
+ )
3622
+ except NotFound:
3623
+ self.log.info(
3624
+ "Dataplex Catalog Entry %s not found.",
3625
+ self.entry_id,
3626
+ )
3627
+ raise AirflowException(NotFound)
3628
+ except Exception as ex:
3629
+ raise AirflowException(ex)
3630
+
3631
+ return Entry.to_dict(entry)
3632
+
3633
+
3634
+ class DataplexCatalogListEntriesOperator(DataplexCatalogBaseOperator):
3635
+ r"""
3636
+ List Entry resources.
3637
+
3638
+ .. seealso::
3639
+ For more information on how to use this operator, take a look at the guide:
3640
+ :ref:`howto/operator:DataplexCatalogListEntriesOperator`
3641
+
3642
+ :param entry_group_id: Required. EntryGroup resource name to which created Entry belongs to.
3643
+ :param filter_by: Optional. A filter on the entries to return. Filters are case-sensitive.
3644
+ You can filter the request by the following fields:
3645
+
3646
+ - entry_type
3647
+ - entry_source.display_name
3648
+
3649
+ The comparison operators are =, !=, <, >, <=, >=. The service compares strings according to
3650
+ lexical order.
3651
+ You can use the logical operators AND, OR, NOT in the filter. You can use Wildcard "*", but for
3652
+ entry_type you need to provide the full project id or number.
3653
+ Example filter expressions:
3654
+
3655
+ - "entry_source.display_name=AnExampleDisplayName"
3656
+ - "entry_type=projects/example-project/locations/global/entryTypes/example-entry_type"
3657
+ - "entry_type=projects/example-project/locations/us/entryTypes/a\*
3658
+ OR entry_type=projects/another-project/locations/\*"
3659
+ - "NOT entry_source.display_name=AnotherExampleDisplayName".
3660
+
3661
+ :param page_size: Optional. Number of items to return per page. If there are remaining results,
3662
+ the service returns a next_page_token. If unspecified, the service returns at most 10 Entries.
3663
+ The maximum value is 100; values above 100 will be coerced to 100.
3664
+ :param page_token: Optional. Page token received from a previous ``ListEntries`` call. Provide
3665
+ this to retrieve the subsequent page.
3666
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
3667
+ :param location: Required. The ID of the Google Cloud region where the service is used.
3668
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
3669
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
3670
+ be retried.
3671
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
3672
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
3673
+ :param metadata: Optional. Additional metadata that is provided to the method.
3674
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
3675
+ credentials, or chained list of accounts required to get the access_token
3676
+ of the last account in the list, which will be impersonated in the request.
3677
+ If set as a string, the account must grant the originating account
3678
+ the Service Account Token Creator IAM role.
3679
+ If set as a sequence, the identities from the list must grant
3680
+ Service Account Token Creator IAM role to the directly preceding identity, with first
3681
+ account from the list granting this role to the originating account (templated).
3682
+ """
3683
+
3684
+ template_fields: Sequence[str] = tuple(DataplexCatalogBaseOperator.template_fields)
3685
+ operator_extra_links = (DataplexCatalogEntryGroupLink(),)
3686
+
3687
+ def __init__(
3688
+ self,
3689
+ entry_group_id: str,
3690
+ page_size: int | None = None,
3691
+ page_token: str | None = None,
3692
+ filter_by: str | None = None,
3693
+ *args,
3694
+ **kwargs,
3695
+ ) -> None:
3696
+ super().__init__(*args, **kwargs)
3697
+ self.entry_group_id = entry_group_id
3698
+ self.page_size = page_size
3699
+ self.page_token = page_token
3700
+ self.filter_by = filter_by
3701
+
3702
+ def execute(self, context: Context):
3703
+ DataplexCatalogEntryGroupLink.persist(
3704
+ context=context,
3705
+ task_instance=self,
3706
+ )
3707
+ self.log.info(
3708
+ "Listing Dataplex Catalog Entry from location %s.",
3709
+ self.location,
3710
+ )
3711
+ try:
3712
+ entries_on_page = self.hook.list_entries(
3713
+ entry_group_id=self.entry_group_id,
3714
+ location=self.location,
3715
+ project_id=self.project_id,
3716
+ page_size=self.page_size,
3717
+ page_token=self.page_token,
3718
+ filter_by=self.filter_by,
3719
+ retry=self.retry,
3720
+ timeout=self.timeout,
3721
+ metadata=self.metadata,
3722
+ )
3723
+ self.log.info("Entries on page: %s", entries_on_page)
3724
+ self.xcom_push(
3725
+ context=context,
3726
+ key="entry_page",
3727
+ value=ListEntriesResponse.to_dict(entries_on_page._response),
3728
+ )
3729
+ except Exception as ex:
3730
+ raise AirflowException(ex)
3731
+
3732
+ # Constructing list to return Entries in readable format
3733
+ entries_list = [
3734
+ MessageToDict(entry._pb, preserving_proto_field_name=True)
3735
+ for entry in next(iter(entries_on_page.pages)).entries
3736
+ ]
3737
+ return entries_list
3738
+
3739
+
3740
+ class DataplexCatalogSearchEntriesOperator(DataplexCatalogBaseOperator):
3741
+ """
3742
+ Search for Entries matching the given query and scope.
3743
+
3744
+ .. seealso::
3745
+ For more information on how to use this operator, take a look at the guide:
3746
+ :ref:`howto/operator:DataplexCatalogSearchEntriesOperator`
3747
+
3748
+ :param query: Required. The query against which entries in scope should be matched. The query
3749
+ syntax is defined in `Search syntax for Dataplex Catalog
3750
+ <https://cloud.google.com/dataplex/docs/search-syntax>`__.
3751
+ :param order_by: Optional. Specifies the ordering of results. Supported values are:
3752
+
3753
+ - ``relevance`` (default)
3754
+ - ``last_modified_timestamp``
3755
+ - ``last_modified_timestamp asc``
3756
+
3757
+ :param scope: Optional. The scope under which the search should be operating. It must either be
3758
+ ``organizations/<org_id>`` or ``projects/<project_ref>``. If it is unspecified, it
3759
+ defaults to the organization where the project provided in ``name`` is located.
3760
+ :param page_size: Optional. Number of items to return per page. If there are remaining results,
3761
+ the service returns a next_page_token. If unspecified, the service returns at most 10 Entries.
3762
+ The maximum value is 100; values above 100 will be coerced to 100.
3763
+ :param page_token: Optional. Page token received from a previous ``ListEntries`` call. Provide
3764
+ this to retrieve the subsequent page.
3765
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
3766
+ :param location: Required. The ID of the Google Cloud region where the service is used.
3767
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
3768
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
3769
+ be retried.
3770
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
3771
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
3772
+ :param metadata: Optional. Additional metadata that is provided to the method.
3773
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
3774
+ credentials, or chained list of accounts required to get the access_token
3775
+ of the last account in the list, which will be impersonated in the request.
3776
+ If set as a string, the account must grant the originating account
3777
+ the Service Account Token Creator IAM role.
3778
+ If set as a sequence, the identities from the list must grant
3779
+ Service Account Token Creator IAM role to the directly preceding identity, with first
3780
+ account from the list granting this role to the originating account (templated).
3781
+ """
3782
+
3783
+ template_fields: Sequence[str] = tuple({"query"} | set(DataplexCatalogBaseOperator.template_fields))
3784
+
3785
+ def __init__(
3786
+ self,
3787
+ query: str,
3788
+ order_by: str | None = None,
3789
+ scope: str | None = None,
3790
+ page_size: int | None = None,
3791
+ page_token: str | None = None,
3792
+ *args,
3793
+ **kwargs,
3794
+ ) -> None:
3795
+ super().__init__(*args, **kwargs)
3796
+ self.query = query
3797
+ self.page_size = page_size
3798
+ self.page_token = page_token
3799
+ self.order_by = order_by
3800
+ self.scope = scope
3801
+
3802
+ def execute(self, context: Context):
3803
+ self.log.info(
3804
+ "Listing Entries from location %s matching the given query %s and scope %s.",
3805
+ self.location,
3806
+ self.query,
3807
+ self.scope,
3808
+ )
3809
+ try:
3810
+ entries_on_page = self.hook.search_entries(
3811
+ query=self.query,
3812
+ location=self.location,
3813
+ project_id=self.project_id,
3814
+ page_size=self.page_size,
3815
+ page_token=self.page_token,
3816
+ order_by=self.order_by,
3817
+ retry=self.retry,
3818
+ timeout=self.timeout,
3819
+ metadata=self.metadata,
3820
+ )
3821
+ self.log.info("Entries on page: %s", entries_on_page)
3822
+ self.xcom_push(
3823
+ context=context,
3824
+ key="entry_page",
3825
+ value=SearchEntriesResponse.to_dict(entries_on_page._response),
3826
+ )
3827
+ except Exception as ex:
3828
+ raise AirflowException(ex)
3829
+
3830
+ # Constructing list to return Entries in readable format
3831
+ entries_list = [
3832
+ MessageToDict(entry._pb, preserving_proto_field_name=True)
3833
+ for entry in next(iter(entries_on_page.pages)).results
3834
+ ]
3835
+ return entries_list
3836
+
3837
+
3838
+ class DataplexCatalogLookupEntryOperator(DataplexCatalogBaseOperator):
3839
+ """
3840
+ Look up a single Entry by name using the permission on the source system.
3841
+
3842
+ .. seealso::
3843
+ For more information on how to use this operator, take a look at the guide:
3844
+ :ref:`howto/operator:DataplexCatalogLookupEntryOperator`
3845
+
3846
+ :param entry_id: Required. Entry identifier. It has to be unique within an Entry Group.
3847
+ Entries corresponding to Google Cloud resources use an Entry ID format based on `full resource
3848
+ names <https://cloud.google.com/apis/design/resource_names#full_resource_name>`__.
3849
+ The format is a full resource name of the resource without the prefix double slashes in the API
3850
+ service name part of the full resource name. This allows retrieval of entries using their associated
3851
+ resource name.
3852
+ For example, if the full resource name of a resource is
3853
+ ``//library.googleapis.com/shelves/shelf1/books/book2``, then the suggested entry_id is
3854
+ ``library.googleapis.com/shelves/shelf1/books/book2``.
3855
+ It is also suggested to follow the same convention for entries corresponding to resources from
3856
+ providers or systems other than Google Cloud.
3857
+ The maximum size of the field is 4000 characters.
3858
+ :param entry_group_id: Required. EntryGroup resource name to which created Entry will belong to.
3859
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
3860
+ :param location: Required. The ID of the Google Cloud region where the service is used.
3861
+ :param view: Optional. View to control which parts of an Entry the service should return.
3862
+ :param aspect_types: Optional. Limits the aspects returned to the provided aspect types. It only works
3863
+ for CUSTOM view.
3864
+ :param paths: Optional. Limits the aspects returned to those associated with the provided paths within
3865
+ the Entry. It only works for CUSTOM view.
3866
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
3867
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
3868
+ be retried.
3869
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
3870
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
3871
+ :param metadata: Optional. Additional metadata that is provided to the method.
3872
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
3873
+ credentials, or chained list of accounts required to get the access_token
3874
+ of the last account in the list, which will be impersonated in the request.
3875
+ If set as a string, the account must grant the originating account
3876
+ the Service Account Token Creator IAM role.
3877
+ If set as a sequence, the identities from the list must grant
3878
+ Service Account Token Creator IAM role to the directly preceding identity, with first
3879
+ account from the list granting this role to the originating account (templated).
3880
+ """
3881
+
3882
+ template_fields: Sequence[str] = tuple(
3883
+ {"entry_id", "entry_group_id"} | set(DataplexCatalogBaseOperator.template_fields)
3884
+ )
3885
+ operator_extra_links = (DataplexCatalogEntryLink(),)
3886
+
3887
+ def __init__(
3888
+ self,
3889
+ entry_id: str,
3890
+ entry_group_id: str,
3891
+ view: EntryView | str | None = None,
3892
+ aspect_types: MutableSequence[str] | None = None,
3893
+ paths: MutableSequence[str] | None = None,
3894
+ *args,
3895
+ **kwargs,
3896
+ ) -> None:
3897
+ super().__init__(*args, **kwargs)
3898
+ self.entry_id = entry_id
3899
+ self.entry_group_id = entry_group_id
3900
+ self.view = view
3901
+ self.aspect_types = aspect_types
3902
+ self.paths = paths
3903
+
3904
+ def execute(self, context: Context):
3905
+ DataplexCatalogEntryLink.persist(
3906
+ context=context,
3907
+ task_instance=self,
3908
+ )
3909
+ self.log.info(
3910
+ "Looking for Dataplex Catalog Entry %s.",
3911
+ self.entry_id,
3912
+ )
3913
+ try:
3914
+ entry = self.hook.lookup_entry(
3915
+ entry_id=self.entry_id,
3916
+ entry_group_id=self.entry_group_id,
3917
+ view=self.view,
3918
+ aspect_types=self.aspect_types,
3919
+ paths=self.paths,
3920
+ location=self.location,
3921
+ project_id=self.project_id,
3922
+ retry=self.retry,
3923
+ timeout=self.timeout,
3924
+ metadata=self.metadata,
3925
+ )
3926
+ except NotFound:
3927
+ self.log.info(
3928
+ "Dataplex Catalog Entry %s not found.",
3929
+ self.entry_id,
3930
+ )
3931
+ raise AirflowException(NotFound)
3932
+ except Exception as ex:
3933
+ raise AirflowException(ex)
3934
+
3935
+ return Entry.to_dict(entry)
3936
+
3937
+
3938
+ class DataplexCatalogUpdateEntryOperator(DataplexCatalogBaseOperator):
3939
+ """
3940
+ Update an Entry resource.
3941
+
3942
+ .. seealso::
3943
+ For more information on how to use this operator, take a look at the guide:
3944
+ :ref:`howto/operator:DataplexCatalogUpdateEntryOperator`
3945
+
3946
+ :param project_id: Required. The ID of the Google Cloud project that the task belongs to.
3947
+ :param location: Required. The ID of the Google Cloud region that the task belongs to.
3948
+ :param entry_id: Required. Entry identifier. It has to be unique within an Entry Group.
3949
+ Entries corresponding to Google Cloud resources use an Entry ID format based on `full resource
3950
+ names <https://cloud.google.com/apis/design/resource_names#full_resource_name>`__.
3951
+ The format is a full resource name of the resource without the prefix double slashes in the API
3952
+ service name part of the full resource name. This allows retrieval of entries using their
3953
+ associated resource name.
3954
+ For example, if the full resource name of a resource is
3955
+ ``//library.googleapis.com/shelves/shelf1/books/book2``, then the suggested entry_id is
3956
+ ``library.googleapis.com/shelves/shelf1/books/book2``.
3957
+ It is also suggested to follow the same convention for entries corresponding to resources from
3958
+ providers or systems other than Google Cloud.
3959
+ The maximum size of the field is 4000 characters.
3960
+ :param entry_group_id: Required. EntryGroup resource name to which created Entry belongs to.
3961
+ :param entry_configuration: Required. The updated configuration body of the Entry.
3962
+ :param allow_missing: Optional. If set to true and entry doesn't exist, the service will create it.
3963
+ :param delete_missing_aspects: Optional. If set to true and the aspect_keys specify aspect
3964
+ ranges, the service deletes any existing aspects from that range that weren't provided
3965
+ in the request.
3966
+ :param aspect_keys: Optional. The map keys of the Aspects which the service should modify.
3967
+ It supports the following syntax:
3968
+
3969
+ - ``<aspect_type_reference>`` - matches an aspect of the given type and empty path.
3970
+ - ``<aspect_type_reference>@path`` - matches an aspect of the given type and specified path.
3971
+ For example, to attach an aspect to a field that is specified by the ``schema``
3972
+ aspect, the path should have the format ``Schema.<field_name>``.
3973
+ - ``<aspect_type_reference>@*`` - matches aspects of the given type for all paths.
3974
+ - ``*@path`` - matches aspects of all types on the given path.
3975
+
3976
+ The service will not remove existing aspects matching the syntax unless ``delete_missing_aspects``
3977
+ is set to true.
3978
+ If this field is left empty, the service treats it as specifying exactly those Aspects present
3979
+ in the request.
3980
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests
3981
+ will not be retried.
3982
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
3983
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
3984
+ :param metadata: Optional. Additional metadata that is provided to the method.
3985
+ :param gcp_conn_id: Optional. The connection ID to use when fetching connection info.
3986
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
3987
+ credentials, or chained list of accounts required to get the access_token
3988
+ of the last account in the list, which will be impersonated in the request.
3989
+ If set as a string, the account must grant the originating account
3990
+ the Service Account Token Creator IAM role.
3991
+ If set as a sequence, the identities from the list must grant
3992
+ Service Account Token Creator IAM role to the directly preceding identity, with first
3993
+ account from the list granting this role to the originating account (templated).
3994
+ """
3995
+
3996
+ template_fields: Sequence[str] = tuple(
3997
+ {"entry_id", "entry_group_id", "entry_configuration"}
3998
+ | set(DataplexCatalogBaseOperator.template_fields)
3999
+ )
4000
+ operator_extra_links = (DataplexCatalogEntryLink(),)
4001
+
4002
+ def __init__(
4003
+ self,
4004
+ entry_id: str,
4005
+ entry_group_id: str,
4006
+ entry_configuration: dict | Entry,
4007
+ allow_missing: bool | None = False,
4008
+ delete_missing_aspects: bool | None = False,
4009
+ aspect_keys: MutableSequence[str] | None = None,
4010
+ update_mask: list[str] | FieldMask | None = None,
4011
+ *args,
4012
+ **kwargs,
4013
+ ) -> None:
4014
+ super().__init__(*args, **kwargs)
4015
+ self.entry_id = entry_id
4016
+ self.entry_group_id = entry_group_id
4017
+ self.entry_configuration = entry_configuration
4018
+ self.update_mask = update_mask
4019
+ self.allow_missing = allow_missing
4020
+ self.delete_missing_aspects = delete_missing_aspects
4021
+ self.aspect_keys = aspect_keys
4022
+
4023
+ def execute(self, context: Context):
4024
+ DataplexCatalogEntryLink.persist(
4025
+ context=context,
4026
+ task_instance=self,
4027
+ )
4028
+
4029
+ try:
4030
+ entry = self.hook.update_entry(
4031
+ location=self.location,
4032
+ project_id=self.project_id,
4033
+ entry_id=self.entry_id,
4034
+ entry_group_id=self.entry_group_id,
4035
+ entry_configuration=self.entry_configuration,
4036
+ update_mask=self.update_mask,
4037
+ allow_missing=self.allow_missing,
4038
+ delete_missing_aspects=self.delete_missing_aspects,
4039
+ aspect_keys=self.aspect_keys,
4040
+ retry=self.retry,
4041
+ timeout=self.timeout,
4042
+ metadata=self.metadata,
4043
+ )
4044
+
4045
+ except NotFound as ex:
4046
+ self.log.info("Specified Entry was not found.")
4047
+ raise AirflowException(ex)
4048
+ except Exception as exc:
4049
+ raise AirflowException(exc)
4050
+ else:
4051
+ result = Entry.to_dict(entry)
4052
+ self.log.info("Entry %s was successfully updated.", self.entry_id)
4053
+ return result
4054
+
4055
+
4056
+ class DataplexCatalogDeleteEntryOperator(DataplexCatalogBaseOperator):
4057
+ """
4058
+ Delete an Entry resource.
4059
+
4060
+ .. seealso::
4061
+ For more information on how to use this operator, take a look at the guide:
4062
+ :ref:`howto/operator:DataplexCatalogDeleteEntryOperator`
4063
+
4064
+ :param entry_id: Required. Entry identifier. It has to be unique within an Entry Group.
4065
+ Entries corresponding to Google Cloud resources use an Entry ID format based on `full resource
4066
+ names <https://cloud.google.com/apis/design/resource_names#full_resource_name>`__.
4067
+ The format is a full resource name of the resource without the prefix double slashes in the API
4068
+ service name part of the full resource name. This allows retrieval of entries using their
4069
+ associated resource name.
4070
+ For example, if the full resource name of a resource is
4071
+ ``//library.googleapis.com/shelves/shelf1/books/book2``, then the suggested entry_id is
4072
+ ``library.googleapis.com/shelves/shelf1/books/book2``.
4073
+ It is also suggested to follow the same convention for entries corresponding to resources from
4074
+ providers or systems other than Google Cloud.
4075
+ The maximum size of the field is 4000 characters.
4076
+ :param entry_group_id: Required. EntryGroup resource name to which created Entry will belong to.
4077
+ :param project_id: Required. The ID of the Google Cloud project where the service is used.
4078
+ :param location: Required. The ID of the Google Cloud region where the service is used.
4079
+ :param gcp_conn_id: Optional. The connection ID to use to connect to Google Cloud.
4080
+ :param retry: Optional. A retry object used to retry requests. If `None` is specified, requests will not
4081
+ be retried.
4082
+ :param timeout: Optional. The amount of time, in seconds, to wait for the request to complete.
4083
+ Note that if `retry` is specified, the timeout applies to each individual attempt.
4084
+ :param metadata: Optional. Additional metadata that is provided to the method.
4085
+ :param impersonation_chain: Optional. Service account to impersonate using short-term
4086
+ credentials, or chained list of accounts required to get the access_token
4087
+ of the last account in the list, which will be impersonated in the request.
4088
+ If set as a string, the account must grant the originating account
4089
+ the Service Account Token Creator IAM role.
4090
+ If set as a sequence, the identities from the list must grant
4091
+ Service Account Token Creator IAM role to the directly preceding identity, with first
4092
+ account from the list granting this role to the originating account (templated).
4093
+ """
4094
+
4095
+ template_fields: Sequence[str] = tuple(
4096
+ {"entry_id", "entry_group_id"} | set(DataplexCatalogBaseOperator.template_fields)
4097
+ )
4098
+
4099
+ def __init__(
4100
+ self,
4101
+ entry_id: str,
4102
+ entry_group_id: str,
4103
+ *args,
4104
+ **kwargs,
4105
+ ) -> None:
4106
+ super().__init__(*args, **kwargs)
4107
+ self.entry_id = entry_id
4108
+ self.entry_group_id = entry_group_id
4109
+
4110
+ def execute(self, context: Context):
4111
+ self.log.info(
4112
+ "Deleting Dataplex Catalog Entry %s.",
4113
+ self.entry_id,
4114
+ )
4115
+ try:
4116
+ entry = self.hook.delete_entry(
4117
+ entry_id=self.entry_id,
4118
+ entry_group_id=self.entry_group_id,
4119
+ location=self.location,
4120
+ project_id=self.project_id,
4121
+ retry=self.retry,
4122
+ timeout=self.timeout,
4123
+ metadata=self.metadata,
4124
+ )
4125
+
4126
+ except NotFound:
4127
+ self.log.info(
4128
+ "Dataplex Catalog Entry %s not found.",
4129
+ self.entry_id,
4130
+ )
4131
+ raise AirflowException(NotFound)
4132
+ except Exception as ex:
4133
+ raise AirflowException(ex)
4134
+ return Entry.to_dict(entry)