apache-airflow-providers-google 15.1.0rc1__py3-none-any.whl → 19.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. airflow/providers/google/3rd-party-licenses/NOTICE +2 -12
  2. airflow/providers/google/__init__.py +3 -3
  3. airflow/providers/google/ads/hooks/ads.py +39 -6
  4. airflow/providers/google/ads/operators/ads.py +2 -2
  5. airflow/providers/google/ads/transfers/ads_to_gcs.py +2 -2
  6. airflow/providers/google/assets/gcs.py +1 -11
  7. airflow/providers/google/cloud/bundles/__init__.py +16 -0
  8. airflow/providers/google/cloud/bundles/gcs.py +161 -0
  9. airflow/providers/google/cloud/hooks/alloy_db.py +1 -1
  10. airflow/providers/google/cloud/hooks/bigquery.py +176 -293
  11. airflow/providers/google/cloud/hooks/cloud_batch.py +1 -1
  12. airflow/providers/google/cloud/hooks/cloud_build.py +1 -1
  13. airflow/providers/google/cloud/hooks/cloud_composer.py +288 -15
  14. airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
  15. airflow/providers/google/cloud/hooks/cloud_memorystore.py +1 -1
  16. airflow/providers/google/cloud/hooks/cloud_run.py +18 -10
  17. airflow/providers/google/cloud/hooks/cloud_sql.py +102 -23
  18. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +29 -7
  19. airflow/providers/google/cloud/hooks/compute.py +1 -1
  20. airflow/providers/google/cloud/hooks/compute_ssh.py +6 -2
  21. airflow/providers/google/cloud/hooks/datacatalog.py +10 -1
  22. airflow/providers/google/cloud/hooks/dataflow.py +72 -95
  23. airflow/providers/google/cloud/hooks/dataform.py +1 -1
  24. airflow/providers/google/cloud/hooks/datafusion.py +21 -19
  25. airflow/providers/google/cloud/hooks/dataplex.py +2 -2
  26. airflow/providers/google/cloud/hooks/dataprep.py +1 -1
  27. airflow/providers/google/cloud/hooks/dataproc.py +73 -72
  28. airflow/providers/google/cloud/hooks/dataproc_metastore.py +1 -1
  29. airflow/providers/google/cloud/hooks/dlp.py +1 -1
  30. airflow/providers/google/cloud/hooks/functions.py +1 -1
  31. airflow/providers/google/cloud/hooks/gcs.py +112 -15
  32. airflow/providers/google/cloud/hooks/gdm.py +1 -1
  33. airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
  34. airflow/providers/google/cloud/hooks/kubernetes_engine.py +3 -3
  35. airflow/providers/google/cloud/hooks/looker.py +6 -2
  36. airflow/providers/google/cloud/hooks/managed_kafka.py +1 -1
  37. airflow/providers/google/cloud/hooks/mlengine.py +4 -3
  38. airflow/providers/google/cloud/hooks/pubsub.py +3 -0
  39. airflow/providers/google/cloud/hooks/secret_manager.py +102 -10
  40. airflow/providers/google/cloud/hooks/spanner.py +74 -9
  41. airflow/providers/google/cloud/hooks/stackdriver.py +11 -9
  42. airflow/providers/google/cloud/hooks/tasks.py +1 -1
  43. airflow/providers/google/cloud/hooks/translate.py +2 -2
  44. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +2 -210
  45. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +3 -3
  46. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +28 -2
  47. airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
  48. airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +308 -8
  49. airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
  50. airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +1 -1
  51. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +1 -1
  52. airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +1 -1
  53. airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
  54. airflow/providers/google/cloud/hooks/vision.py +3 -3
  55. airflow/providers/google/cloud/hooks/workflows.py +1 -1
  56. airflow/providers/google/cloud/links/alloy_db.py +0 -46
  57. airflow/providers/google/cloud/links/base.py +77 -13
  58. airflow/providers/google/cloud/links/bigquery.py +0 -47
  59. airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
  60. airflow/providers/google/cloud/links/bigtable.py +0 -48
  61. airflow/providers/google/cloud/links/cloud_build.py +0 -73
  62. airflow/providers/google/cloud/links/cloud_functions.py +0 -33
  63. airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
  64. airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
  65. airflow/providers/google/cloud/links/cloud_sql.py +0 -33
  66. airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -44
  67. airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
  68. airflow/providers/google/cloud/links/compute.py +0 -58
  69. airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
  70. airflow/providers/google/cloud/links/datacatalog.py +23 -54
  71. airflow/providers/google/cloud/links/dataflow.py +0 -34
  72. airflow/providers/google/cloud/links/dataform.py +0 -64
  73. airflow/providers/google/cloud/links/datafusion.py +1 -96
  74. airflow/providers/google/cloud/links/dataplex.py +0 -154
  75. airflow/providers/google/cloud/links/dataprep.py +0 -24
  76. airflow/providers/google/cloud/links/dataproc.py +11 -95
  77. airflow/providers/google/cloud/links/datastore.py +0 -31
  78. airflow/providers/google/cloud/links/kubernetes_engine.py +9 -60
  79. airflow/providers/google/cloud/links/managed_kafka.py +0 -70
  80. airflow/providers/google/cloud/links/mlengine.py +0 -70
  81. airflow/providers/google/cloud/links/pubsub.py +0 -32
  82. airflow/providers/google/cloud/links/spanner.py +0 -33
  83. airflow/providers/google/cloud/links/stackdriver.py +0 -30
  84. airflow/providers/google/cloud/links/translate.py +17 -187
  85. airflow/providers/google/cloud/links/vertex_ai.py +28 -195
  86. airflow/providers/google/cloud/links/workflows.py +0 -52
  87. airflow/providers/google/cloud/log/gcs_task_handler.py +58 -22
  88. airflow/providers/google/cloud/log/stackdriver_task_handler.py +9 -6
  89. airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
  90. airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
  91. airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
  92. airflow/providers/google/cloud/openlineage/facets.py +102 -1
  93. airflow/providers/google/cloud/openlineage/mixins.py +10 -8
  94. airflow/providers/google/cloud/openlineage/utils.py +15 -1
  95. airflow/providers/google/cloud/operators/alloy_db.py +71 -56
  96. airflow/providers/google/cloud/operators/bigquery.py +73 -636
  97. airflow/providers/google/cloud/operators/bigquery_dts.py +4 -6
  98. airflow/providers/google/cloud/operators/bigtable.py +37 -8
  99. airflow/providers/google/cloud/operators/cloud_base.py +21 -1
  100. airflow/providers/google/cloud/operators/cloud_batch.py +3 -3
  101. airflow/providers/google/cloud/operators/cloud_build.py +76 -33
  102. airflow/providers/google/cloud/operators/cloud_composer.py +129 -41
  103. airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
  104. airflow/providers/google/cloud/operators/cloud_memorystore.py +69 -43
  105. airflow/providers/google/cloud/operators/cloud_run.py +24 -6
  106. airflow/providers/google/cloud/operators/cloud_sql.py +8 -17
  107. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +93 -12
  108. airflow/providers/google/cloud/operators/compute.py +9 -41
  109. airflow/providers/google/cloud/operators/datacatalog.py +157 -21
  110. airflow/providers/google/cloud/operators/dataflow.py +40 -16
  111. airflow/providers/google/cloud/operators/dataform.py +15 -5
  112. airflow/providers/google/cloud/operators/datafusion.py +42 -21
  113. airflow/providers/google/cloud/operators/dataplex.py +194 -110
  114. airflow/providers/google/cloud/operators/dataprep.py +1 -5
  115. airflow/providers/google/cloud/operators/dataproc.py +80 -36
  116. airflow/providers/google/cloud/operators/dataproc_metastore.py +97 -89
  117. airflow/providers/google/cloud/operators/datastore.py +23 -7
  118. airflow/providers/google/cloud/operators/dlp.py +6 -29
  119. airflow/providers/google/cloud/operators/functions.py +17 -8
  120. airflow/providers/google/cloud/operators/gcs.py +12 -9
  121. airflow/providers/google/cloud/operators/gen_ai.py +389 -0
  122. airflow/providers/google/cloud/operators/kubernetes_engine.py +62 -100
  123. airflow/providers/google/cloud/operators/looker.py +2 -2
  124. airflow/providers/google/cloud/operators/managed_kafka.py +108 -53
  125. airflow/providers/google/cloud/operators/natural_language.py +1 -1
  126. airflow/providers/google/cloud/operators/pubsub.py +68 -15
  127. airflow/providers/google/cloud/operators/spanner.py +26 -13
  128. airflow/providers/google/cloud/operators/speech_to_text.py +2 -3
  129. airflow/providers/google/cloud/operators/stackdriver.py +1 -9
  130. airflow/providers/google/cloud/operators/tasks.py +1 -12
  131. airflow/providers/google/cloud/operators/text_to_speech.py +2 -3
  132. airflow/providers/google/cloud/operators/translate.py +41 -17
  133. airflow/providers/google/cloud/operators/translate_speech.py +2 -3
  134. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +39 -19
  135. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +30 -10
  136. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +55 -27
  137. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +70 -8
  138. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +43 -9
  139. airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
  140. airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
  141. airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -115
  142. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +12 -10
  143. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +57 -11
  144. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +31 -8
  145. airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
  146. airflow/providers/google/cloud/operators/video_intelligence.py +1 -1
  147. airflow/providers/google/cloud/operators/vision.py +2 -2
  148. airflow/providers/google/cloud/operators/workflows.py +18 -15
  149. airflow/providers/google/cloud/secrets/secret_manager.py +3 -2
  150. airflow/providers/google/cloud/sensors/bigquery.py +3 -3
  151. airflow/providers/google/cloud/sensors/bigquery_dts.py +2 -3
  152. airflow/providers/google/cloud/sensors/bigtable.py +11 -4
  153. airflow/providers/google/cloud/sensors/cloud_composer.py +533 -30
  154. airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +2 -3
  155. airflow/providers/google/cloud/sensors/dataflow.py +26 -10
  156. airflow/providers/google/cloud/sensors/dataform.py +2 -3
  157. airflow/providers/google/cloud/sensors/datafusion.py +4 -5
  158. airflow/providers/google/cloud/sensors/dataplex.py +2 -3
  159. airflow/providers/google/cloud/sensors/dataprep.py +2 -2
  160. airflow/providers/google/cloud/sensors/dataproc.py +2 -3
  161. airflow/providers/google/cloud/sensors/dataproc_metastore.py +2 -3
  162. airflow/providers/google/cloud/sensors/gcs.py +4 -5
  163. airflow/providers/google/cloud/sensors/looker.py +2 -3
  164. airflow/providers/google/cloud/sensors/pubsub.py +4 -5
  165. airflow/providers/google/cloud/sensors/tasks.py +2 -2
  166. airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -3
  167. airflow/providers/google/cloud/sensors/workflows.py +2 -3
  168. airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
  169. airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
  170. airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +4 -3
  171. airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
  172. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +10 -5
  173. airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
  174. airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
  175. airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
  176. airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
  177. airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
  178. airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +3 -3
  179. airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -4
  180. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +21 -13
  181. airflow/providers/google/cloud/transfers/gcs_to_gcs.py +4 -3
  182. airflow/providers/google/cloud/transfers/gcs_to_local.py +6 -4
  183. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +11 -5
  184. airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
  185. airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
  186. airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
  187. airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
  188. airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
  189. airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
  190. airflow/providers/google/cloud/transfers/postgres_to_gcs.py +42 -9
  191. airflow/providers/google/cloud/transfers/s3_to_gcs.py +13 -7
  192. airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
  193. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +14 -5
  194. airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
  195. airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
  196. airflow/providers/google/cloud/triggers/bigquery.py +76 -35
  197. airflow/providers/google/cloud/triggers/cloud_build.py +1 -1
  198. airflow/providers/google/cloud/triggers/cloud_composer.py +303 -47
  199. airflow/providers/google/cloud/triggers/cloud_run.py +3 -3
  200. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +92 -2
  201. airflow/providers/google/cloud/triggers/dataflow.py +122 -0
  202. airflow/providers/google/cloud/triggers/datafusion.py +1 -1
  203. airflow/providers/google/cloud/triggers/dataplex.py +14 -2
  204. airflow/providers/google/cloud/triggers/dataproc.py +123 -53
  205. airflow/providers/google/cloud/triggers/kubernetes_engine.py +47 -28
  206. airflow/providers/google/cloud/triggers/mlengine.py +1 -1
  207. airflow/providers/google/cloud/triggers/pubsub.py +15 -19
  208. airflow/providers/google/cloud/triggers/vertex_ai.py +1 -1
  209. airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
  210. airflow/providers/google/cloud/utils/credentials_provider.py +2 -2
  211. airflow/providers/google/cloud/utils/field_sanitizer.py +1 -1
  212. airflow/providers/google/cloud/utils/field_validator.py +2 -3
  213. airflow/providers/google/common/auth_backend/google_openid.py +4 -4
  214. airflow/providers/google/common/deprecated.py +2 -1
  215. airflow/providers/google/common/hooks/base_google.py +27 -9
  216. airflow/providers/google/common/hooks/operation_helpers.py +1 -1
  217. airflow/providers/google/common/links/storage.py +0 -22
  218. airflow/providers/google/common/utils/get_secret.py +31 -0
  219. airflow/providers/google/common/utils/id_token_credentials.py +3 -4
  220. airflow/providers/google/firebase/hooks/firestore.py +1 -1
  221. airflow/providers/google/firebase/operators/firestore.py +3 -3
  222. airflow/providers/google/get_provider_info.py +56 -52
  223. airflow/providers/google/go_module_utils.py +35 -3
  224. airflow/providers/google/leveldb/hooks/leveldb.py +27 -2
  225. airflow/providers/google/leveldb/operators/leveldb.py +2 -2
  226. airflow/providers/google/marketing_platform/hooks/campaign_manager.py +1 -1
  227. airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
  228. airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
  229. airflow/providers/google/marketing_platform/links/analytics_admin.py +5 -14
  230. airflow/providers/google/marketing_platform/operators/analytics_admin.py +2 -3
  231. airflow/providers/google/marketing_platform/operators/campaign_manager.py +6 -6
  232. airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
  233. airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
  234. airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
  235. airflow/providers/google/marketing_platform/sensors/display_video.py +3 -64
  236. airflow/providers/google/suite/hooks/calendar.py +2 -2
  237. airflow/providers/google/suite/hooks/sheets.py +16 -2
  238. airflow/providers/google/suite/operators/sheets.py +8 -3
  239. airflow/providers/google/suite/sensors/drive.py +2 -2
  240. airflow/providers/google/suite/transfers/gcs_to_gdrive.py +3 -3
  241. airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
  242. airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
  243. airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
  244. airflow/providers/google/version_compat.py +15 -1
  245. {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.3.0.dist-info}/METADATA +90 -46
  246. apache_airflow_providers_google-19.3.0.dist-info/RECORD +331 -0
  247. apache_airflow_providers_google-19.3.0.dist-info/licenses/NOTICE +5 -0
  248. airflow/providers/google/cloud/hooks/automl.py +0 -673
  249. airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
  250. airflow/providers/google/cloud/links/automl.py +0 -193
  251. airflow/providers/google/cloud/operators/automl.py +0 -1362
  252. airflow/providers/google/cloud/operators/life_sciences.py +0 -119
  253. airflow/providers/google/cloud/operators/mlengine.py +0 -112
  254. apache_airflow_providers_google-15.1.0rc1.dist-info/RECORD +0 -321
  255. {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.3.0.dist-info}/WHEEL +0 -0
  256. {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.3.0.dist-info}/entry_points.txt +0 -0
  257. {airflow/providers/google → apache_airflow_providers_google-19.3.0.dist-info/licenses}/LICENSE +0 -0
@@ -19,494 +19,23 @@
19
19
 
20
20
  from __future__ import annotations
21
21
 
22
- import csv
23
- import json
24
- import shutil
22
+ import os
25
23
  import tempfile
26
- import urllib.request
24
+ import zipfile
27
25
  from collections.abc import Sequence
28
26
  from typing import TYPE_CHECKING, Any
29
- from urllib.parse import urlsplit
30
27
 
31
- from airflow.exceptions import AirflowException
32
- from airflow.models import BaseOperator
33
28
  from airflow.providers.google.cloud.hooks.gcs import GCSHook
34
29
  from airflow.providers.google.marketing_platform.hooks.display_video import GoogleDisplayVideo360Hook
30
+ from airflow.providers.google.version_compat import BaseOperator
35
31
 
36
32
  if TYPE_CHECKING:
37
- from airflow.utils.context import Context
38
-
39
-
40
- class GoogleDisplayVideo360CreateQueryOperator(BaseOperator):
41
- """
42
- Creates a query.
43
-
44
- .. seealso::
45
- For more information on how to use this operator, take a look at the guide:
46
- ``GoogleDisplayVideo360CreateQueryOperator``
47
-
48
- .. seealso::
49
- Check also the official API docs:
50
- `https://developers.google.com/bid-manager/v2/queries/create`
51
-
52
- :param body: Report object passed to the request's body as described here:
53
- https://developers.google.com/bid-manager/v2/queries#Query
54
- :param api_version: The version of the api that will be requested for example 'v3'.
55
- :param gcp_conn_id: The connection ID to use when fetching connection info.
56
- :param impersonation_chain: Optional service account to impersonate using short-term
57
- credentials, or chained list of accounts required to get the access_token
58
- of the last account in the list, which will be impersonated in the request.
59
- If set as a string, the account must grant the originating account
60
- the Service Account Token Creator IAM role.
61
- If set as a sequence, the identities from the list must grant
62
- Service Account Token Creator IAM role to the directly preceding identity, with first
63
- account from the list granting this role to the originating account (templated).
64
- """
65
-
66
- template_fields: Sequence[str] = (
67
- "body",
68
- "impersonation_chain",
69
- )
70
- template_ext: Sequence[str] = (".json",)
71
-
72
- def __init__(
73
- self,
74
- *,
75
- body: dict[str, Any],
76
- api_version: str = "v2",
77
- gcp_conn_id: str = "google_cloud_default",
78
- impersonation_chain: str | Sequence[str] | None = None,
79
- **kwargs,
80
- ) -> None:
81
- super().__init__(**kwargs)
82
- self.body = body
83
- self.api_version = api_version
84
- self.gcp_conn_id = gcp_conn_id
85
- self.impersonation_chain = impersonation_chain
86
-
87
- def prepare_template(self) -> None:
88
- # If .json is passed then we have to read the file
89
- if isinstance(self.body, str) and self.body.endswith(".json"):
90
- with open(self.body) as file:
91
- self.body = json.load(file)
92
-
93
- def execute(self, context: Context) -> dict:
94
- hook = GoogleDisplayVideo360Hook(
95
- gcp_conn_id=self.gcp_conn_id,
96
- api_version=self.api_version,
97
- impersonation_chain=self.impersonation_chain,
98
- )
99
- self.log.info("Creating Display & Video 360 query.")
100
- response = hook.create_query(query=self.body)
101
- query_id = response["queryId"]
102
- self.xcom_push(context, key="query_id", value=query_id)
103
- self.log.info("Created query with ID: %s", query_id)
104
- return response
105
-
106
-
107
- class GoogleDisplayVideo360DeleteReportOperator(BaseOperator):
108
- """
109
- Deletes a stored query as well as the associated stored reports.
110
-
111
- .. seealso::
112
- For more information on how to use this operator, take a look at the guide:
113
- :ref:`howto/operator:GoogleDisplayVideo360DeleteReportOperator`
114
-
115
- .. seealso::
116
- Check also the official API docs:
117
- `https://developers.google.com/bid-manager/v2/queries/delete`
118
-
119
- :param report_id: Report ID to delete.
120
- :param report_name: Name of the report to delete.
121
- :param api_version: The version of the api that will be requested for example 'v3'.
122
- :param gcp_conn_id: The connection ID to use when fetching connection info.
123
- :param impersonation_chain: Optional service account to impersonate using short-term
124
- credentials, or chained list of accounts required to get the access_token
125
- of the last account in the list, which will be impersonated in the request.
126
- If set as a string, the account must grant the originating account
127
- the Service Account Token Creator IAM role.
128
- If set as a sequence, the identities from the list must grant
129
- Service Account Token Creator IAM role to the directly preceding identity, with first
130
- account from the list granting this role to the originating account (templated).
131
- """
132
-
133
- template_fields: Sequence[str] = (
134
- "report_id",
135
- "impersonation_chain",
136
- )
137
-
138
- def __init__(
139
- self,
140
- *,
141
- report_id: str | None = None,
142
- report_name: str | None = None,
143
- api_version: str = "v2",
144
- gcp_conn_id: str = "google_cloud_default",
145
- impersonation_chain: str | Sequence[str] | None = None,
146
- **kwargs,
147
- ) -> None:
148
- super().__init__(**kwargs)
149
- self.report_id = report_id
150
- self.report_name = report_name
151
- self.api_version = api_version
152
- self.gcp_conn_id = gcp_conn_id
153
- self.impersonation_chain = impersonation_chain
154
-
155
- if report_name and report_id:
156
- raise AirflowException("Use only one value - `report_name` or `report_id`.")
157
-
158
- if not (report_name or report_id):
159
- raise AirflowException("Provide one of the values: `report_name` or `report_id`.")
160
-
161
- def execute(self, context: Context) -> None:
162
- hook = GoogleDisplayVideo360Hook(
163
- gcp_conn_id=self.gcp_conn_id,
164
- api_version=self.api_version,
165
- impersonation_chain=self.impersonation_chain,
166
- )
167
- if self.report_id:
168
- reports_ids_to_delete = [self.report_id]
169
- else:
170
- reports = hook.list_queries()
171
- reports_ids_to_delete = [
172
- report["queryId"] for report in reports if report["metadata"]["title"] == self.report_name
173
- ]
174
-
175
- for report_id in reports_ids_to_delete:
176
- self.log.info("Deleting report with id: %s", report_id)
177
- hook.delete_query(query_id=report_id)
178
- self.log.info("Report deleted.")
179
-
180
-
181
- class GoogleDisplayVideo360DownloadReportV2Operator(BaseOperator):
182
- """
183
- Retrieves a stored query.
184
-
185
- .. seealso::
186
- For more information on how to use this operator, take a look at the guide:
187
- :ref:`howto/operator:GoogleDisplayVideo360DownloadReportV2Operator`
188
-
189
- .. seealso::
190
- Check also the official API docs:
191
- `https://developers.google.com/bid-manager/v2/queries/get`
192
-
193
- :param report_id: Report ID to retrieve.
194
- :param bucket_name: The bucket to upload to.
195
- :param report_name: The report name to set when uploading the local file.
196
- :param chunk_size: File will be downloaded in chunks of this many bytes.
197
- :param gzip: Option to compress local file or file data for upload
198
- :param api_version: The version of the api that will be requested for example 'v3'.
199
- :param gcp_conn_id: The connection ID to use when fetching connection info.
200
- :param impersonation_chain: Optional service account to impersonate using short-term
201
- credentials, or chained list of accounts required to get the access_token
202
- of the last account in the list, which will be impersonated in the request.
203
- If set as a string, the account must grant the originating account
204
- the Service Account Token Creator IAM role.
205
- If set as a sequence, the identities from the list must grant
206
- Service Account Token Creator IAM role to the directly preceding identity, with first
207
- account from the list granting this role to the originating account (templated).
208
- """
209
-
210
- template_fields: Sequence[str] = (
211
- "query_id",
212
- "report_id",
213
- "bucket_name",
214
- "report_name",
215
- "impersonation_chain",
216
- )
217
-
218
- def __init__(
219
- self,
220
- *,
221
- query_id: str,
222
- report_id: str,
223
- bucket_name: str,
224
- report_name: str | None = None,
225
- gzip: bool = True,
226
- chunk_size: int = 10 * 1024 * 1024,
227
- api_version: str = "v2",
228
- gcp_conn_id: str = "google_cloud_default",
229
- impersonation_chain: str | Sequence[str] | None = None,
230
- **kwargs,
231
- ) -> None:
232
- super().__init__(**kwargs)
233
- self.query_id = query_id
234
- self.report_id = report_id
235
- self.chunk_size = chunk_size
236
- self.gzip = gzip
237
- self.bucket_name = bucket_name
238
- self.report_name = report_name
239
- self.api_version = api_version
240
- self.gcp_conn_id = gcp_conn_id
241
- self.impersonation_chain = impersonation_chain
242
-
243
- def _resolve_file_name(self, name: str) -> str:
244
- new_name = name if name.endswith(".csv") else f"{name}.csv"
245
- new_name = f"{new_name}.gz" if self.gzip else new_name
246
- return new_name
247
-
248
- @staticmethod
249
- def _set_bucket_name(name: str) -> str:
250
- bucket = name if not name.startswith("gs://") else name[5:]
251
- return bucket.strip("/")
252
-
253
- def execute(self, context: Context):
254
- hook = GoogleDisplayVideo360Hook(
255
- gcp_conn_id=self.gcp_conn_id,
256
- api_version=self.api_version,
257
- impersonation_chain=self.impersonation_chain,
258
- )
259
- gcs_hook = GCSHook(
260
- gcp_conn_id=self.gcp_conn_id,
261
- impersonation_chain=self.impersonation_chain,
262
- )
263
-
264
- resource = hook.get_report(query_id=self.query_id, report_id=self.report_id)
265
- status = resource.get("metadata", {}).get("status", {}).get("state")
266
- if resource and status not in ["DONE", "FAILED"]:
267
- raise AirflowException(f"Report {self.report_id} for query {self.query_id} is still running")
268
-
269
- # If no custom report_name provided, use DV360 name
270
- file_url = resource["metadata"]["googleCloudStoragePath"]
271
- if urllib.parse.urlparse(file_url).scheme == "file":
272
- raise AirflowException("Accessing local file is not allowed in this operator")
273
- report_name = self.report_name or urlsplit(file_url).path.split("/")[-1]
274
- report_name = self._resolve_file_name(report_name)
275
-
276
- # Download the report
277
- self.log.info("Starting downloading report %s", self.report_id)
278
- with tempfile.NamedTemporaryFile(delete=False) as temp_file:
279
- with urllib.request.urlopen(file_url) as response: # nosec
280
- shutil.copyfileobj(response, temp_file, length=self.chunk_size)
281
-
282
- temp_file.flush()
283
- # Upload the local file to bucket
284
- bucket_name = self._set_bucket_name(self.bucket_name)
285
- gcs_hook.upload(
286
- bucket_name=bucket_name,
287
- object_name=report_name,
288
- gzip=self.gzip,
289
- filename=temp_file.name,
290
- mime_type="text/csv",
291
- )
292
- self.log.info(
293
- "Report %s was saved in bucket %s as %s.",
294
- self.report_id,
295
- self.bucket_name,
296
- report_name,
297
- )
298
- self.xcom_push(context, key="report_name", value=report_name)
299
-
300
-
301
- class GoogleDisplayVideo360RunQueryOperator(BaseOperator):
302
- """
303
- Runs a stored query to generate a report.
304
-
305
- .. seealso::
306
- For more information on how to use this operator, take a look at the guide:
307
- :ref:`howto/operator:GoogleDisplayVideo360RunQueryOperator`
308
-
309
- .. seealso::
310
- Check also the official API docs:
311
- `https://developers.google.com/bid-manager/v2/queries/run`
312
-
313
- :param report_id: Report ID to run.
314
- :param parameters: Parameters for running a report as described here:
315
- https://developers.google.com/bid-manager/v2/queries/run
316
- :param api_version: The version of the api that will be requested for example 'v3'.
317
- :param gcp_conn_id: The connection ID to use when fetching connection info.
318
- :param impersonation_chain: Optional service account to impersonate using short-term
319
- credentials, or chained list of accounts required to get the access_token
320
- of the last account in the list, which will be impersonated in the request.
321
- If set as a string, the account must grant the originating account
322
- the Service Account Token Creator IAM role.
323
- If set as a sequence, the identities from the list must grant
324
- Service Account Token Creator IAM role to the directly preceding identity, with first
325
- account from the list granting this role to the originating account (templated).
326
- """
327
-
328
- template_fields: Sequence[str] = (
329
- "query_id",
330
- "parameters",
331
- "impersonation_chain",
332
- )
333
-
334
- def __init__(
335
- self,
336
- *,
337
- query_id: str,
338
- parameters: dict[str, Any] | None = None,
339
- api_version: str = "v2",
340
- gcp_conn_id: str = "google_cloud_default",
341
- impersonation_chain: str | Sequence[str] | None = None,
342
- **kwargs,
343
- ) -> None:
344
- super().__init__(**kwargs)
345
- self.query_id = query_id
346
- self.api_version = api_version
347
- self.gcp_conn_id = gcp_conn_id
348
- self.parameters = parameters
349
- self.impersonation_chain = impersonation_chain
350
-
351
- def execute(self, context: Context) -> dict:
352
- hook = GoogleDisplayVideo360Hook(
353
- gcp_conn_id=self.gcp_conn_id,
354
- api_version=self.api_version,
355
- impersonation_chain=self.impersonation_chain,
356
- )
357
- self.log.info(
358
- "Running query %s with the following parameters:\n %s",
359
- self.query_id,
360
- self.parameters,
361
- )
362
- response = hook.run_query(query_id=self.query_id, params=self.parameters)
363
- self.xcom_push(context, key="query_id", value=response["key"]["queryId"])
364
- self.xcom_push(context, key="report_id", value=response["key"]["reportId"])
365
- return response
366
-
367
-
368
- class GoogleDisplayVideo360DownloadLineItemsOperator(BaseOperator):
369
- """
370
- Retrieves line items in CSV format.
371
-
372
- .. seealso::
373
- For more information on how to use this operator, take a look at the guide:
374
- :ref:`howto/operator:GoogleDisplayVideo360DownloadLineItemsOperator`
375
-
376
- .. seealso::
377
- Check also the official API docs:
378
- `https://developers.google.com/bid-manager/v1.1/lineitems/downloadlineitems`
379
-
380
- :param request_body: dictionary with parameters that should be passed into.
381
- More information about it can be found here:
382
- https://developers.google.com/bid-manager/v1.1/lineitems/downloadlineitems
383
- """
384
-
385
- template_fields: Sequence[str] = (
386
- "request_body",
387
- "bucket_name",
388
- "object_name",
389
- "impersonation_chain",
390
- )
391
-
392
- def __init__(
393
- self,
394
- *,
395
- request_body: dict[str, Any],
396
- bucket_name: str,
397
- object_name: str,
398
- gzip: bool = False,
399
- api_version: str = "v1.1",
400
- gcp_conn_id: str = "google_cloud_default",
401
- impersonation_chain: str | Sequence[str] | None = None,
402
- **kwargs,
403
- ) -> None:
404
- super().__init__(**kwargs)
405
- self.request_body = request_body
406
- self.object_name = object_name
407
- self.bucket_name = bucket_name
408
- self.gzip = gzip
409
- self.api_version = api_version
410
- self.gcp_conn_id = gcp_conn_id
411
- self.impersonation_chain = impersonation_chain
412
-
413
- def execute(self, context: Context) -> str:
414
- gcs_hook = GCSHook(
415
- gcp_conn_id=self.gcp_conn_id,
416
- impersonation_chain=self.impersonation_chain,
417
- )
418
- hook = GoogleDisplayVideo360Hook(
419
- gcp_conn_id=self.gcp_conn_id,
420
- api_version=self.api_version,
421
- impersonation_chain=self.impersonation_chain,
422
- )
423
-
424
- self.log.info("Retrieving report...")
425
- content: list[str] = hook.download_line_items(request_body=self.request_body)
426
-
427
- with tempfile.NamedTemporaryFile("w+") as temp_file:
428
- writer = csv.writer(temp_file)
429
- writer.writerows(content)
430
- temp_file.flush()
431
- gcs_hook.upload(
432
- bucket_name=self.bucket_name,
433
- object_name=self.object_name,
434
- filename=temp_file.name,
435
- mime_type="text/csv",
436
- gzip=self.gzip,
437
- )
438
- return f"{self.bucket_name}/{self.object_name}"
439
-
440
-
441
- class GoogleDisplayVideo360UploadLineItemsOperator(BaseOperator):
442
- """
443
- Uploads line items in CSV format.
444
-
445
- .. seealso::
446
- For more information on how to use this operator, take a look at the guide:
447
- :ref:`howto/operator:GoogleDisplayVideo360UploadLineItemsOperator`
448
-
449
- .. seealso::
450
- Check also the official API docs:
451
- `https://developers.google.com/bid-manager/v1.1/lineitems/uploadlineitems`
452
-
453
- :param request_body: request to upload line items.
454
- :param bucket_name: The bucket form data is downloaded.
455
- :param object_name: The object to fetch.
456
- :param filename: The filename to fetch.
457
- :param dry_run: Upload status without actually persisting the line items.
458
- """
459
-
460
- template_fields: Sequence[str] = (
461
- "bucket_name",
462
- "object_name",
463
- "impersonation_chain",
464
- )
465
-
466
- def __init__(
467
- self,
468
- *,
469
- bucket_name: str,
470
- object_name: str,
471
- api_version: str = "v1.1",
472
- gcp_conn_id: str = "google_cloud_default",
473
- impersonation_chain: str | Sequence[str] | None = None,
474
- **kwargs,
475
- ) -> None:
476
- super().__init__(**kwargs)
477
- self.bucket_name = bucket_name
478
- self.object_name = object_name
479
- self.api_version = api_version
480
- self.gcp_conn_id = gcp_conn_id
481
- self.impersonation_chain = impersonation_chain
482
-
483
- def execute(self, context: Context) -> None:
484
- gcs_hook = GCSHook(
485
- gcp_conn_id=self.gcp_conn_id,
486
- impersonation_chain=self.impersonation_chain,
487
- )
488
- hook = GoogleDisplayVideo360Hook(
489
- gcp_conn_id=self.gcp_conn_id,
490
- api_version=self.api_version,
491
- impersonation_chain=self.impersonation_chain,
492
- )
493
-
494
- self.log.info("Uploading file %s...")
495
- # Saving file in the temporary directory,
496
- # downloaded file from the GCS could be a 1GB size or even more
497
- with tempfile.NamedTemporaryFile("w+") as f:
498
- line_items = gcs_hook.download(
499
- bucket_name=self.bucket_name,
500
- object_name=self.object_name,
501
- filename=f.name,
502
- )
503
- f.flush()
504
- hook.upload_line_items(line_items=line_items)
33
+ from airflow.providers.common.compat.sdk import Context
505
34
 
506
35
 
507
36
  class GoogleDisplayVideo360CreateSDFDownloadTaskOperator(BaseOperator):
508
37
  """
509
- Creates SDF operation task.
38
+ Creates an SDF operation task.
510
39
 
511
40
  .. seealso::
512
41
  For more information on how to use this operator, take a look at the guide:
@@ -542,7 +71,7 @@ class GoogleDisplayVideo360CreateSDFDownloadTaskOperator(BaseOperator):
542
71
  self,
543
72
  *,
544
73
  body_request: dict[str, Any],
545
- api_version: str = "v1",
74
+ api_version: str = "v4",
546
75
  gcp_conn_id: str = "google_cloud_default",
547
76
  impersonation_chain: str | Sequence[str] | None = None,
548
77
  **kwargs,
@@ -564,7 +93,7 @@ class GoogleDisplayVideo360CreateSDFDownloadTaskOperator(BaseOperator):
564
93
  operation = hook.create_sdf_download_operation(body_request=self.body_request)
565
94
 
566
95
  name = operation["name"]
567
- self.xcom_push(context, key="name", value=name)
96
+ context["task_instance"].xcom_push(key="name", value=name)
568
97
  self.log.info("Created SDF operation with name: %s", name)
569
98
 
570
99
  return operation
@@ -613,7 +142,7 @@ class GoogleDisplayVideo360SDFtoGCSOperator(BaseOperator):
613
142
  bucket_name: str,
614
143
  object_name: str,
615
144
  gzip: bool = False,
616
- api_version: str = "v1",
145
+ api_version: str = "v4",
617
146
  gcp_conn_id: str = "google_cloud_default",
618
147
  impersonation_chain: str | Sequence[str] | None = None,
619
148
  **kwargs,
@@ -645,14 +174,24 @@ class GoogleDisplayVideo360SDFtoGCSOperator(BaseOperator):
645
174
  media = hook.download_media(resource_name=operation_state["response"]["resourceName"])
646
175
 
647
176
  self.log.info("Sending file to the Google Cloud Storage...")
648
- with tempfile.NamedTemporaryFile() as temp_file:
649
- hook.download_content_from_request(temp_file, media, chunk_size=1024 * 1024)
650
- temp_file.flush()
651
- gcs_hook.upload(
652
- bucket_name=self.bucket_name,
653
- object_name=self.object_name,
654
- filename=temp_file.name,
655
- gzip=self.gzip,
656
- )
657
-
177
+ with tempfile.TemporaryDirectory() as tmp_dir:
178
+ zip_path = os.path.join(tmp_dir, "sdf.zip")
179
+
180
+ # Download the ZIP
181
+ with open(zip_path, "wb") as f:
182
+ hook.download_content_from_request(f, media, chunk_size=1024 * 1024)
183
+
184
+ # Extract CSV
185
+ with zipfile.ZipFile(zip_path, "r") as zip_ref:
186
+ zip_ref.extractall(tmp_dir)
187
+
188
+ # Upload CSV file
189
+ for fname in os.listdir(tmp_dir):
190
+ if fname.endswith(".csv"):
191
+ gcs_hook.upload(
192
+ bucket_name=self.bucket_name,
193
+ object_name=self.object_name,
194
+ filename=os.path.join(tmp_dir, fname),
195
+ gzip=False,
196
+ )
658
197
  return f"{self.bucket_name}/{self.object_name}"
@@ -23,11 +23,11 @@ from collections.abc import Sequence
23
23
  from functools import cached_property
24
24
  from typing import TYPE_CHECKING, Any
25
25
 
26
- from airflow.models import BaseOperator
27
26
  from airflow.providers.google.marketing_platform.hooks.search_ads import GoogleSearchAdsReportingHook
27
+ from airflow.providers.google.version_compat import BaseOperator
28
28
 
29
29
  if TYPE_CHECKING:
30
- from airflow.utils.context import Context
30
+ from airflow.providers.common.compat.sdk import Context
31
31
 
32
32
 
33
33
  class _GoogleSearchAdsBaseOperator(BaseOperator):
@@ -22,11 +22,11 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
+ from airflow.providers.common.compat.sdk import BaseSensorOperator
25
26
  from airflow.providers.google.marketing_platform.hooks.campaign_manager import GoogleCampaignManagerHook
26
- from airflow.sensors.base import BaseSensorOperator
27
27
 
28
28
  if TYPE_CHECKING:
29
- from airflow.utils.context import Context
29
+ from airflow.providers.common.compat.sdk import Context
30
30
 
31
31
 
32
32
  class GoogleCampaignManagerReportSensor(BaseSensorOperator):
@@ -21,12 +21,11 @@ from __future__ import annotations
21
21
  from collections.abc import Sequence
22
22
  from typing import TYPE_CHECKING
23
23
 
24
- from airflow.exceptions import AirflowException
24
+ from airflow.providers.common.compat.sdk import AirflowException, BaseSensorOperator
25
25
  from airflow.providers.google.marketing_platform.hooks.display_video import GoogleDisplayVideo360Hook
26
- from airflow.sensors.base import BaseSensorOperator
27
26
 
28
27
  if TYPE_CHECKING:
29
- from airflow.utils.context import Context
28
+ from airflow.providers.common.compat.sdk import Context
30
29
 
31
30
 
32
31
  class GoogleDisplayVideo360GetSDFDownloadOperationSensor(BaseSensorOperator):
@@ -59,7 +58,7 @@ class GoogleDisplayVideo360GetSDFDownloadOperationSensor(BaseSensorOperator):
59
58
  def __init__(
60
59
  self,
61
60
  operation_name: str,
62
- api_version: str = "v1",
61
+ api_version: str = "v4",
63
62
  gcp_conn_id: str = "google_cloud_default",
64
63
  mode: str = "reschedule",
65
64
  poke_interval: int = 60 * 5,
@@ -88,63 +87,3 @@ class GoogleDisplayVideo360GetSDFDownloadOperationSensor(BaseSensorOperator):
88
87
  if operation and operation.get("done"):
89
88
  return True
90
89
  return False
91
-
92
-
93
- class GoogleDisplayVideo360RunQuerySensor(BaseSensorOperator):
94
- """
95
- Sensor for detecting the completion of DV360 reports for API v2.
96
-
97
- .. seealso::
98
- For more information on how to use this operator, take a look at the guide:
99
- :ref:`howto/operator:GoogleDisplayVideo360RunQuerySensor`
100
-
101
- :param query_id: Query ID for which report was generated
102
- :param report_id: Report ID for which you want to wait
103
- :param api_version: The version of the api that will be requested for example 'v3'.
104
- :param gcp_conn_id: The connection ID to use when fetching connection info.
105
- :param impersonation_chain: Optional service account to impersonate using short-term
106
- credentials, or chained list of accounts required to get the access_token
107
- of the last account in the list, which will be impersonated in the request.
108
- If set as a string, the account must grant the originating account
109
- the Service Account Token Creator IAM role.
110
- If set as a sequence, the identities from the list must grant
111
- Service Account Token Creator IAM role to the directly preceding identity, with first
112
- account from the list granting this role to the originating account (templated).
113
- """
114
-
115
- template_fields: Sequence[str] = (
116
- "query_id",
117
- "report_id",
118
- "impersonation_chain",
119
- )
120
-
121
- def __init__(
122
- self,
123
- *,
124
- query_id: str,
125
- report_id: str,
126
- api_version: str = "v2",
127
- gcp_conn_id: str = "google_cloud_default",
128
- impersonation_chain: str | Sequence[str] | None = None,
129
- **kwargs,
130
- ) -> None:
131
- super().__init__(**kwargs)
132
- self.query_id = query_id
133
- self.report_id = report_id
134
- self.api_version = api_version
135
- self.gcp_conn_id = gcp_conn_id
136
- self.impersonation_chain = impersonation_chain
137
-
138
- def poke(self, context: Context) -> bool:
139
- hook = GoogleDisplayVideo360Hook(
140
- gcp_conn_id=self.gcp_conn_id,
141
- api_version=self.api_version,
142
- impersonation_chain=self.impersonation_chain,
143
- )
144
-
145
- response = hook.get_report(query_id=self.query_id, report_id=self.report_id)
146
- status = response.get("metadata", {}).get("status", {}).get("state")
147
- self.log.info("STATUS OF THE REPORT %s FOR QUERY %s: %s", self.report_id, self.query_id, status)
148
- if response and status in ["DONE", "FAILED"]:
149
- return True
150
- return False