apache-airflow-providers-google 16.0.0a1__py3-none-any.whl → 16.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/ads/hooks/ads.py +34 -0
- airflow/providers/google/cloud/hooks/bigquery.py +63 -76
- airflow/providers/google/cloud/hooks/gcs.py +3 -3
- airflow/providers/google/cloud/hooks/looker.py +5 -0
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +0 -36
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +1 -66
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +1 -1
- airflow/providers/google/cloud/links/cloud_run.py +59 -0
- airflow/providers/google/cloud/log/gcs_task_handler.py +4 -4
- airflow/providers/google/cloud/operators/bigquery.py +49 -10
- airflow/providers/google/cloud/operators/cloud_run.py +10 -1
- airflow/providers/google/cloud/operators/gcs.py +1 -0
- airflow/providers/google/cloud/operators/kubernetes_engine.py +3 -85
- airflow/providers/google/cloud/operators/pubsub.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +0 -92
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +4 -0
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +9 -5
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +11 -5
- airflow/providers/google/cloud/triggers/bigquery.py +32 -5
- airflow/providers/google/cloud/triggers/dataproc.py +62 -10
- airflow/providers/google/get_provider_info.py +14 -5
- airflow/providers/google/leveldb/hooks/leveldb.py +25 -0
- {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.0.0rc1.dist-info}/METADATA +23 -22
- {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.0.0rc1.dist-info}/RECORD +29 -28
- airflow/providers/google/cloud/links/automl.py +0 -193
- {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.0.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.0.0rc1.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,193 @@
|
|
1
|
+
#
|
2
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
3
|
+
# or more contributor license agreements. See the NOTICE file
|
4
|
+
# distributed with this work for additional information
|
5
|
+
# regarding copyright ownership. The ASF licenses this file
|
6
|
+
# to you under the Apache License, Version 2.0 (the
|
7
|
+
# "License"); you may not use this file except in compliance
|
8
|
+
# with the License. You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing,
|
13
|
+
# software distributed under the License is distributed on an
|
14
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
15
|
+
# KIND, either express or implied. See the License for the
|
16
|
+
# specific language governing permissions and limitations
|
17
|
+
# under the License.
|
18
|
+
"""This module contains operator to move data from HTTP endpoint to GCS."""
|
19
|
+
|
20
|
+
from __future__ import annotations
|
21
|
+
|
22
|
+
from functools import cached_property
|
23
|
+
from typing import TYPE_CHECKING, Any
|
24
|
+
|
25
|
+
from airflow.models import BaseOperator
|
26
|
+
from airflow.providers.google.cloud.hooks.gcs import GCSHook
|
27
|
+
from airflow.providers.http.hooks.http import HttpHook
|
28
|
+
|
29
|
+
if TYPE_CHECKING:
|
30
|
+
from collections.abc import Sequence
|
31
|
+
|
32
|
+
from requests.auth import AuthBase
|
33
|
+
|
34
|
+
from airflow.utils.context import Context
|
35
|
+
|
36
|
+
|
37
|
+
class HttpToGCSOperator(BaseOperator):
|
38
|
+
"""
|
39
|
+
Calls an endpoint on an HTTP system to execute an action and store the result in GCS.
|
40
|
+
|
41
|
+
:param http_conn_id: The :ref:`http connection<howto/connection:http>` to run
|
42
|
+
the operator against
|
43
|
+
:param endpoint: The relative part of the full url. (templated)
|
44
|
+
:param method: The HTTP method to use, default = "POST"
|
45
|
+
:param data: The data to pass. POST-data in POST/PUT and params
|
46
|
+
in the URL for a GET request. (templated)
|
47
|
+
:param headers: The HTTP headers to be added to the GET request
|
48
|
+
:param response_check: A check against the 'requests' response object.
|
49
|
+
The callable takes the response object as the first positional argument
|
50
|
+
and optionally any number of keyword arguments available in the context dictionary.
|
51
|
+
It should return True for 'pass' and False otherwise.
|
52
|
+
:param response_filter: A function allowing you to manipulate the response
|
53
|
+
text. e.g response_filter=lambda response: json.loads(response.text).
|
54
|
+
The callable takes the response object as the first positional argument
|
55
|
+
and optionally any number of keyword arguments available in the context dictionary.
|
56
|
+
:param extra_options: Extra options for the 'requests' library, see the
|
57
|
+
'requests' documentation (options to modify timeout, ssl, etc.)
|
58
|
+
:param log_response: Log the response (default: False)
|
59
|
+
:param auth_type: The auth type for the service
|
60
|
+
:param tcp_keep_alive: Enable TCP Keep Alive for the connection.
|
61
|
+
:param tcp_keep_alive_idle: The TCP Keep Alive Idle parameter (corresponds to ``socket.TCP_KEEPIDLE``).
|
62
|
+
:param tcp_keep_alive_count: The TCP Keep Alive count parameter (corresponds to ``socket.TCP_KEEPCNT``)
|
63
|
+
:param tcp_keep_alive_interval: The TCP Keep Alive interval parameter (corresponds to
|
64
|
+
``socket.TCP_KEEPINTVL``)
|
65
|
+
:param gcp_conn_id: The connection ID to use when fetching connection info.
|
66
|
+
:param impersonation_chain: Optional service account to impersonate using short-term credentials,
|
67
|
+
or chained list of accounts required to get the access_token of the last account in the list,
|
68
|
+
which will be impersonated in the request. If set as a string,
|
69
|
+
the account must grant the originating account the Service Account Token Creator IAM role.
|
70
|
+
If set as a sequence, the identities from the list must grant Service Account Token Creator IAM role to the directly preceding identity,
|
71
|
+
with first account from the list granting this role to the originating account.
|
72
|
+
:param bucket_name: The bucket to upload to.
|
73
|
+
:param object_name: The object name to set when uploading the file.
|
74
|
+
:param mime_type: The file mime type set when uploading the file.
|
75
|
+
:param gzip: Option to compress local file or file data for upload
|
76
|
+
:param encoding: bytes encoding for file data if provided as string
|
77
|
+
:param chunk_size: Blob chunk size.
|
78
|
+
:param timeout: Request timeout in seconds.
|
79
|
+
:param num_max_attempts: Number of attempts to try to upload the file.
|
80
|
+
:param metadata: The metadata to be uploaded with the file.
|
81
|
+
:param cache_contro: Cache-Control metadata field.
|
82
|
+
:param user_project: The identifier of the Google Cloud project to bill for the request. Required for Requester Pays buckets.
|
83
|
+
"""
|
84
|
+
|
85
|
+
template_fields: Sequence[str] = (
|
86
|
+
"http_conn_id",
|
87
|
+
"endpoint",
|
88
|
+
"data",
|
89
|
+
"headers",
|
90
|
+
"gcp_conn_id",
|
91
|
+
"bucket_name",
|
92
|
+
"object_name",
|
93
|
+
)
|
94
|
+
template_fields_renderers = {"headers": "json", "data": "py"}
|
95
|
+
template_ext: Sequence[str] = ()
|
96
|
+
ui_color = "#f4a460"
|
97
|
+
|
98
|
+
def __init__(
|
99
|
+
self,
|
100
|
+
*,
|
101
|
+
endpoint: str | None = None,
|
102
|
+
method: str = "GET",
|
103
|
+
data: Any = None,
|
104
|
+
headers: dict[str, str] | None = None,
|
105
|
+
extra_options: dict[str, Any] | None = None,
|
106
|
+
http_conn_id: str = "http_default",
|
107
|
+
log_response: bool = False,
|
108
|
+
auth_type: type[AuthBase] | None = None,
|
109
|
+
tcp_keep_alive: bool = True,
|
110
|
+
tcp_keep_alive_idle: int = 120,
|
111
|
+
tcp_keep_alive_count: int = 20,
|
112
|
+
tcp_keep_alive_interval: int = 30,
|
113
|
+
gcp_conn_id: str = "google_cloud_default",
|
114
|
+
impersonation_chain: str | Sequence[str] | None = None,
|
115
|
+
bucket_name: str,
|
116
|
+
object_name: str,
|
117
|
+
mime_type: str | None = None,
|
118
|
+
gzip: bool = False,
|
119
|
+
encoding: str | None = None,
|
120
|
+
chunk_size: int | None = None,
|
121
|
+
timeout: int | None = None,
|
122
|
+
num_max_attempts: int = 3,
|
123
|
+
metadata: dict | None = None,
|
124
|
+
cache_control: str | None = None,
|
125
|
+
user_project: str | None = None,
|
126
|
+
**kwargs,
|
127
|
+
):
|
128
|
+
super().__init__(**kwargs)
|
129
|
+
self.http_conn_id = http_conn_id
|
130
|
+
self.method = method
|
131
|
+
self.endpoint = endpoint
|
132
|
+
self.headers = headers or {}
|
133
|
+
self.data = data or {}
|
134
|
+
self.extra_options = extra_options or {}
|
135
|
+
self.log_response = log_response
|
136
|
+
self.auth_type = auth_type
|
137
|
+
self.tcp_keep_alive = tcp_keep_alive
|
138
|
+
self.tcp_keep_alive_idle = tcp_keep_alive_idle
|
139
|
+
self.tcp_keep_alive_count = tcp_keep_alive_count
|
140
|
+
self.tcp_keep_alive_interval = tcp_keep_alive_interval
|
141
|
+
self.gcp_conn_id = gcp_conn_id
|
142
|
+
self.impersonation_chain = impersonation_chain
|
143
|
+
self.bucket_name = bucket_name
|
144
|
+
self.object_name = object_name
|
145
|
+
self.mime_type = mime_type
|
146
|
+
self.gzip = gzip
|
147
|
+
self.encoding = encoding
|
148
|
+
self.chunk_size = chunk_size
|
149
|
+
self.timeout = timeout
|
150
|
+
self.num_max_attempts = num_max_attempts
|
151
|
+
self.metadata = metadata
|
152
|
+
self.cache_control = cache_control
|
153
|
+
self.user_project = user_project
|
154
|
+
|
155
|
+
@cached_property
|
156
|
+
def http_hook(self) -> HttpHook:
|
157
|
+
"""Create and return an HttpHook."""
|
158
|
+
return HttpHook(
|
159
|
+
self.method,
|
160
|
+
http_conn_id=self.http_conn_id,
|
161
|
+
auth_type=self.auth_type,
|
162
|
+
tcp_keep_alive=self.tcp_keep_alive,
|
163
|
+
tcp_keep_alive_idle=self.tcp_keep_alive_idle,
|
164
|
+
tcp_keep_alive_count=self.tcp_keep_alive_count,
|
165
|
+
tcp_keep_alive_interval=self.tcp_keep_alive_interval,
|
166
|
+
)
|
167
|
+
|
168
|
+
@cached_property
|
169
|
+
def gcs_hook(self) -> GCSHook:
|
170
|
+
"""Create and return an GCSHook."""
|
171
|
+
return GCSHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
|
172
|
+
|
173
|
+
def execute(self, context: Context):
|
174
|
+
self.log.info("Calling HTTP method")
|
175
|
+
response = self.http_hook.run(
|
176
|
+
endpoint=self.endpoint, data=self.data, headers=self.headers, extra_options=self.extra_options
|
177
|
+
)
|
178
|
+
|
179
|
+
self.log.info("Uploading to GCS")
|
180
|
+
self.gcs_hook.upload(
|
181
|
+
data=response.content,
|
182
|
+
bucket_name=self.bucket_name,
|
183
|
+
object_name=self.object_name,
|
184
|
+
mime_type=self.mime_type,
|
185
|
+
gzip=self.gzip,
|
186
|
+
encoding=self.encoding or response.encoding,
|
187
|
+
chunk_size=self.chunk_size,
|
188
|
+
timeout=self.timeout,
|
189
|
+
num_max_attempts=self.num_max_attempts,
|
190
|
+
metadata=self.metadata,
|
191
|
+
cache_control=self.cache_control,
|
192
|
+
user_project=self.user_project,
|
193
|
+
)
|
@@ -181,21 +181,27 @@ class S3ToGCSOperator(S3ListOperator):
|
|
181
181
|
'The destination Google Cloud Storage path must end with a slash "/" or be empty.'
|
182
182
|
)
|
183
183
|
|
184
|
-
def
|
185
|
-
self._check_inputs()
|
184
|
+
def _get_files(self, context: Context, gcs_hook: GCSHook) -> list[str]:
|
186
185
|
# use the super method to list all the files in an S3 bucket/key
|
187
186
|
s3_objects = super().execute(context)
|
188
187
|
|
188
|
+
if not self.replace:
|
189
|
+
s3_objects = self.exclude_existing_objects(s3_objects=s3_objects, gcs_hook=gcs_hook)
|
190
|
+
|
191
|
+
return s3_objects
|
192
|
+
|
193
|
+
def execute(self, context: Context):
|
194
|
+
self._check_inputs()
|
189
195
|
gcs_hook = GCSHook(
|
190
196
|
gcp_conn_id=self.gcp_conn_id,
|
191
197
|
impersonation_chain=self.google_impersonation_chain,
|
192
198
|
)
|
193
|
-
if not self.replace:
|
194
|
-
s3_objects = self.exclude_existing_objects(s3_objects=s3_objects, gcs_hook=gcs_hook)
|
195
|
-
|
196
199
|
s3_hook = S3Hook(aws_conn_id=self.aws_conn_id, verify=self.verify)
|
200
|
+
|
201
|
+
s3_objects = self._get_files(context, gcs_hook)
|
197
202
|
if not s3_objects:
|
198
203
|
self.log.info("In sync, no files needed to be uploaded to Google Cloud Storage")
|
204
|
+
|
199
205
|
elif self.deferrable:
|
200
206
|
self.transfer_files_async(s3_objects, gcs_hook, s3_hook)
|
201
207
|
else:
|
@@ -22,10 +22,12 @@ from typing import TYPE_CHECKING, Any, SupportsAbs
|
|
22
22
|
|
23
23
|
from aiohttp import ClientSession
|
24
24
|
from aiohttp.client_exceptions import ClientResponseError
|
25
|
+
from asgiref.sync import sync_to_async
|
25
26
|
|
26
27
|
from airflow.exceptions import AirflowException
|
27
28
|
from airflow.models.taskinstance import TaskInstance
|
28
29
|
from airflow.providers.google.cloud.hooks.bigquery import BigQueryAsyncHook, BigQueryTableAsyncHook
|
30
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
29
31
|
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
30
32
|
from airflow.utils.session import provide_session
|
31
33
|
from airflow.utils.state import TaskInstanceState
|
@@ -116,16 +118,41 @@ class BigQueryInsertJobTrigger(BaseTrigger):
|
|
116
118
|
)
|
117
119
|
return task_instance
|
118
120
|
|
119
|
-
def
|
121
|
+
async def get_task_state(self):
|
122
|
+
from airflow.sdk.execution_time.task_runner import RuntimeTaskInstance
|
123
|
+
|
124
|
+
task_states_response = await sync_to_async(RuntimeTaskInstance.get_task_states)(
|
125
|
+
dag_id=self.task_instance.dag_id,
|
126
|
+
task_ids=[self.task_instance.task_id],
|
127
|
+
run_ids=[self.task_instance.run_id],
|
128
|
+
map_index=self.task_instance.map_index,
|
129
|
+
)
|
130
|
+
try:
|
131
|
+
task_state = task_states_response[self.task_instance.run_id][self.task_instance.task_id]
|
132
|
+
except Exception:
|
133
|
+
raise AirflowException(
|
134
|
+
"TaskInstance with dag_id: %s, task_id: %s, run_id: %s and map_index: %s is not found",
|
135
|
+
self.task_instance.dag_id,
|
136
|
+
self.task_instance.task_id,
|
137
|
+
self.task_instance.run_id,
|
138
|
+
self.task_instance.map_index,
|
139
|
+
)
|
140
|
+
return task_state
|
141
|
+
|
142
|
+
async def safe_to_cancel(self) -> bool:
|
120
143
|
"""
|
121
144
|
Whether it is safe to cancel the external job which is being executed by this trigger.
|
122
145
|
|
123
146
|
This is to avoid the case that `asyncio.CancelledError` is called because the trigger itself is stopped.
|
124
147
|
Because in those cases, we should NOT cancel the external job.
|
125
148
|
"""
|
126
|
-
|
127
|
-
|
128
|
-
|
149
|
+
if AIRFLOW_V_3_0_PLUS:
|
150
|
+
task_state = await self.get_task_state()
|
151
|
+
else:
|
152
|
+
# Database query is needed to get the latest state of the task instance.
|
153
|
+
task_instance = self.get_task_instance() # type: ignore[call-arg]
|
154
|
+
task_state = task_instance.state
|
155
|
+
return task_state != TaskInstanceState.DEFERRED
|
129
156
|
|
130
157
|
async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override]
|
131
158
|
"""Get current job execution status and yields a TriggerEvent."""
|
@@ -155,7 +182,7 @@ class BigQueryInsertJobTrigger(BaseTrigger):
|
|
155
182
|
)
|
156
183
|
await asyncio.sleep(self.poll_interval)
|
157
184
|
except asyncio.CancelledError:
|
158
|
-
if self.job_id and self.cancel_on_kill and self.safe_to_cancel():
|
185
|
+
if self.job_id and self.cancel_on_kill and await self.safe_to_cancel():
|
159
186
|
self.log.info(
|
160
187
|
"The job is safe to cancel the as airflow TaskInstance is not in deferred state."
|
161
188
|
)
|
@@ -25,6 +25,7 @@ import time
|
|
25
25
|
from collections.abc import AsyncIterator, Sequence
|
26
26
|
from typing import TYPE_CHECKING, Any
|
27
27
|
|
28
|
+
from asgiref.sync import sync_to_async
|
28
29
|
from google.api_core.exceptions import NotFound
|
29
30
|
from google.cloud.dataproc_v1 import Batch, Cluster, ClusterStatus, JobStatus
|
30
31
|
|
@@ -33,6 +34,7 @@ from airflow.models.taskinstance import TaskInstance
|
|
33
34
|
from airflow.providers.google.cloud.hooks.dataproc import DataprocAsyncHook, DataprocHook
|
34
35
|
from airflow.providers.google.cloud.utils.dataproc import DataprocOperationType
|
35
36
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
37
|
+
from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
|
36
38
|
from airflow.triggers.base import BaseTrigger, TriggerEvent
|
37
39
|
from airflow.utils.session import provide_session
|
38
40
|
from airflow.utils.state import TaskInstanceState
|
@@ -141,16 +143,41 @@ class DataprocSubmitTrigger(DataprocBaseTrigger):
|
|
141
143
|
)
|
142
144
|
return task_instance
|
143
145
|
|
144
|
-
def
|
146
|
+
async def get_task_state(self):
|
147
|
+
from airflow.sdk.execution_time.task_runner import RuntimeTaskInstance
|
148
|
+
|
149
|
+
task_states_response = await sync_to_async(RuntimeTaskInstance.get_task_states)(
|
150
|
+
dag_id=self.task_instance.dag_id,
|
151
|
+
task_ids=[self.task_instance.task_id],
|
152
|
+
run_ids=[self.task_instance.run_id],
|
153
|
+
map_index=self.task_instance.map_index,
|
154
|
+
)
|
155
|
+
try:
|
156
|
+
task_state = task_states_response[self.task_instance.run_id][self.task_instance.task_id]
|
157
|
+
except Exception:
|
158
|
+
raise AirflowException(
|
159
|
+
"TaskInstance with dag_id: %s, task_id: %s, run_id: %s and map_index: %s is not found",
|
160
|
+
self.task_instance.dag_id,
|
161
|
+
self.task_instance.task_id,
|
162
|
+
self.task_instance.run_id,
|
163
|
+
self.task_instance.map_index,
|
164
|
+
)
|
165
|
+
return task_state
|
166
|
+
|
167
|
+
async def safe_to_cancel(self) -> bool:
|
145
168
|
"""
|
146
169
|
Whether it is safe to cancel the external job which is being executed by this trigger.
|
147
170
|
|
148
171
|
This is to avoid the case that `asyncio.CancelledError` is called because the trigger itself is stopped.
|
149
172
|
Because in those cases, we should NOT cancel the external job.
|
150
173
|
"""
|
151
|
-
|
152
|
-
|
153
|
-
|
174
|
+
if AIRFLOW_V_3_0_PLUS:
|
175
|
+
task_state = await self.get_task_state()
|
176
|
+
else:
|
177
|
+
# Database query is needed to get the latest state of the task instance.
|
178
|
+
task_instance = self.get_task_instance() # type: ignore[call-arg]
|
179
|
+
task_state = task_instance.state
|
180
|
+
return task_state != TaskInstanceState.DEFERRED
|
154
181
|
|
155
182
|
async def run(self):
|
156
183
|
try:
|
@@ -167,7 +194,7 @@ class DataprocSubmitTrigger(DataprocBaseTrigger):
|
|
167
194
|
except asyncio.CancelledError:
|
168
195
|
self.log.info("Task got cancelled.")
|
169
196
|
try:
|
170
|
-
if self.job_id and self.cancel_on_kill and self.safe_to_cancel():
|
197
|
+
if self.job_id and self.cancel_on_kill and await self.safe_to_cancel():
|
171
198
|
self.log.info(
|
172
199
|
"Cancelling the job as it is safe to do so. Note that the airflow TaskInstance is not"
|
173
200
|
" in deferred state."
|
@@ -243,16 +270,41 @@ class DataprocClusterTrigger(DataprocBaseTrigger):
|
|
243
270
|
)
|
244
271
|
return task_instance
|
245
272
|
|
246
|
-
def
|
273
|
+
async def get_task_state(self):
|
274
|
+
from airflow.sdk.execution_time.task_runner import RuntimeTaskInstance
|
275
|
+
|
276
|
+
task_states_response = await sync_to_async(RuntimeTaskInstance.get_task_states)(
|
277
|
+
dag_id=self.task_instance.dag_id,
|
278
|
+
task_ids=[self.task_instance.task_id],
|
279
|
+
run_ids=[self.task_instance.run_id],
|
280
|
+
map_index=self.task_instance.map_index,
|
281
|
+
)
|
282
|
+
try:
|
283
|
+
task_state = task_states_response[self.task_instance.run_id][self.task_instance.task_id]
|
284
|
+
except Exception:
|
285
|
+
raise AirflowException(
|
286
|
+
"TaskInstance with dag_id: %s, task_id: %s, run_id: %s and map_index: %s is not found",
|
287
|
+
self.task_instance.dag_id,
|
288
|
+
self.task_instance.task_id,
|
289
|
+
self.task_instance.run_id,
|
290
|
+
self.task_instance.map_index,
|
291
|
+
)
|
292
|
+
return task_state
|
293
|
+
|
294
|
+
async def safe_to_cancel(self) -> bool:
|
247
295
|
"""
|
248
296
|
Whether it is safe to cancel the external job which is being executed by this trigger.
|
249
297
|
|
250
298
|
This is to avoid the case that `asyncio.CancelledError` is called because the trigger itself is stopped.
|
251
299
|
Because in those cases, we should NOT cancel the external job.
|
252
300
|
"""
|
253
|
-
|
254
|
-
|
255
|
-
|
301
|
+
if AIRFLOW_V_3_0_PLUS:
|
302
|
+
task_state = await self.get_task_state()
|
303
|
+
else:
|
304
|
+
# Database query is needed to get the latest state of the task instance.
|
305
|
+
task_instance = self.get_task_instance() # type: ignore[call-arg]
|
306
|
+
task_state = task_instance.state
|
307
|
+
return task_state != TaskInstanceState.DEFERRED
|
256
308
|
|
257
309
|
async def run(self) -> AsyncIterator[TriggerEvent]:
|
258
310
|
try:
|
@@ -283,7 +335,7 @@ class DataprocClusterTrigger(DataprocBaseTrigger):
|
|
283
335
|
await asyncio.sleep(self.polling_interval_seconds)
|
284
336
|
except asyncio.CancelledError:
|
285
337
|
try:
|
286
|
-
if self.delete_on_error and self.safe_to_cancel():
|
338
|
+
if self.delete_on_error and await self.safe_to_cancel():
|
287
339
|
self.log.info(
|
288
340
|
"Deleting the cluster as it is safe to delete as the airflow TaskInstance is not in "
|
289
341
|
"deferred state."
|
@@ -1338,6 +1338,12 @@ def get_provider_info():
|
|
1338
1338
|
"python-module": "airflow.providers.google.cloud.transfers.azure_blob_to_gcs",
|
1339
1339
|
"how-to-guide": "/docs/apache-airflow-providers-google/operators/transfer/azure_blob_to_gcs.rst",
|
1340
1340
|
},
|
1341
|
+
{
|
1342
|
+
"source-integration-name": "Hypertext Transfer Protocol (HTTP)",
|
1343
|
+
"target-integration-name": "Google Cloud Storage (GCS)",
|
1344
|
+
"python-module": "airflow.providers.google.cloud.transfers.http_to_gcs",
|
1345
|
+
"how-to-guide": "/docs/apache-airflow-providers-google/operators/transfer/http_to_gcs.rst",
|
1346
|
+
},
|
1341
1347
|
],
|
1342
1348
|
"connection-types": [
|
1343
1349
|
{
|
@@ -1368,6 +1374,14 @@ def get_provider_info():
|
|
1368
1374
|
"hook-class-name": "airflow.providers.google.leveldb.hooks.leveldb.LevelDBHook",
|
1369
1375
|
"connection-type": "leveldb",
|
1370
1376
|
},
|
1377
|
+
{
|
1378
|
+
"hook-class-name": "airflow.providers.google.ads.hooks.ads.GoogleAdsHook",
|
1379
|
+
"connection-type": "google_ads",
|
1380
|
+
},
|
1381
|
+
{
|
1382
|
+
"hook-class-name": "airflow.providers.google.cloud.hooks.looker.LookerHook",
|
1383
|
+
"connection-type": "gcp_looker",
|
1384
|
+
},
|
1371
1385
|
],
|
1372
1386
|
"extra-links": [
|
1373
1387
|
"airflow.providers.google.cloud.links.alloy_db.AlloyDBBackupsLink",
|
@@ -1461,11 +1475,6 @@ def get_provider_info():
|
|
1461
1475
|
"airflow.providers.google.cloud.links.cloud_build.CloudBuildListLink",
|
1462
1476
|
"airflow.providers.google.cloud.links.cloud_build.CloudBuildTriggersListLink",
|
1463
1477
|
"airflow.providers.google.cloud.links.cloud_build.CloudBuildTriggerDetailsLink",
|
1464
|
-
"airflow.providers.google.cloud.links.automl.AutoMLDatasetLink",
|
1465
|
-
"airflow.providers.google.cloud.links.automl.AutoMLDatasetListLink",
|
1466
|
-
"airflow.providers.google.cloud.links.automl.AutoMLModelLink",
|
1467
|
-
"airflow.providers.google.cloud.links.automl.AutoMLModelTrainLink",
|
1468
|
-
"airflow.providers.google.cloud.links.automl.AutoMLModelPredictLink",
|
1469
1478
|
"airflow.providers.google.cloud.links.life_sciences.LifeSciencesLink",
|
1470
1479
|
"airflow.providers.google.cloud.links.cloud_functions.CloudFunctionsDetailsLink",
|
1471
1480
|
"airflow.providers.google.cloud.links.cloud_functions.CloudFunctionsListLink",
|
@@ -18,6 +18,8 @@
|
|
18
18
|
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
|
+
from typing import Any
|
22
|
+
|
21
23
|
from airflow.exceptions import AirflowException, AirflowOptionalProviderFeatureException
|
22
24
|
from airflow.hooks.base import BaseHook
|
23
25
|
|
@@ -46,6 +48,29 @@ class LevelDBHook(BaseHook):
|
|
46
48
|
conn_type = "leveldb"
|
47
49
|
hook_name = "LevelDB"
|
48
50
|
|
51
|
+
@classmethod
|
52
|
+
def get_connection_form_widgets(cls) -> dict[str, Any]:
|
53
|
+
"""Return connection widgets to add to LevelDB connection form."""
|
54
|
+
from flask_babel import lazy_gettext
|
55
|
+
from wtforms import BooleanField
|
56
|
+
|
57
|
+
return {
|
58
|
+
"create_if_missing": BooleanField(
|
59
|
+
lazy_gettext("Create a database if it does not exist"), default=False
|
60
|
+
),
|
61
|
+
"error_if_exists": BooleanField(
|
62
|
+
lazy_gettext("Raise an exception if the database already exists"), default=False
|
63
|
+
),
|
64
|
+
}
|
65
|
+
|
66
|
+
@classmethod
|
67
|
+
def get_ui_field_behaviour(cls) -> dict[str, Any]:
|
68
|
+
"""Return custom UI field behaviour for LevelDB connection."""
|
69
|
+
return {
|
70
|
+
"hidden_fields": ["login", "password", "schema", "port"],
|
71
|
+
"relabeling": {},
|
72
|
+
}
|
73
|
+
|
49
74
|
def __init__(self, leveldb_conn_id: str = default_conn_name):
|
50
75
|
super().__init__()
|
51
76
|
self.leveldb_conn_id = leveldb_conn_id
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: apache-airflow-providers-google
|
3
|
-
Version: 16.0.
|
3
|
+
Version: 16.0.0rc1
|
4
4
|
Summary: Provider package apache-airflow-providers-google for Apache Airflow
|
5
5
|
Keywords: airflow-provider,google,airflow,integration
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
@@ -20,9 +20,9 @@ Classifier: Programming Language :: Python :: 3.10
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.11
|
21
21
|
Classifier: Programming Language :: Python :: 3.12
|
22
22
|
Classifier: Topic :: System :: Monitoring
|
23
|
-
Requires-Dist: apache-airflow>=2.10.
|
24
|
-
Requires-Dist: apache-airflow-providers-common-compat>=1.4.
|
25
|
-
Requires-Dist: apache-airflow-providers-common-sql>=1.
|
23
|
+
Requires-Dist: apache-airflow>=2.10.0rc1
|
24
|
+
Requires-Dist: apache-airflow-providers-common-compat>=1.4.0rc1
|
25
|
+
Requires-Dist: apache-airflow-providers-common-sql>=1.27.0rc1
|
26
26
|
Requires-Dist: asgiref>=3.5.2
|
27
27
|
Requires-Dist: dill>=0.2.3
|
28
28
|
Requires-Dist: gcloud-aio-auth>=5.2.0
|
@@ -81,29 +81,29 @@ Requires-Dist: grpcio-gcp>=0.2.2
|
|
81
81
|
Requires-Dist: httpx>=0.25.0
|
82
82
|
Requires-Dist: looker-sdk>=22.4.0,!=24.18.0
|
83
83
|
Requires-Dist: pandas-gbq>=0.7.0
|
84
|
-
Requires-Dist: pandas>=2.1.2,<2.2
|
85
84
|
Requires-Dist: proto-plus>=1.19.6
|
86
85
|
Requires-Dist: pyarrow>=14.0.1
|
87
86
|
Requires-Dist: python-slugify>=7.0.0
|
88
87
|
Requires-Dist: PyOpenSSL>=23.0.0
|
89
88
|
Requires-Dist: sqlalchemy-bigquery>=1.2.1
|
90
89
|
Requires-Dist: sqlalchemy-spanner>=1.6.2
|
91
|
-
Requires-Dist: tenacity>=8.
|
90
|
+
Requires-Dist: tenacity>=8.3.0
|
92
91
|
Requires-Dist: immutabledict>=4.2.0
|
93
92
|
Requires-Dist: types-protobuf!=5.29.1.20250402,>=5.27.0
|
94
|
-
Requires-Dist: apache-airflow-providers-amazon>=2.6.
|
93
|
+
Requires-Dist: apache-airflow-providers-amazon>=2.6.0rc1 ; extra == "amazon"
|
95
94
|
Requires-Dist: apache-beam[gcp]>=2.53.0 ; extra == "apache-beam" and ( python_version < "3.12")
|
96
95
|
Requires-Dist: apache-beam[gcp]>=2.57.0 ; extra == "apache-beam" and ( python_version >= "3.12")
|
97
96
|
Requires-Dist: apache-airflow-providers-apache-cassandra ; extra == "apache-cassandra"
|
98
|
-
Requires-Dist: apache-airflow-providers-cncf-kubernetes>=10.1.
|
99
|
-
Requires-Dist: apache-airflow-providers-fab>=2.0.
|
100
|
-
Requires-Dist: apache-airflow-providers-facebook>=2.2.
|
97
|
+
Requires-Dist: apache-airflow-providers-cncf-kubernetes>=10.1.0rc1 ; extra == "cncf-kubernetes"
|
98
|
+
Requires-Dist: apache-airflow-providers-fab>=2.0.0rc1 ; extra == "fab"
|
99
|
+
Requires-Dist: apache-airflow-providers-facebook>=2.2.0rc1 ; extra == "facebook"
|
100
|
+
Requires-Dist: apache-airflow-providers-http ; extra == "http"
|
101
101
|
Requires-Dist: plyvel>=1.5.1 ; extra == "leveldb"
|
102
102
|
Requires-Dist: apache-airflow-providers-microsoft-azure ; extra == "microsoft-azure"
|
103
103
|
Requires-Dist: apache-airflow-providers-microsoft-mssql ; extra == "microsoft-mssql"
|
104
104
|
Requires-Dist: apache-airflow-providers-mysql ; extra == "mysql"
|
105
105
|
Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
|
106
|
-
Requires-Dist: apache-airflow-providers-oracle>=3.1.
|
106
|
+
Requires-Dist: apache-airflow-providers-oracle>=3.1.0rc1 ; extra == "oracle"
|
107
107
|
Requires-Dist: apache-airflow-providers-postgres ; extra == "postgres"
|
108
108
|
Requires-Dist: apache-airflow-providers-presto ; extra == "presto"
|
109
109
|
Requires-Dist: apache-airflow-providers-salesforce ; extra == "salesforce"
|
@@ -111,8 +111,8 @@ Requires-Dist: apache-airflow-providers-sftp ; extra == "sftp"
|
|
111
111
|
Requires-Dist: apache-airflow-providers-ssh ; extra == "ssh"
|
112
112
|
Requires-Dist: apache-airflow-providers-trino ; extra == "trino"
|
113
113
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
114
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-google/16.0.0/changelog.html
|
115
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-google/16.0.0
|
114
|
+
Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-google/16.0.0/changelog.html
|
115
|
+
Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-google/16.0.0
|
116
116
|
Project-URL: Mastodon, https://fosstodon.org/@airflow
|
117
117
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
118
118
|
Project-URL: Source Code, https://github.com/apache/airflow
|
@@ -123,6 +123,7 @@ Provides-Extra: apache-cassandra
|
|
123
123
|
Provides-Extra: cncf-kubernetes
|
124
124
|
Provides-Extra: fab
|
125
125
|
Provides-Extra: facebook
|
126
|
+
Provides-Extra: http
|
126
127
|
Provides-Extra: leveldb
|
127
128
|
Provides-Extra: microsoft-azure
|
128
129
|
Provides-Extra: microsoft-mssql
|
@@ -161,7 +162,7 @@ Provides-Extra: trino
|
|
161
162
|
|
162
163
|
Package ``apache-airflow-providers-google``
|
163
164
|
|
164
|
-
Release: ``
|
165
|
+
Release: ``16.0.0``
|
165
166
|
|
166
167
|
|
167
168
|
Google services including:
|
@@ -181,7 +182,7 @@ This is a provider package for ``google`` provider. All classes for this provide
|
|
181
182
|
are in ``airflow.providers.google`` python package.
|
182
183
|
|
183
184
|
You can find package information and changelog for the provider
|
184
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-google/
|
185
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-google/16.0.0/>`_.
|
185
186
|
|
186
187
|
Installation
|
187
188
|
------------
|
@@ -200,7 +201,7 @@ PIP package Version required
|
|
200
201
|
=========================================== ======================================
|
201
202
|
``apache-airflow`` ``>=2.10.0``
|
202
203
|
``apache-airflow-providers-common-compat`` ``>=1.4.0``
|
203
|
-
``apache-airflow-providers-common-sql`` ``>=1.
|
204
|
+
``apache-airflow-providers-common-sql`` ``>=1.27.0``
|
204
205
|
``asgiref`` ``>=3.5.2``
|
205
206
|
``dill`` ``>=0.2.3``
|
206
207
|
``gcloud-aio-auth`` ``>=5.2.0``
|
@@ -215,8 +216,8 @@ PIP package Version required
|
|
215
216
|
``google-auth-httplib2`` ``>=0.0.1``
|
216
217
|
``google-cloud-aiplatform[evaluation,ray]`` ``>=1.73.0; python_version < "3.12"``
|
217
218
|
``google-cloud-aiplatform[evaluation]`` ``>=1.73.0; python_version >= "3.12"``
|
218
|
-
``ray[default]``
|
219
|
-
``google-cloud-bigquery-storage``
|
219
|
+
``ray[default]`` ``>=2.42.0; python_version >= "3.12"``
|
220
|
+
``google-cloud-bigquery-storage`` ``>=2.31.0; python_version >= "3.12"``
|
220
221
|
``google-cloud-alloydb`` ``>=0.4.0``
|
221
222
|
``google-cloud-automl`` ``>=2.12.0``
|
222
223
|
``google-cloud-bigquery`` ``!=3.21.*,!=3.22.0,!=3.23.*,>=3.4.0``
|
@@ -259,16 +260,15 @@ PIP package Version required
|
|
259
260
|
``httpx`` ``>=0.25.0``
|
260
261
|
``looker-sdk`` ``>=22.4.0,!=24.18.0``
|
261
262
|
``pandas-gbq`` ``>=0.7.0``
|
262
|
-
``pandas`` ``>=2.1.2,<2.2``
|
263
263
|
``proto-plus`` ``>=1.19.6``
|
264
264
|
``pyarrow`` ``>=14.0.1``
|
265
265
|
``python-slugify`` ``>=7.0.0``
|
266
266
|
``PyOpenSSL`` ``>=23.0.0``
|
267
267
|
``sqlalchemy-bigquery`` ``>=1.2.1``
|
268
268
|
``sqlalchemy-spanner`` ``>=1.6.2``
|
269
|
-
``tenacity`` ``>=8.
|
269
|
+
``tenacity`` ``>=8.3.0``
|
270
270
|
``immutabledict`` ``>=4.2.0``
|
271
|
-
``types-protobuf``
|
271
|
+
``types-protobuf`` ``>=5.27.0,!=5.29.1.20250402``
|
272
272
|
=========================================== ======================================
|
273
273
|
|
274
274
|
Cross provider package dependencies
|
@@ -294,6 +294,7 @@ Dependent package
|
|
294
294
|
`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_ ``common.compat``
|
295
295
|
`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_ ``common.sql``
|
296
296
|
`apache-airflow-providers-facebook <https://airflow.apache.org/docs/apache-airflow-providers-facebook>`_ ``facebook``
|
297
|
+
`apache-airflow-providers-http <https://airflow.apache.org/docs/apache-airflow-providers-http>`_ ``http``
|
297
298
|
`apache-airflow-providers-microsoft-azure <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure>`_ ``microsoft.azure``
|
298
299
|
`apache-airflow-providers-microsoft-mssql <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-mssql>`_ ``microsoft.mssql``
|
299
300
|
`apache-airflow-providers-mysql <https://airflow.apache.org/docs/apache-airflow-providers-mysql>`_ ``mysql``
|
@@ -308,5 +309,5 @@ Dependent package
|
|
308
309
|
======================================================================================================================== ====================
|
309
310
|
|
310
311
|
The changelog for the provider package can be found in the
|
311
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-google/
|
312
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-google/16.0.0/changelog.html>`_.
|
312
313
|
|