apache-airflow-providers-yandex 3.9.0__py3-none-any.whl → 3.9.1rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/yandex/__init__.py +1 -1
- airflow/providers/yandex/get_provider_info.py +17 -3
- airflow/providers/yandex/hooks/yandex.py +6 -6
- airflow/providers/yandex/hooks/yq.py +112 -0
- airflow/providers/yandex/links/__init__.py +16 -0
- airflow/providers/yandex/links/yq.py +41 -0
- airflow/providers/yandex/operators/yandexcloud_dataproc.py +1 -1
- airflow/providers/yandex/operators/yq.py +92 -0
- airflow/providers/yandex/secrets/lockbox.py +43 -38
- {apache_airflow_providers_yandex-3.9.0.dist-info → apache_airflow_providers_yandex-3.9.1rc1.dist-info}/METADATA +17 -14
- apache_airflow_providers_yandex-3.9.1rc1.dist-info/RECORD +23 -0
- apache_airflow_providers_yandex-3.9.0.dist-info/RECORD +0 -19
- {apache_airflow_providers_yandex-3.9.0.dist-info → apache_airflow_providers_yandex-3.9.1rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_yandex-3.9.0.dist-info → apache_airflow_providers_yandex-3.9.1rc1.dist-info}/entry_points.txt +0 -0
@@ -28,8 +28,9 @@ def get_provider_info():
|
|
28
28
|
"name": "Yandex",
|
29
29
|
"description": "This package is for Yandex, including:\n\n - `Yandex.Cloud <https://cloud.yandex.com/>`__\n",
|
30
30
|
"state": "ready",
|
31
|
-
"source-date-epoch":
|
31
|
+
"source-date-epoch": 1712666581,
|
32
32
|
"versions": [
|
33
|
+
"3.9.1",
|
33
34
|
"3.9.0",
|
34
35
|
"3.8.0",
|
35
36
|
"3.7.1",
|
@@ -50,7 +51,7 @@ def get_provider_info():
|
|
50
51
|
"1.0.1",
|
51
52
|
"1.0.0",
|
52
53
|
],
|
53
|
-
"dependencies": ["apache-airflow>=2.6.0", "yandexcloud>=0.228.0"],
|
54
|
+
"dependencies": ["apache-airflow>=2.6.0", "yandexcloud>=0.228.0", "yandex-query-client>=0.1.2"],
|
54
55
|
"integrations": [
|
55
56
|
{
|
56
57
|
"integration-name": "Yandex.Cloud",
|
@@ -65,12 +66,23 @@ def get_provider_info():
|
|
65
66
|
"logo": "/integration-logos/yandex/Yandex-Cloud.png",
|
66
67
|
"tags": ["service"],
|
67
68
|
},
|
69
|
+
{
|
70
|
+
"integration-name": "Yandex.Cloud YQ",
|
71
|
+
"external-doc-url": "https://cloud.yandex.com/en/services/query",
|
72
|
+
"how-to-guide": ["/docs/apache-airflow-providers-yandex/operators.rst"],
|
73
|
+
"logo": "/integration-logos/yandex/Yandex-Cloud.png",
|
74
|
+
"tags": ["service"],
|
75
|
+
},
|
68
76
|
],
|
69
77
|
"operators": [
|
70
78
|
{
|
71
79
|
"integration-name": "Yandex.Cloud Dataproc",
|
72
80
|
"python-modules": ["airflow.providers.yandex.operators.yandexcloud_dataproc"],
|
73
|
-
}
|
81
|
+
},
|
82
|
+
{
|
83
|
+
"integration-name": "Yandex.Cloud YQ",
|
84
|
+
"python-modules": ["airflow.providers.yandex.operators.yq"],
|
85
|
+
},
|
74
86
|
],
|
75
87
|
"hooks": [
|
76
88
|
{"integration-name": "Yandex.Cloud", "python-modules": ["airflow.providers.yandex.hooks.yandex"]},
|
@@ -78,6 +90,7 @@ def get_provider_info():
|
|
78
90
|
"integration-name": "Yandex.Cloud Dataproc",
|
79
91
|
"python-modules": ["airflow.providers.yandex.hooks.yandexcloud_dataproc"],
|
80
92
|
},
|
93
|
+
{"integration-name": "Yandex.Cloud YQ", "python-modules": ["airflow.providers.yandex.hooks.yq"]},
|
81
94
|
],
|
82
95
|
"connection-types": [
|
83
96
|
{
|
@@ -86,6 +99,7 @@ def get_provider_info():
|
|
86
99
|
}
|
87
100
|
],
|
88
101
|
"secrets-backends": ["airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend"],
|
102
|
+
"extra-links": ["airflow.providers.yandex.links.yq.YQLink"],
|
89
103
|
"config": {
|
90
104
|
"yandex": {
|
91
105
|
"description": "This section contains settings for Yandex Cloud integration.",
|
@@ -79,14 +79,14 @@ class YandexCloudBaseHook(BaseHook):
|
|
79
79
|
"folder_id": StringField(
|
80
80
|
lazy_gettext("Default folder ID"),
|
81
81
|
widget=BS3TextFieldWidget(),
|
82
|
-
description="Optional.
|
83
|
-
"
|
82
|
+
description="Optional. "
|
83
|
+
"If specified, this ID will be used by default when creating nodes and clusters.",
|
84
84
|
),
|
85
85
|
"public_ssh_key": StringField(
|
86
86
|
lazy_gettext("Public SSH key"),
|
87
87
|
widget=BS3TextFieldWidget(),
|
88
|
-
description="Optional.
|
89
|
-
"
|
88
|
+
description="Optional. The key will be placed to all created Compute nodes, "
|
89
|
+
"allowing you to have a root shell there.",
|
90
90
|
),
|
91
91
|
"endpoint": StringField(
|
92
92
|
lazy_gettext("API endpoint"),
|
@@ -132,13 +132,13 @@ class YandexCloudBaseHook(BaseHook):
|
|
132
132
|
self.connection_id = yandex_conn_id or connection_id or default_conn_name
|
133
133
|
self.connection = self.get_connection(self.connection_id)
|
134
134
|
self.extras = self.connection.extra_dejson
|
135
|
-
credentials = get_credentials(
|
135
|
+
self.credentials = get_credentials(
|
136
136
|
oauth_token=self._get_field("oauth"),
|
137
137
|
service_account_json=self._get_field("service_account_json"),
|
138
138
|
service_account_json_path=self._get_field("service_account_json_path"),
|
139
139
|
)
|
140
140
|
sdk_config = self._get_endpoint()
|
141
|
-
self.sdk = yandexcloud.SDK(user_agent=provider_user_agent(), **sdk_config, **credentials)
|
141
|
+
self.sdk = yandexcloud.SDK(user_agent=provider_user_agent(), **sdk_config, **self.credentials)
|
142
142
|
self.default_folder_id = default_folder_id or self._get_field("folder_id")
|
143
143
|
self.default_public_ssh_key = default_public_ssh_key or self._get_field("public_ssh_key")
|
144
144
|
self.default_service_account_id = default_service_account_id or get_service_account_id(
|
@@ -0,0 +1,112 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
from __future__ import annotations
|
18
|
+
|
19
|
+
from datetime import timedelta
|
20
|
+
from typing import Any
|
21
|
+
|
22
|
+
import yandexcloud
|
23
|
+
import yandexcloud._auth_fabric as auth_fabric
|
24
|
+
from yandex.cloud.iam.v1.iam_token_service_pb2_grpc import IamTokenServiceStub
|
25
|
+
from yandex_query_client import YQHttpClient, YQHttpClientConfig
|
26
|
+
|
27
|
+
from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook
|
28
|
+
from airflow.providers.yandex.utils.user_agent import provider_user_agent
|
29
|
+
|
30
|
+
|
31
|
+
class YQHook(YandexCloudBaseHook):
|
32
|
+
"""A hook for Yandex Query."""
|
33
|
+
|
34
|
+
def __init__(self, *args, **kwargs) -> None:
|
35
|
+
super().__init__(*args, **kwargs)
|
36
|
+
|
37
|
+
config = YQHttpClientConfig(
|
38
|
+
token=self._get_iam_token(), project=self.default_folder_id, user_agent=provider_user_agent()
|
39
|
+
)
|
40
|
+
|
41
|
+
self.client: YQHttpClient = YQHttpClient(config=config)
|
42
|
+
|
43
|
+
def close(self):
|
44
|
+
"""Release all resources."""
|
45
|
+
self.client.close()
|
46
|
+
|
47
|
+
def create_query(self, query_text: str | None, name: str | None = None) -> str:
|
48
|
+
"""Create and run query.
|
49
|
+
|
50
|
+
:param query_text: SQL text.
|
51
|
+
:param name: name for the query
|
52
|
+
"""
|
53
|
+
return self.client.create_query(
|
54
|
+
name=name,
|
55
|
+
query_text=query_text,
|
56
|
+
)
|
57
|
+
|
58
|
+
def wait_results(self, query_id: str, execution_timeout: timedelta = timedelta(minutes=30)) -> Any:
|
59
|
+
"""Wait for query complete and get results.
|
60
|
+
|
61
|
+
:param query_id: ID of query.
|
62
|
+
:param execution_timeout: how long to wait for the query to complete.
|
63
|
+
"""
|
64
|
+
result_set_count = self.client.wait_query_to_succeed(
|
65
|
+
query_id, execution_timeout=execution_timeout, stop_on_timeout=True
|
66
|
+
)
|
67
|
+
|
68
|
+
return self.client.get_query_all_result_sets(query_id=query_id, result_set_count=result_set_count)
|
69
|
+
|
70
|
+
def stop_query(self, query_id: str) -> None:
|
71
|
+
"""Stop the query.
|
72
|
+
|
73
|
+
:param query_id: ID of the query.
|
74
|
+
"""
|
75
|
+
self.client.stop_query(query_id)
|
76
|
+
|
77
|
+
def get_query(self, query_id: str) -> Any:
|
78
|
+
"""Get query info.
|
79
|
+
|
80
|
+
:param query_id: ID of the query.
|
81
|
+
"""
|
82
|
+
return self.client.get_query(query_id)
|
83
|
+
|
84
|
+
def get_query_status(self, query_id: str) -> str:
|
85
|
+
"""Get status fo the query.
|
86
|
+
|
87
|
+
:param query_id: ID of query.
|
88
|
+
"""
|
89
|
+
return self.client.get_query_status(query_id)
|
90
|
+
|
91
|
+
def compose_query_web_link(self, query_id: str):
|
92
|
+
"""Compose web link to query in Yandex Query UI.
|
93
|
+
|
94
|
+
:param query_id: ID of query.
|
95
|
+
"""
|
96
|
+
return self.client.compose_query_web_link(query_id)
|
97
|
+
|
98
|
+
def _get_iam_token(self) -> str:
|
99
|
+
iam_token = self.credentials.get("token")
|
100
|
+
if iam_token is not None:
|
101
|
+
return iam_token
|
102
|
+
|
103
|
+
service_account_key = self.credentials.get("service_account_key")
|
104
|
+
# if service_account_key is None metadata server will be used
|
105
|
+
token_requester = auth_fabric.get_auth_token_requester(service_account_key=service_account_key)
|
106
|
+
|
107
|
+
if service_account_key is None:
|
108
|
+
return token_requester.get_token()
|
109
|
+
|
110
|
+
sdk = yandexcloud.SDK()
|
111
|
+
client = sdk.client(IamTokenServiceStub)
|
112
|
+
return client.Create(token_requester.get_token_request()).iam_token
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
from __future__ import annotations
|
18
|
+
|
19
|
+
from typing import TYPE_CHECKING
|
20
|
+
|
21
|
+
from airflow.models import BaseOperatorLink, XCom
|
22
|
+
|
23
|
+
if TYPE_CHECKING:
|
24
|
+
from airflow.models import BaseOperator
|
25
|
+
from airflow.models.taskinstancekey import TaskInstanceKey
|
26
|
+
from airflow.utils.context import Context
|
27
|
+
|
28
|
+
XCOM_WEBLINK_KEY = "web_link"
|
29
|
+
|
30
|
+
|
31
|
+
class YQLink(BaseOperatorLink):
|
32
|
+
"""Web link to query in Yandex Query UI."""
|
33
|
+
|
34
|
+
name = "Yandex Query"
|
35
|
+
|
36
|
+
def get_link(self, operator: BaseOperator, *, ti_key: TaskInstanceKey):
|
37
|
+
return XCom.get_value(key=XCOM_WEBLINK_KEY, ti_key=ti_key) or "https://yq.cloud.yandex.ru"
|
38
|
+
|
39
|
+
@staticmethod
|
40
|
+
def persist(context: Context, task_instance: BaseOperator, web_link: str) -> None:
|
41
|
+
task_instance.xcom_push(context, key=XCOM_WEBLINK_KEY, value=web_link)
|
@@ -97,7 +97,7 @@ class DataprocCreateClusterOperator(BaseOperator):
|
|
97
97
|
:param initialization_actions: Set of init-actions to run when cluster starts.
|
98
98
|
Docs: https://cloud.yandex.com/docs/data-proc/concepts/init-action
|
99
99
|
:param labels: Cluster labels as key:value pairs. No more than 64 per resource.
|
100
|
-
Docs: https://cloud.yandex.
|
100
|
+
Docs: https://cloud.yandex.com/docs/resource-manager/concepts/labels
|
101
101
|
"""
|
102
102
|
|
103
103
|
def __init__(
|
@@ -0,0 +1,92 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
from __future__ import annotations
|
18
|
+
|
19
|
+
from functools import cached_property
|
20
|
+
from typing import TYPE_CHECKING, Any, Sequence
|
21
|
+
|
22
|
+
from airflow.models import BaseOperator
|
23
|
+
from airflow.providers.yandex.hooks.yq import YQHook
|
24
|
+
from airflow.providers.yandex.links.yq import YQLink
|
25
|
+
|
26
|
+
if TYPE_CHECKING:
|
27
|
+
from airflow.utils.context import Context
|
28
|
+
|
29
|
+
|
30
|
+
class YQExecuteQueryOperator(BaseOperator):
|
31
|
+
"""
|
32
|
+
Executes sql code using Yandex Query service.
|
33
|
+
|
34
|
+
:param sql: the SQL code to be executed as a single string
|
35
|
+
:param name: name of the query in YandexQuery
|
36
|
+
:param folder_id: cloud folder id where to create query
|
37
|
+
:param yandex_conn_id: Airflow connection ID to get parameters from
|
38
|
+
"""
|
39
|
+
|
40
|
+
operator_extra_links = (YQLink(),)
|
41
|
+
template_fields: Sequence[str] = ("sql",)
|
42
|
+
template_fields_renderers = {"sql": "sql"}
|
43
|
+
template_ext: Sequence[str] = (".sql",)
|
44
|
+
ui_color = "#ededed"
|
45
|
+
|
46
|
+
def __init__(
|
47
|
+
self,
|
48
|
+
*,
|
49
|
+
name: str | None = None,
|
50
|
+
folder_id: str | None = None,
|
51
|
+
yandex_conn_id: str | None = None,
|
52
|
+
public_ssh_key: str | None = None,
|
53
|
+
service_account_id: str | None = None,
|
54
|
+
sql: str,
|
55
|
+
**kwargs,
|
56
|
+
) -> None:
|
57
|
+
super().__init__(**kwargs)
|
58
|
+
self.name = name
|
59
|
+
self.folder_id = folder_id
|
60
|
+
self.yandex_conn_id = yandex_conn_id
|
61
|
+
self.public_ssh_key = public_ssh_key
|
62
|
+
self.service_account_id = service_account_id
|
63
|
+
self.sql = sql
|
64
|
+
|
65
|
+
self.query_id: str | None = None
|
66
|
+
|
67
|
+
@cached_property
|
68
|
+
def hook(self) -> YQHook:
|
69
|
+
"""Get valid hook."""
|
70
|
+
return YQHook(
|
71
|
+
yandex_conn_id=self.yandex_conn_id,
|
72
|
+
default_folder_id=self.folder_id,
|
73
|
+
default_public_ssh_key=self.public_ssh_key,
|
74
|
+
default_service_account_id=self.service_account_id,
|
75
|
+
)
|
76
|
+
|
77
|
+
def execute(self, context: Context) -> Any:
|
78
|
+
self.query_id = self.hook.create_query(query_text=self.sql, name=self.name)
|
79
|
+
|
80
|
+
# pass to YQLink
|
81
|
+
web_link = self.hook.compose_query_web_link(self.query_id)
|
82
|
+
YQLink.persist(context, self, web_link)
|
83
|
+
|
84
|
+
results = self.hook.wait_results(self.query_id)
|
85
|
+
# forget query to avoid 'stop_query' in on_kill
|
86
|
+
self.query_id = None
|
87
|
+
return results
|
88
|
+
|
89
|
+
def on_kill(self) -> None:
|
90
|
+
if self.hook is not None and self.query_id is not None:
|
91
|
+
self.hook.stop_query(self.query_id)
|
92
|
+
self.hook.close()
|
@@ -15,6 +15,7 @@
|
|
15
15
|
# specific language governing permissions and limitations
|
16
16
|
# under the License.
|
17
17
|
"""Objects relating to sourcing secrets from Yandex Cloud Lockbox."""
|
18
|
+
|
18
19
|
from __future__ import annotations
|
19
20
|
|
20
21
|
from functools import cached_property
|
@@ -39,7 +40,7 @@ from airflow.utils.log.logging_mixin import LoggingMixin
|
|
39
40
|
|
40
41
|
class LockboxSecretBackend(BaseSecretsBackend, LoggingMixin):
|
41
42
|
"""
|
42
|
-
Retrieves
|
43
|
+
Retrieves connections or variables or configs from Yandex Lockbox.
|
43
44
|
|
44
45
|
Configurable via ``airflow.cfg`` like so:
|
45
46
|
|
@@ -60,7 +61,7 @@ class LockboxSecretBackend(BaseSecretsBackend, LoggingMixin):
|
|
60
61
|
the path ``airflow/config/sql_alchemy_conn``, the config with key ``sql_alchemy_conn`` would be
|
61
62
|
accessible.
|
62
63
|
|
63
|
-
|
64
|
+
If the prefix is empty, the requests will not be sent to Yandex Lockbox.
|
64
65
|
|
65
66
|
.. code-block:: ini
|
66
67
|
|
@@ -68,43 +69,49 @@ class LockboxSecretBackend(BaseSecretsBackend, LoggingMixin):
|
|
68
69
|
backend = airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend
|
69
70
|
backend_kwargs = {"yc_connection_id": "<connection_ID>", "folder_id": "<folder_ID>"}
|
70
71
|
|
71
|
-
You need to specify credentials or
|
72
|
-
|
72
|
+
You need to specify credentials or the ID of the ``yandexcloud`` connection to connect to Yandex Lockbox.
|
73
|
+
The credentials will be used with the following priority:
|
73
74
|
|
74
|
-
* OAuth
|
75
|
-
* Service
|
76
|
-
* Service
|
77
|
-
* Yandex Cloud
|
75
|
+
* OAuth token
|
76
|
+
* Service account key in JSON from file
|
77
|
+
* Service account key in JSON
|
78
|
+
* Yandex Cloud connection
|
78
79
|
|
79
|
-
If
|
80
|
+
If you do not specify any credentials,
|
81
|
+
the system will use the default connection ID:``yandexcloud_default``.
|
80
82
|
|
81
83
|
Also, you need to specify the Yandex Cloud folder ID to search for Yandex Lockbox secrets in.
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
84
|
+
If you do not specify folder ID, the requests will use the connection ``folder_id`` if specified.
|
85
|
+
|
86
|
+
:param yc_oauth_token: Specifies the user account OAuth token to connect to Yandex Lockbox.
|
87
|
+
The parameter value should look like ``y3_xx123``.
|
88
|
+
:param yc_sa_key_json: Specifies the service account key in JSON.
|
89
|
+
The parameter value should look like
|
90
|
+
``{"id": "...", "service_account_id": "...", "private_key": "..."}``.
|
91
|
+
:param yc_sa_key_json_path: Specifies the service account key in JSON file path.
|
92
|
+
The parameter value should look like ``/home/airflow/authorized_key.json``,
|
93
|
+
while the file content should have the following format:
|
94
|
+
``{"id": "...", "service_account_id": "...", "private_key": "..."}``.
|
95
|
+
:param yc_connection_id: Specifies the connection ID to connect to Yandex Lockbox.
|
96
|
+
The default value is ``yandexcloud_default``.
|
92
97
|
:param folder_id: Specifies the folder ID to search for Yandex Lockbox secrets in.
|
93
|
-
If set to None (null in JSON),
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
:param
|
106
|
-
|
107
|
-
|
98
|
+
If set to ``None`` (``null`` in JSON),
|
99
|
+
the requests will use the connection ``folder_id``, if specified.
|
100
|
+
:param connections_prefix: Specifies the prefix of the secret to read to get connections.
|
101
|
+
If set to ``None`` (``null`` in JSON),
|
102
|
+
the requests for connections will not be sent to Yandex Lockbox.
|
103
|
+
The default value is ``airflow/connections``.
|
104
|
+
:param variables_prefix: Specifies the prefix of the secret to read to get variables.
|
105
|
+
If set to ``None`` (``null`` in JSON), the requests for variables will not be sent to Yandex Lockbox.
|
106
|
+
The default value is ``airflow/variables``.
|
107
|
+
:param config_prefix: Specifies the prefix of the secret to read to get configurations.
|
108
|
+
If set to ``None`` (``null`` in JSON), the requests for variables will not be sent to Yandex Lockbox.
|
109
|
+
The default value is ``airflow/config``.
|
110
|
+
:param sep: Specifies the separator to concatenate ``secret_prefix`` and ``secret_id``.
|
111
|
+
The default value is ``/``.
|
112
|
+
:param endpoint: Specifies the API endpoint.
|
113
|
+
If set to ``None`` (``null`` in JSON), the requests will use the connection endpoint, if specified;
|
114
|
+
otherwise, they will use the default endpoint.
|
108
115
|
"""
|
109
116
|
|
110
117
|
def __init__(
|
@@ -128,10 +135,8 @@ class LockboxSecretBackend(BaseSecretsBackend, LoggingMixin):
|
|
128
135
|
self.yc_connection_id = None
|
129
136
|
if not any([yc_oauth_token, yc_sa_key_json, yc_sa_key_json_path]):
|
130
137
|
self.yc_connection_id = yc_connection_id or default_conn_name
|
131
|
-
|
132
|
-
|
133
|
-
yc_connection_id is None
|
134
|
-
), "yc_connection_id should not be used if other credentials are specified"
|
138
|
+
elif yc_connection_id is not None:
|
139
|
+
raise ValueError("`yc_connection_id` should not be used if other credentials are specified")
|
135
140
|
|
136
141
|
self.folder_id = folder_id
|
137
142
|
self.connections_prefix = connections_prefix.rstrip(sep) if connections_prefix is not None else None
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: apache-airflow-providers-yandex
|
3
|
-
Version: 3.9.
|
3
|
+
Version: 3.9.1rc1
|
4
4
|
Summary: Provider package apache-airflow-providers-yandex for Apache Airflow
|
5
5
|
Keywords: airflow-provider,yandex,airflow,integration
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
@@ -19,12 +19,14 @@ Classifier: Programming Language :: Python :: 3.8
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.9
|
20
20
|
Classifier: Programming Language :: Python :: 3.10
|
21
21
|
Classifier: Programming Language :: Python :: 3.11
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
22
23
|
Classifier: Topic :: System :: Monitoring
|
23
|
-
Requires-Dist: apache-airflow>=2.6.
|
24
|
+
Requires-Dist: apache-airflow>=2.6.0rc0
|
25
|
+
Requires-Dist: yandex-query-client>=0.1.2
|
24
26
|
Requires-Dist: yandexcloud>=0.228.0
|
25
27
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
26
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.9.
|
27
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.9.
|
28
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.9.1/changelog.html
|
29
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.9.1
|
28
30
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
29
31
|
Project-URL: Source Code, https://github.com/apache/airflow
|
30
32
|
Project-URL: Twitter, https://twitter.com/ApacheAirflow
|
@@ -74,7 +76,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
|
|
74
76
|
|
75
77
|
Package ``apache-airflow-providers-yandex``
|
76
78
|
|
77
|
-
Release: ``3.9.
|
79
|
+
Release: ``3.9.1.rc1``
|
78
80
|
|
79
81
|
|
80
82
|
This package is for Yandex, including:
|
@@ -89,7 +91,7 @@ This is a provider package for ``yandex`` provider. All classes for this provide
|
|
89
91
|
are in ``airflow.providers.yandex`` python package.
|
90
92
|
|
91
93
|
You can find package information and changelog for the provider
|
92
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.9.
|
94
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.9.1/>`_.
|
93
95
|
|
94
96
|
Installation
|
95
97
|
------------
|
@@ -98,17 +100,18 @@ You can install this package on top of an existing Airflow 2 installation (see `
|
|
98
100
|
for the minimum Airflow version supported) via
|
99
101
|
``pip install apache-airflow-providers-yandex``
|
100
102
|
|
101
|
-
The package supports the following python versions: 3.8,3.9,3.10,3.11
|
103
|
+
The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
|
102
104
|
|
103
105
|
Requirements
|
104
106
|
------------
|
105
107
|
|
106
|
-
|
107
|
-
PIP package
|
108
|
-
|
109
|
-
``apache-airflow``
|
110
|
-
``yandexcloud``
|
111
|
-
|
108
|
+
======================= ==================
|
109
|
+
PIP package Version required
|
110
|
+
======================= ==================
|
111
|
+
``apache-airflow`` ``>=2.6.0``
|
112
|
+
``yandexcloud`` ``>=0.228.0``
|
113
|
+
``yandex-query-client`` ``>=0.1.2``
|
114
|
+
======================= ==================
|
112
115
|
|
113
116
|
The changelog for the provider package can be found in the
|
114
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.9.
|
117
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.9.1/changelog.html>`_.
|
@@ -0,0 +1,23 @@
|
|
1
|
+
airflow/providers/yandex/LICENSE,sha256=ywUBpKZc7Jb96rVt5I3IDbg7dIJAbUSHkuoDcF3jbH4,13569
|
2
|
+
airflow/providers/yandex/__init__.py,sha256=Ih1LL6FBS1TK9rYKKWya372P9c93wNlHESnK8uPJoWU,1581
|
3
|
+
airflow/providers/yandex/get_provider_info.py,sha256=voh1E6seGyGmf6MUoaXIHpZKT6T9YGwom1MZ5LfuJhY,4684
|
4
|
+
airflow/providers/yandex/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
5
|
+
airflow/providers/yandex/hooks/yandex.py,sha256=xJMUzGo0sNpb5-LQvgq6jDxWHK3XkNzlpoeEELREeow,7097
|
6
|
+
airflow/providers/yandex/hooks/yandexcloud_dataproc.py,sha256=1UdqxDMI7uL6fNkG6oU6l2tFITF_nHXiV1VUgRqF7KY,1379
|
7
|
+
airflow/providers/yandex/hooks/yq.py,sha256=WpKL_Ic1BkqLU4JX8Lv8oPRk5RVXmHLMmL34AxTo_BU,3978
|
8
|
+
airflow/providers/yandex/links/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
9
|
+
airflow/providers/yandex/links/yq.py,sha256=jsy3liqQFk1eSSdK9YDbor0Epp7ng_q2ueVIwsD2i-8,1578
|
10
|
+
airflow/providers/yandex/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
11
|
+
airflow/providers/yandex/operators/yandexcloud_dataproc.py,sha256=0-g6AzP0KiQ6pJFLMFXHtB1YFaUPkl_4FQJZyH0ce9E,25957
|
12
|
+
airflow/providers/yandex/operators/yq.py,sha256=lGqbogakylV4s5D5movQRL4v3IU2Qt1JHH8ygo3Hd2Q,3223
|
13
|
+
airflow/providers/yandex/secrets/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
14
|
+
airflow/providers/yandex/secrets/lockbox.py,sha256=9Vi95RXd6VT36Rh1PVMUfFzm42oyrlMl77DoL9ivxVc,12161
|
15
|
+
airflow/providers/yandex/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
16
|
+
airflow/providers/yandex/utils/credentials.py,sha256=6McJIitAuTROJRUSKTdWChfcZ9o4WthP6GmBJ4uz-j8,3335
|
17
|
+
airflow/providers/yandex/utils/defaults.py,sha256=CXt75MhGJe8echoDpl1vR4VG5bEvYDDjIHmFqckDh2w,950
|
18
|
+
airflow/providers/yandex/utils/fields.py,sha256=1D8SDWH8h0djj5Hnk50w6BpPeNJyP-689Qfjpkr-yCg,1728
|
19
|
+
airflow/providers/yandex/utils/user_agent.py,sha256=AC-WEzhjxkgUYOy4LdX2-nnUZdMhKRRUCJ2_TjfNm6k,1839
|
20
|
+
apache_airflow_providers_yandex-3.9.1rc1.dist-info/entry_points.txt,sha256=ApXKRkvdgU2QNSQovjewC0b-LptwfBGBnJB3LTgBNx8,102
|
21
|
+
apache_airflow_providers_yandex-3.9.1rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
|
22
|
+
apache_airflow_providers_yandex-3.9.1rc1.dist-info/METADATA,sha256=nRCaC1ajv4tkEzWnkiu3ZNVX_c5XH46xFlb1HijwXHw,4913
|
23
|
+
apache_airflow_providers_yandex-3.9.1rc1.dist-info/RECORD,,
|
@@ -1,19 +0,0 @@
|
|
1
|
-
airflow/providers/yandex/LICENSE,sha256=ywUBpKZc7Jb96rVt5I3IDbg7dIJAbUSHkuoDcF3jbH4,13569
|
2
|
-
airflow/providers/yandex/__init__.py,sha256=zb_tpjfdegG0Y029vC5NyfR67uNuwiKJWAjRIx6lC_8,1581
|
3
|
-
airflow/providers/yandex/get_provider_info.py,sha256=mSgbC76sLo-WiSfCOhgXn6jD-MuJx1LU-3yz-ONUqIg,3930
|
4
|
-
airflow/providers/yandex/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
5
|
-
airflow/providers/yandex/hooks/yandex.py,sha256=_yPNJCrwAEubMkD4UQ7sF7YvKNhQtkbzM2mVIKhUnx0,7074
|
6
|
-
airflow/providers/yandex/hooks/yandexcloud_dataproc.py,sha256=1UdqxDMI7uL6fNkG6oU6l2tFITF_nHXiV1VUgRqF7KY,1379
|
7
|
-
airflow/providers/yandex/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
8
|
-
airflow/providers/yandex/operators/yandexcloud_dataproc.py,sha256=OUZgH3WdtgwRtaDITgHXQwHA4lL1LSOR6jOfHMS3f4w,25956
|
9
|
-
airflow/providers/yandex/secrets/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
10
|
-
airflow/providers/yandex/secrets/lockbox.py,sha256=k62nqyzrdC11gPYObD3tRvpr-W0NwXTdAfO12hJgZN4,11689
|
11
|
-
airflow/providers/yandex/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
12
|
-
airflow/providers/yandex/utils/credentials.py,sha256=6McJIitAuTROJRUSKTdWChfcZ9o4WthP6GmBJ4uz-j8,3335
|
13
|
-
airflow/providers/yandex/utils/defaults.py,sha256=CXt75MhGJe8echoDpl1vR4VG5bEvYDDjIHmFqckDh2w,950
|
14
|
-
airflow/providers/yandex/utils/fields.py,sha256=1D8SDWH8h0djj5Hnk50w6BpPeNJyP-689Qfjpkr-yCg,1728
|
15
|
-
airflow/providers/yandex/utils/user_agent.py,sha256=AC-WEzhjxkgUYOy4LdX2-nnUZdMhKRRUCJ2_TjfNm6k,1839
|
16
|
-
apache_airflow_providers_yandex-3.9.0.dist-info/entry_points.txt,sha256=ApXKRkvdgU2QNSQovjewC0b-LptwfBGBnJB3LTgBNx8,102
|
17
|
-
apache_airflow_providers_yandex-3.9.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
|
18
|
-
apache_airflow_providers_yandex-3.9.0.dist-info/METADATA,sha256=P7h-o9pbVIlhQS22BW1bU5aDvjs6mH4Ccu1uQ3YoEdY,4738
|
19
|
-
apache_airflow_providers_yandex-3.9.0.dist-info/RECORD,,
|
File without changes
|