nucliadb-utils 5.0.0.post899__py3-none-any.whl → 5.0.0.post906__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nucliadb_utils/aiopynecone/client.py +34 -12
- nucliadb_utils/indexing.py +15 -52
- nucliadb_utils/tests/azure.py +28 -15
- nucliadb_utils/tests/fixtures.py +11 -11
- nucliadb_utils/tests/gcs.py +39 -15
- nucliadb_utils/tests/local.py +22 -9
- nucliadb_utils/tests/s3.py +47 -15
- nucliadb_utils/utilities.py +1 -1
- {nucliadb_utils-5.0.0.post899.dist-info → nucliadb_utils-5.0.0.post906.dist-info}/METADATA +3 -3
- {nucliadb_utils-5.0.0.post899.dist-info → nucliadb_utils-5.0.0.post906.dist-info}/RECORD +13 -13
- {nucliadb_utils-5.0.0.post899.dist-info → nucliadb_utils-5.0.0.post906.dist-info}/WHEEL +0 -0
- {nucliadb_utils-5.0.0.post899.dist-info → nucliadb_utils-5.0.0.post906.dist-info}/top_level.txt +0 -0
- {nucliadb_utils-5.0.0.post899.dist-info → nucliadb_utils-5.0.0.post906.dist-info}/zip-safe +0 -0
@@ -60,6 +60,10 @@ INDEX_HOST_BASE_URL = "https://{index_host}/"
|
|
60
60
|
BASE_API_HEADERS = {
|
61
61
|
"Content-Type": "application/json",
|
62
62
|
"Accept": "application/json",
|
63
|
+
# This is needed so that it is easier for Pinecone to track which api requests
|
64
|
+
# are coming from the Nuclia integration:
|
65
|
+
# https://docs.pinecone.io/integrations/build-integration/attribute-usage-to-your-integration
|
66
|
+
"User-Agent": "source_tag=nuclia",
|
63
67
|
}
|
64
68
|
MEGA_BYTE = 1024 * 1024
|
65
69
|
MAX_UPSERT_PAYLOAD_SIZE = 2 * MEGA_BYTE
|
@@ -73,13 +77,6 @@ RETRIABLE_EXCEPTIONS = (
|
|
73
77
|
httpx.NetworkError,
|
74
78
|
)
|
75
79
|
|
76
|
-
backoff_handler = backoff.on_exception(
|
77
|
-
backoff.expo,
|
78
|
-
RETRIABLE_EXCEPTIONS,
|
79
|
-
jitter=backoff.random_jitter,
|
80
|
-
max_tries=4,
|
81
|
-
)
|
82
|
-
|
83
80
|
|
84
81
|
class ControlPlane:
|
85
82
|
"""
|
@@ -154,7 +151,12 @@ class DataPlane:
|
|
154
151
|
def _get_request_timeout(self, timeout: Optional[float] = None) -> Optional[float]:
|
155
152
|
return timeout or self.client_timeout
|
156
153
|
|
157
|
-
@
|
154
|
+
@backoff.on_exception(
|
155
|
+
backoff.expo,
|
156
|
+
RETRIABLE_EXCEPTIONS,
|
157
|
+
jitter=backoff.random_jitter,
|
158
|
+
max_tries=4,
|
159
|
+
)
|
158
160
|
@pinecone_observer.wrap({"type": "upsert"})
|
159
161
|
async def upsert(self, vectors: list[Vector], timeout: Optional[float] = None) -> None:
|
160
162
|
"""
|
@@ -231,7 +233,12 @@ class DataPlane:
|
|
231
233
|
|
232
234
|
await asyncio.gather(*tasks)
|
233
235
|
|
234
|
-
@
|
236
|
+
@backoff.on_exception(
|
237
|
+
backoff.expo,
|
238
|
+
RETRIABLE_EXCEPTIONS,
|
239
|
+
jitter=backoff.random_jitter,
|
240
|
+
max_tries=4,
|
241
|
+
)
|
235
242
|
@pinecone_observer.wrap({"type": "delete"})
|
236
243
|
async def delete(self, ids: list[str], timeout: Optional[float] = None) -> None:
|
237
244
|
"""
|
@@ -256,7 +263,12 @@ class DataPlane:
|
|
256
263
|
response = await self.http_session.post("/vectors/delete", **post_kwargs)
|
257
264
|
raise_for_status("delete", response)
|
258
265
|
|
259
|
-
@
|
266
|
+
@backoff.on_exception(
|
267
|
+
backoff.expo,
|
268
|
+
RETRIABLE_EXCEPTIONS,
|
269
|
+
jitter=backoff.random_jitter,
|
270
|
+
max_tries=4,
|
271
|
+
)
|
260
272
|
@pinecone_observer.wrap({"type": "list_page"})
|
261
273
|
async def list_page(
|
262
274
|
self,
|
@@ -324,7 +336,12 @@ class DataPlane:
|
|
324
336
|
break
|
325
337
|
pagination_token = response.pagination.next
|
326
338
|
|
327
|
-
@
|
339
|
+
@backoff.on_exception(
|
340
|
+
backoff.expo,
|
341
|
+
RETRIABLE_EXCEPTIONS,
|
342
|
+
jitter=backoff.random_jitter,
|
343
|
+
max_tries=4,
|
344
|
+
)
|
328
345
|
@pinecone_observer.wrap({"type": "delete_all"})
|
329
346
|
async def delete_all(self, timeout: Optional[float] = None):
|
330
347
|
"""
|
@@ -383,7 +400,12 @@ class DataPlane:
|
|
383
400
|
|
384
401
|
await asyncio.gather(*tasks)
|
385
402
|
|
386
|
-
@
|
403
|
+
@backoff.on_exception(
|
404
|
+
backoff.expo,
|
405
|
+
RETRIABLE_EXCEPTIONS,
|
406
|
+
jitter=backoff.random_jitter,
|
407
|
+
max_tries=4,
|
408
|
+
)
|
387
409
|
@pinecone_observer.wrap({"type": "query"})
|
388
410
|
async def query(
|
389
411
|
self,
|
nucliadb_utils/indexing.py
CHANGED
@@ -17,84 +17,47 @@
|
|
17
17
|
# You should have received a copy of the GNU Affero General Public License
|
18
18
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
19
|
#
|
20
|
-
from typing import
|
21
|
-
|
22
|
-
import nats
|
23
|
-
from nats.aio.client import Client
|
24
|
-
from nats.js.client import JetStreamContext
|
20
|
+
from typing import List, Optional, Tuple
|
25
21
|
|
26
22
|
from nucliadb_protos.nodewriter_pb2 import IndexMessage
|
27
|
-
from nucliadb_telemetry.jetstream import JetStreamContextTelemetry
|
28
23
|
from nucliadb_utils import const, logger
|
29
|
-
from nucliadb_utils.nats import
|
24
|
+
from nucliadb_utils.nats import NatsConnectionManager
|
30
25
|
|
31
26
|
|
32
27
|
class IndexingUtility:
|
33
|
-
nc: Optional[Client] = None
|
34
|
-
js: Optional[Union[JetStreamContext, JetStreamContextTelemetry]] = None
|
35
|
-
|
36
28
|
def __init__(
|
37
29
|
self,
|
38
30
|
nats_servers: List[str],
|
39
31
|
nats_creds: Optional[str] = None,
|
40
32
|
dummy: bool = False,
|
41
33
|
):
|
42
|
-
self.nats_creds = nats_creds
|
43
|
-
self.nats_servers = nats_servers
|
44
34
|
self.dummy = dummy
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
async def error_cb(self, e):
|
55
|
-
logger.error(
|
56
|
-
"There was an error connecting to NATS indexing: {}".format(e),
|
57
|
-
exc_info=True,
|
58
|
-
)
|
59
|
-
|
60
|
-
async def closed_cb(self):
|
61
|
-
logger.info("Connection is closed on NATS")
|
35
|
+
if dummy:
|
36
|
+
self._calls: List[Tuple[str, IndexMessage]] = []
|
37
|
+
else:
|
38
|
+
self.nats_connection_manager = NatsConnectionManager(
|
39
|
+
service_name="IndexingUtility",
|
40
|
+
nats_servers=nats_servers,
|
41
|
+
nats_creds=nats_creds,
|
42
|
+
)
|
62
43
|
|
63
44
|
async def initialize(self, service_name: Optional[str] = None):
|
64
45
|
if self.dummy:
|
65
46
|
return
|
66
|
-
|
67
|
-
options: Dict[str, Any] = {
|
68
|
-
"error_cb": self.error_cb,
|
69
|
-
"closed_cb": self.closed_cb,
|
70
|
-
"reconnected_cb": self.reconnected_cb,
|
71
|
-
}
|
72
|
-
|
73
|
-
if self.nats_creds:
|
74
|
-
options["user_credentials"] = self.nats_creds
|
75
|
-
|
76
|
-
if len(self.nats_servers) > 0:
|
77
|
-
options["servers"] = self.nats_servers
|
78
|
-
|
79
|
-
self.nc = await nats.connect(**options)
|
80
|
-
self.js = get_traced_jetstream(self.nc, service_name or "nucliadb_indexing")
|
47
|
+
await self.nats_connection_manager.initialize()
|
81
48
|
|
82
49
|
async def finalize(self):
|
83
|
-
if self.
|
84
|
-
|
85
|
-
|
86
|
-
self.nc = None
|
50
|
+
if self.dummy:
|
51
|
+
return
|
52
|
+
await self.nats_connection_manager.finalize()
|
87
53
|
|
88
54
|
async def index(self, writer: IndexMessage, node: str) -> int:
|
89
55
|
if self.dummy:
|
90
56
|
self._calls.append((node, writer))
|
91
57
|
return 0
|
92
58
|
|
93
|
-
if self.js is None:
|
94
|
-
raise AttributeError()
|
95
|
-
|
96
59
|
subject = const.Streams.INDEX.subject.format(node=node)
|
97
|
-
res = await self.js.publish(subject, writer.SerializeToString())
|
60
|
+
res = await self.nats_connection_manager.js.publish(subject, writer.SerializeToString())
|
98
61
|
logger.info(
|
99
62
|
f" - Pushed message to index {subject}. shard: {writer.shard}, txid: {writer.txid} seqid: {res.seq}" # noqa
|
100
63
|
)
|
nucliadb_utils/tests/azure.py
CHANGED
@@ -17,16 +17,17 @@
|
|
17
17
|
# You should have received a copy of the GNU Affero General Public License
|
18
18
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
19
|
#
|
20
|
+
from contextlib import ExitStack
|
20
21
|
from dataclasses import dataclass
|
21
|
-
from typing import Generator
|
22
|
+
from typing import Any, Generator
|
23
|
+
from unittest.mock import patch
|
22
24
|
|
23
25
|
import pytest
|
24
|
-
from pytest_docker_fixtures import images # type: ignore
|
25
|
-
from pytest_docker_fixtures.containers._base import BaseImage # type: ignore
|
26
|
+
from pytest_docker_fixtures import images # type: ignore # type: ignore
|
27
|
+
from pytest_docker_fixtures.containers._base import BaseImage # type: ignore # type: ignore
|
26
28
|
|
29
|
+
from nucliadb_utils.settings import FileBackendConfig, storage_settings
|
27
30
|
from nucliadb_utils.storages.azure import AzureStorage
|
28
|
-
from nucliadb_utils.store import MAIN
|
29
|
-
from nucliadb_utils.utilities import Utility
|
30
31
|
|
31
32
|
images.settings["azurite"] = {
|
32
33
|
"image": "mcr.microsoft.com/azure-storage/azurite",
|
@@ -107,16 +108,28 @@ def azurite() -> Generator[AzuriteFixture, None, None]:
|
|
107
108
|
|
108
109
|
|
109
110
|
@pytest.fixture(scope="function")
|
110
|
-
|
111
|
+
def azure_storage_settings(azurite: AzuriteFixture) -> dict[str, Any]:
|
112
|
+
settings = {
|
113
|
+
"file_backend": FileBackendConfig.AZURE,
|
114
|
+
"azure_account_url": azurite.account_url,
|
115
|
+
"azure_connection_string": azurite.connection_string,
|
116
|
+
}
|
117
|
+
with ExitStack() as stack:
|
118
|
+
for key, value in settings.items():
|
119
|
+
context = patch.object(storage_settings, key, value)
|
120
|
+
stack.enter_context(context)
|
121
|
+
|
122
|
+
yield settings
|
123
|
+
|
124
|
+
|
125
|
+
@pytest.fixture(scope="function")
|
126
|
+
async def azure_storage(azurite, azure_storage_settings: dict[str, Any]):
|
127
|
+
assert storage_settings.azure_account_url is not None
|
128
|
+
|
111
129
|
storage = AzureStorage(
|
112
|
-
account_url=
|
113
|
-
connection_string=
|
130
|
+
account_url=storage_settings.azure_account_url,
|
131
|
+
connection_string=storage_settings.azure_connection_string,
|
114
132
|
)
|
115
|
-
MAIN[Utility.STORAGE] = storage
|
116
133
|
await storage.initialize()
|
117
|
-
|
118
|
-
|
119
|
-
finally:
|
120
|
-
await storage.finalize()
|
121
|
-
if Utility.STORAGE in MAIN:
|
122
|
-
del MAIN[Utility.STORAGE]
|
134
|
+
yield storage
|
135
|
+
await storage.finalize()
|
nucliadb_utils/tests/fixtures.py
CHANGED
@@ -22,6 +22,8 @@ import os
|
|
22
22
|
import pytest
|
23
23
|
from pytest_lazy_fixtures import lazy_fixture
|
24
24
|
|
25
|
+
from nucliadb_utils.utilities import Utility, clean_utility, set_utility
|
26
|
+
|
25
27
|
|
26
28
|
@pytest.fixture(scope="function")
|
27
29
|
def onprem_nucliadb():
|
@@ -47,17 +49,10 @@ def get_testing_storage_backend(default="gcs"):
|
|
47
49
|
return os.environ.get("TESTING_STORAGE_BACKEND", default)
|
48
50
|
|
49
51
|
|
50
|
-
def lazy_storage_fixture():
|
52
|
+
def lazy_storage_fixture(default="gcs"):
|
51
53
|
backend = get_testing_storage_backend()
|
52
|
-
|
53
|
-
|
54
|
-
elif backend == "s3":
|
55
|
-
return [lazy_fixture.lf("s3_storage")]
|
56
|
-
elif backend == "local":
|
57
|
-
return [lazy_fixture.lf("local_storage")]
|
58
|
-
else:
|
59
|
-
print(f"Unknown storage backend {backend}, using gcs")
|
60
|
-
return [lazy_fixture.lf("gcs_storage")]
|
54
|
+
fixture_name = f"{backend}_storage"
|
55
|
+
return [lazy_fixture.lf(fixture_name)]
|
61
56
|
|
62
57
|
|
63
58
|
@pytest.fixture(scope="function", params=lazy_storage_fixture())
|
@@ -65,4 +60,9 @@ async def storage(request):
|
|
65
60
|
"""
|
66
61
|
Generic storage fixture that allows us to run the same tests for different storage backends.
|
67
62
|
"""
|
68
|
-
|
63
|
+
storage_driver = request.param
|
64
|
+
set_utility(Utility.STORAGE, storage_driver)
|
65
|
+
|
66
|
+
yield storage_driver
|
67
|
+
|
68
|
+
clean_utility(Utility.STORAGE)
|
nucliadb_utils/tests/gcs.py
CHANGED
@@ -19,18 +19,20 @@
|
|
19
19
|
#
|
20
20
|
import re
|
21
21
|
from concurrent.futures.thread import ThreadPoolExecutor
|
22
|
-
from
|
22
|
+
from contextlib import ExitStack
|
23
|
+
from typing import Any, Optional
|
24
|
+
from unittest.mock import patch
|
23
25
|
|
24
|
-
import docker # type: ignore
|
26
|
+
import docker # type: ignore # type: ignore
|
25
27
|
import pytest
|
26
28
|
import requests
|
27
29
|
from pytest_docker_fixtures import images # type: ignore
|
28
30
|
from pytest_docker_fixtures.containers._base import BaseImage # type: ignore
|
29
31
|
|
32
|
+
from nucliadb_utils.settings import FileBackendConfig, storage_settings
|
30
33
|
from nucliadb_utils.storages.gcs import GCSStorage
|
31
|
-
from nucliadb_utils.
|
34
|
+
from nucliadb_utils.storages.settings import settings as extended_storage_settings
|
32
35
|
from nucliadb_utils.tests import free_port
|
33
|
-
from nucliadb_utils.utilities import Utility
|
34
36
|
|
35
37
|
# IMPORTANT!
|
36
38
|
# Without this, tests running in a remote docker host won't work
|
@@ -77,20 +79,42 @@ def gcs():
|
|
77
79
|
|
78
80
|
|
79
81
|
@pytest.fixture(scope="function")
|
80
|
-
|
82
|
+
def gcs_storage_settings(gcs) -> dict[str, Any]:
|
83
|
+
settings = {
|
84
|
+
"file_backend": FileBackendConfig.GCS,
|
85
|
+
"gcs_endpoint_url": gcs,
|
86
|
+
"gcs_base64_creds": None,
|
87
|
+
"gcs_bucket": "test_{kbid}",
|
88
|
+
"gcs_location": "location",
|
89
|
+
}
|
90
|
+
extended_settings = {
|
91
|
+
"gcs_deadletter_bucket": "deadletters",
|
92
|
+
"gcs_indexing_bucket": "indexing",
|
93
|
+
}
|
94
|
+
with ExitStack() as stack:
|
95
|
+
for key, value in settings.items():
|
96
|
+
context = patch.object(storage_settings, key, value)
|
97
|
+
stack.enter_context(context)
|
98
|
+
for key, value in extended_settings.items():
|
99
|
+
context = patch.object(extended_storage_settings, key, value)
|
100
|
+
stack.enter_context(context)
|
101
|
+
|
102
|
+
yield settings | extended_settings
|
103
|
+
|
104
|
+
|
105
|
+
@pytest.fixture(scope="function")
|
106
|
+
async def gcs_storage(gcs, gcs_storage_settings: dict[str, Any]):
|
81
107
|
storage = GCSStorage(
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
108
|
+
url=storage_settings.gcs_endpoint_url,
|
109
|
+
account_credentials=storage_settings.gcs_base64_creds,
|
110
|
+
bucket=storage_settings.gcs_bucket,
|
111
|
+
location=storage_settings.gcs_location,
|
112
|
+
project=storage_settings.gcs_project,
|
86
113
|
executor=ThreadPoolExecutor(1),
|
87
|
-
deadletter_bucket=
|
88
|
-
indexing_bucket=
|
89
|
-
labels=
|
90
|
-
url=gcs,
|
114
|
+
deadletter_bucket=extended_storage_settings.gcs_deadletter_bucket,
|
115
|
+
indexing_bucket=extended_storage_settings.gcs_indexing_bucket,
|
116
|
+
labels=storage_settings.gcs_bucket_labels,
|
91
117
|
)
|
92
|
-
MAIN[Utility.STORAGE] = storage
|
93
118
|
await storage.initialize()
|
94
119
|
yield storage
|
95
120
|
await storage.finalize()
|
96
|
-
MAIN.pop(Utility.STORAGE, None)
|
nucliadb_utils/tests/local.py
CHANGED
@@ -17,23 +17,36 @@
|
|
17
17
|
# You should have received a copy of the GNU Affero General Public License
|
18
18
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
19
|
#
|
20
|
-
import
|
20
|
+
from contextlib import ExitStack
|
21
|
+
from pathlib import Path
|
22
|
+
from typing import Any
|
23
|
+
from unittest.mock import patch
|
21
24
|
|
22
25
|
import pytest
|
23
26
|
|
27
|
+
from nucliadb_utils.settings import FileBackendConfig, storage_settings
|
24
28
|
from nucliadb_utils.storages.local import LocalStorage
|
25
|
-
from nucliadb_utils.store import MAIN
|
26
|
-
from nucliadb_utils.utilities import Utility
|
27
29
|
|
28
30
|
|
29
31
|
@pytest.fixture(scope="function")
|
30
|
-
|
31
|
-
|
32
|
-
|
32
|
+
def local_storage_settings(tmp_path: Path) -> dict[str, Any]:
|
33
|
+
settings = {
|
34
|
+
"file_backend": FileBackendConfig.LOCAL,
|
35
|
+
"local_files": str((tmp_path / "blob").absolute()),
|
36
|
+
}
|
37
|
+
with ExitStack() as stack:
|
38
|
+
for key, value in settings.items():
|
39
|
+
context = patch.object(storage_settings, key, value)
|
40
|
+
stack.enter_context(context)
|
33
41
|
|
34
|
-
|
42
|
+
yield settings
|
43
|
+
|
44
|
+
|
45
|
+
@pytest.fixture(scope="function")
|
46
|
+
async def local_storage(local_storage_settings: dict[str, Any]):
|
47
|
+
assert storage_settings.local_files is not None
|
48
|
+
|
49
|
+
storage = LocalStorage(local_testing_files=storage_settings.local_files)
|
35
50
|
await storage.initialize()
|
36
51
|
yield storage
|
37
52
|
await storage.finalize()
|
38
|
-
folder.cleanup()
|
39
|
-
MAIN.pop(Utility.STORAGE, None)
|
nucliadb_utils/tests/s3.py
CHANGED
@@ -17,14 +17,18 @@
|
|
17
17
|
# You should have received a copy of the GNU Affero General Public License
|
18
18
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
19
|
#
|
20
|
+
from contextlib import ExitStack
|
21
|
+
from typing import Any
|
22
|
+
from unittest.mock import patch
|
23
|
+
|
20
24
|
import pytest
|
21
25
|
import requests
|
22
26
|
from pytest_docker_fixtures import images # type: ignore
|
23
27
|
from pytest_docker_fixtures.containers._base import BaseImage # type: ignore
|
24
28
|
|
29
|
+
from nucliadb_utils.settings import FileBackendConfig, storage_settings
|
25
30
|
from nucliadb_utils.storages.s3 import S3Storage
|
26
|
-
from nucliadb_utils.
|
27
|
-
from nucliadb_utils.utilities import Utility
|
31
|
+
from nucliadb_utils.storages.settings import settings as extended_storage_settings
|
28
32
|
|
29
33
|
images.settings["s3"] = {
|
30
34
|
"image": "localstack/localstack",
|
@@ -58,21 +62,49 @@ def s3():
|
|
58
62
|
|
59
63
|
|
60
64
|
@pytest.fixture(scope="function")
|
61
|
-
async def
|
65
|
+
async def s3_storage_settings(s3) -> dict[str, Any]:
|
66
|
+
settings = {
|
67
|
+
"file_backend": FileBackendConfig.S3,
|
68
|
+
"s3_endpoint": s3,
|
69
|
+
"s3_client_id": "",
|
70
|
+
"s3_client_secret": "",
|
71
|
+
"s3_ssl": False,
|
72
|
+
"s3_verify_ssl": False,
|
73
|
+
"s3_region_name": None,
|
74
|
+
"s3_bucket": "test-{kbid}",
|
75
|
+
"s3_bucket_tags": {
|
76
|
+
"testTag": "test",
|
77
|
+
},
|
78
|
+
}
|
79
|
+
extended_settings = {
|
80
|
+
"s3_indexing_bucket": "indexing",
|
81
|
+
"s3_deadletter_bucket": "deadletter",
|
82
|
+
}
|
83
|
+
with ExitStack() as stack:
|
84
|
+
for key, value in settings.items():
|
85
|
+
context = patch.object(storage_settings, key, value)
|
86
|
+
stack.enter_context(context)
|
87
|
+
for key, value in extended_settings.items():
|
88
|
+
context = patch.object(extended_storage_settings, key, value)
|
89
|
+
stack.enter_context(context)
|
90
|
+
|
91
|
+
yield settings | extended_settings
|
92
|
+
|
93
|
+
|
94
|
+
@pytest.fixture(scope="function")
|
95
|
+
async def s3_storage(s3, s3_storage_settings: dict[str, Any]):
|
62
96
|
storage = S3Storage(
|
63
|
-
aws_client_id=
|
64
|
-
aws_client_secret=
|
65
|
-
deadletter_bucket=
|
66
|
-
indexing_bucket=
|
67
|
-
endpoint_url=
|
68
|
-
|
69
|
-
|
70
|
-
region_name=
|
71
|
-
bucket=
|
72
|
-
bucket_tags=
|
97
|
+
aws_client_id=storage_settings.s3_client_id,
|
98
|
+
aws_client_secret=storage_settings.s3_client_secret,
|
99
|
+
deadletter_bucket=extended_storage_settings.s3_deadletter_bucket,
|
100
|
+
indexing_bucket=extended_storage_settings.s3_indexing_bucket,
|
101
|
+
endpoint_url=storage_settings.s3_endpoint,
|
102
|
+
use_ssl=storage_settings.s3_ssl,
|
103
|
+
verify_ssl=storage_settings.s3_verify_ssl,
|
104
|
+
region_name=storage_settings.s3_region_name,
|
105
|
+
bucket=storage_settings.s3_bucket,
|
106
|
+
bucket_tags=storage_settings.s3_bucket_tags,
|
73
107
|
)
|
74
108
|
await storage.initialize()
|
75
|
-
MAIN[Utility.STORAGE] = storage
|
76
109
|
yield storage
|
77
110
|
await storage.finalize()
|
78
|
-
MAIN.pop(Utility.STORAGE, None)
|
nucliadb_utils/utilities.py
CHANGED
@@ -313,8 +313,8 @@ async def start_indexing_utility(service_name: Optional[str] = None) -> Indexing
|
|
313
313
|
async def stop_indexing_utility():
|
314
314
|
indexing_utility = get_indexing()
|
315
315
|
if indexing_utility:
|
316
|
-
await indexing_utility.finalize()
|
317
316
|
clean_utility(Utility.INDEXING)
|
317
|
+
await indexing_utility.finalize()
|
318
318
|
|
319
319
|
|
320
320
|
def get_indexing() -> IndexingUtility:
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: nucliadb_utils
|
3
|
-
Version: 5.0.0.
|
3
|
+
Version: 5.0.0.post906
|
4
4
|
Home-page: https://nuclia.com
|
5
5
|
License: BSD
|
6
6
|
Classifier: Development Status :: 4 - Beta
|
@@ -23,8 +23,8 @@ Requires-Dist: PyNaCl
|
|
23
23
|
Requires-Dist: pyjwt >=2.4.0
|
24
24
|
Requires-Dist: memorylru >=1.1.2
|
25
25
|
Requires-Dist: mrflagly
|
26
|
-
Requires-Dist: nucliadb-protos >=5.0.0.
|
27
|
-
Requires-Dist: nucliadb-telemetry >=5.0.0.
|
26
|
+
Requires-Dist: nucliadb-protos >=5.0.0.post906
|
27
|
+
Requires-Dist: nucliadb-telemetry >=5.0.0.post906
|
28
28
|
Provides-Extra: cache
|
29
29
|
Requires-Dist: redis >=4.3.4 ; extra == 'cache'
|
30
30
|
Requires-Dist: orjson >=3.6.7 ; extra == 'cache'
|
@@ -7,7 +7,7 @@ nucliadb_utils/exceptions.py,sha256=y_3wk77WLVUtdo-5FtbBsdSkCtK_DsJkdWb5BoPn3qo,
|
|
7
7
|
nucliadb_utils/featureflagging.py,sha256=Pxd9esUARDovXvQ6ZfvhooQ1Yf8GdMzXyH7kuEUzPlk,2846
|
8
8
|
nucliadb_utils/grpc.py,sha256=apu0uePnkGHCAT7GRQ9YZfRYyFj26kJ440i8jitbM3U,3314
|
9
9
|
nucliadb_utils/helpers.py,sha256=nPw8yod3hP-pxq80VF8QC36s7ygSg0dBUdfI-LatvCs,1600
|
10
|
-
nucliadb_utils/indexing.py,sha256=
|
10
|
+
nucliadb_utils/indexing.py,sha256=Luaqcar3CySpdYOFp6Q9Fyr8ZYwhYhaKRHQ_VGL78f8,2318
|
11
11
|
nucliadb_utils/nats.py,sha256=zTAXECDXeCPtydk3F_6EMFDZ059kK0UYUU_tnWoxgXs,8208
|
12
12
|
nucliadb_utils/partition.py,sha256=jBgy4Hu5Iwn4gjbPPcthSykwf-qNx-GcLAIwbzPd1d0,1157
|
13
13
|
nucliadb_utils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -16,9 +16,9 @@ nucliadb_utils/settings.py,sha256=QR51SX0T17-_YofMNpci-nkI77l_CUWFy4H7i8hNOHU,79
|
|
16
16
|
nucliadb_utils/signals.py,sha256=JRNv2y9zLtBjOANBf7krGfDGfOc9qcoXZ6N1nKWS2FE,2674
|
17
17
|
nucliadb_utils/store.py,sha256=kQ35HemE0v4_Qg6xVqNIJi8vSFAYQtwI3rDtMsNy62Y,890
|
18
18
|
nucliadb_utils/transaction.py,sha256=mwcI3aIHAvU5KOGqd_Uz_d1XQzXhk_-NWY8NqU1lfb0,7307
|
19
|
-
nucliadb_utils/utilities.py,sha256=
|
19
|
+
nucliadb_utils/utilities.py,sha256=trAez4j2TYYT5n1qpimyHOEAnsQqDqD2qCFgVSDi184,16747
|
20
20
|
nucliadb_utils/aiopynecone/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
|
21
|
-
nucliadb_utils/aiopynecone/client.py,sha256=
|
21
|
+
nucliadb_utils/aiopynecone/client.py,sha256=JsFmHmPjs73JU3ZQY3LmrpOma4oBAhuXxIHgQ987Yp8,19464
|
22
22
|
nucliadb_utils/aiopynecone/exceptions.py,sha256=hFhq-UEY4slqNWjObXr_LPnRf_AQ1vpcG4SF2XRFd1E,2873
|
23
23
|
nucliadb_utils/aiopynecone/models.py,sha256=B_ihJhHZGp3ivQVUxhV49uoUnHe1PLDKxTgHNbHgSS0,2937
|
24
24
|
nucliadb_utils/audit/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
|
@@ -57,15 +57,15 @@ nucliadb_utils/storages/storage.py,sha256=gknIjRiHZ_f0EhTVYKVRgT4VqMlgDy9Ia6Av_1
|
|
57
57
|
nucliadb_utils/storages/utils.py,sha256=8g2rIwJeYIumQLOB47Yw1rx3twlhRB_cJxer65QfZmk,1479
|
58
58
|
nucliadb_utils/tests/__init__.py,sha256=Oo9CAE7B0eW5VHn8sHd6o30SQzOWUhktLPRXdlDOleA,1456
|
59
59
|
nucliadb_utils/tests/asyncbenchmark.py,sha256=x4be2IwCawle9zWgMOJkmwoUwk5p1tv7cLQGmybkEOg,10587
|
60
|
-
nucliadb_utils/tests/azure.py,sha256=
|
61
|
-
nucliadb_utils/tests/fixtures.py,sha256=
|
62
|
-
nucliadb_utils/tests/gcs.py,sha256=
|
60
|
+
nucliadb_utils/tests/azure.py,sha256=uf1Ltt1sHyoN0Yt5QwkAm7GZxx-SXST1JdB4x3P_dEY,4372
|
61
|
+
nucliadb_utils/tests/fixtures.py,sha256=b-e6Ciewh7GSh9rriG01bN9GiMW6xbGgrp3KiX7HNiY,2089
|
62
|
+
nucliadb_utils/tests/gcs.py,sha256=Bx-kxf2NoYYeyNFPpBFxbRN8kIiYFpELEvqG0V1_9Ks,4183
|
63
63
|
nucliadb_utils/tests/indexing.py,sha256=YW2QhkhO9Q_8A4kKWJaWSvXvyQ_AiAwY1VylcfVQFxk,1513
|
64
|
-
nucliadb_utils/tests/local.py,sha256=
|
64
|
+
nucliadb_utils/tests/local.py,sha256=7nuP8EFUAiA8ZH50R1iPV9EUXBySQxOanVm3Zht_e0g,1835
|
65
65
|
nucliadb_utils/tests/nats.py,sha256=xqpww4jZjTKY9oPGlJdDJG67L3FIBQsa9qDHxILR8r8,7687
|
66
|
-
nucliadb_utils/tests/s3.py,sha256=
|
67
|
-
nucliadb_utils-5.0.0.
|
68
|
-
nucliadb_utils-5.0.0.
|
69
|
-
nucliadb_utils-5.0.0.
|
70
|
-
nucliadb_utils-5.0.0.
|
71
|
-
nucliadb_utils-5.0.0.
|
66
|
+
nucliadb_utils/tests/s3.py,sha256=IdMxK_cNdSHLvO1u8BwsKFzD87Hk1MVPDZ57zx6h-rA,3656
|
67
|
+
nucliadb_utils-5.0.0.post906.dist-info/METADATA,sha256=GGSNxtcd_ycKhtG-WGcGGOZfIDhNGB0wRV40SUpnaHs,2073
|
68
|
+
nucliadb_utils-5.0.0.post906.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
|
69
|
+
nucliadb_utils-5.0.0.post906.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
|
70
|
+
nucliadb_utils-5.0.0.post906.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
71
|
+
nucliadb_utils-5.0.0.post906.dist-info/RECORD,,
|
File without changes
|
{nucliadb_utils-5.0.0.post899.dist-info → nucliadb_utils-5.0.0.post906.dist-info}/top_level.txt
RENAMED
File without changes
|
File without changes
|