nucliadb 2.46.1.post382__py3-none-any.whl → 6.2.1.post2777__py3-none-any.whl

nucliadb/writer/tests/test_tus.py

@@ -1,203 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import tempfile
-import uuid
-
-import asyncpg
-import pytest
-
-from nucliadb.writer.settings import settings
-from nucliadb.writer.tus import get_dm
-from nucliadb.writer.tus.exceptions import CloudFileNotFound
-from nucliadb.writer.tus.gcs import GCloudBlobStore, GCloudFileStorageManager
-from nucliadb.writer.tus.local import LocalBlobStore, LocalFileStorageManager
-from nucliadb.writer.tus.pg import PGBlobStore, PGFileStorageManager
-from nucliadb.writer.tus.s3 import S3BlobStore, S3FileStorageManager
-from nucliadb.writer.tus.storage import BlobStore, FileStorageManager
-from nucliadb_utils.storages.pg import PostgresStorage
-from nucliadb_utils.storages.storage import KB_RESOURCE_FIELD
-
-
-@pytest.fixture(scope="function")
-async def s3_storage_tus(s3):
-    storage = S3BlobStore()
-    await storage.initialize(
-        client_id="",
-        client_secret="",
-        max_pool_connections=2,
-        endpoint_url=s3,
-        verify_ssl=False,
-        ssl=False,
-        region_name=None,
-        bucket="test_{kbid}",
-        bucket_tags={"testTag": "test"},
-    )
-    yield storage
-    await storage.finalize()
-
-
-@pytest.fixture(scope="function")
-async def gcs_storage_tus(gcs):
-    storage = GCloudBlobStore()
-    await storage.initialize(
-        json_credentials=None,
-        bucket="test_{kbid}",
-        location="location",
-        project="project",
-        bucket_labels={},
-        object_base_url=gcs,
-    )
-    yield storage
-    await storage.finalize()
-
-
-@pytest.fixture(scope="function")
-async def local_storage_tus():
-    folder = tempfile.TemporaryDirectory()
-    storage = LocalBlobStore(local_testing_files=folder.name)
-    await storage.initialize()
-    yield storage
-    await storage.finalize()
-    folder.cleanup()
-
-
-@pytest.fixture(scope="function")
-async def pg_storage_tus(pg):
-    dsn = f"postgresql://postgres:postgres@{pg[0]}:{pg[1]}/postgres"
-    conn = await asyncpg.connect(dsn)
-    await conn.execute(
-        """
-DROP table IF EXISTS kb_files;
-DROP table IF EXISTS kb_files_fileparts;
-"""
-    )
-    await conn.close()
-    fstorage = PostgresStorage(dsn)  # set everything up
-    await fstorage.initialize()
-    await fstorage.finalize()
-
-    storage = PGBlobStore(dsn)
-    await storage.initialize()
-    yield storage
-    await storage.finalize()
-
-
-async def clean_dm():
-    from nucliadb.writer.tus import REDIS_FILE_DATA_MANAGER_FACTORY
-
-    if REDIS_FILE_DATA_MANAGER_FACTORY is not None:
-        await REDIS_FILE_DATA_MANAGER_FACTORY.finalize()
-        REDIS_FILE_DATA_MANAGER_FACTORY = None
-
-
-@pytest.fixture(scope="function")
-async def redis_dm(redis):
-    prev = settings.dm_enabled
-
-    settings.dm_enabled = True
-    settings.dm_redis_host = redis[0]
-    settings.dm_redis_port = redis[1]
-
-    dm = get_dm()
-
-    yield dm
-
-    await clean_dm()
-
-    settings.dm_enabled = prev
-
-
-@pytest.mark.asyncio
-async def test_pg_driver(redis_dm, pg_storage_tus: PGBlobStore):
-    await storage_test(pg_storage_tus, PGFileStorageManager(pg_storage_tus))
-
-
-@pytest.mark.asyncio
-async def test_s3_driver(redis_dm, s3_storage_tus: S3BlobStore):
-    await storage_test(s3_storage_tus, S3FileStorageManager(s3_storage_tus))
-
-
-@pytest.mark.asyncio
-async def test_gcs_driver(redis_dm, gcs_storage_tus: GCloudBlobStore):
-    await storage_test(gcs_storage_tus, GCloudFileStorageManager(gcs_storage_tus))
-
-
-@pytest.mark.asyncio
-async def test_local_driver(local_storage_tus: LocalBlobStore):
-    settings.dm_enabled = False
-    await storage_test(local_storage_tus, LocalFileStorageManager(local_storage_tus))
-    settings.dm_enabled = True
-
-
-async def storage_test(storage: BlobStore, file_storage_manager: FileStorageManager):
-    example = b"mytestinfo"
-    field = "myfield"
-    rid = "myrid"
-    kbid = "mykb_tus_test"
-
-    metadata: dict[str, str] = {}
-    bucket_name = storage.get_bucket_name(kbid)
-    assert bucket_name in [
-        "test_mykb_tus_test",
-        "test-mykb-tus-test",
-        "ndb_mykb_tus_test",
-        "mykb_tus_test",
-    ]
-
-    if not isinstance(storage, PGBlobStore):
-        # this is silly, but we don't need this for pg
-        assert await storage.check_exists(bucket_name) is False
-
-        exists = await storage.create_bucket(bucket_name)
-        assert exists is False
-
-    upload_id = uuid.uuid4().hex
-    dm = get_dm()
-    await dm.load(upload_id)
-    await dm.start({})
-    await dm.update(
-        upload_file_id=f"{upload_id}",
-        rid=rid,
-        field=field,
-        metadata=metadata,
-        deferred_length=True,
-        offset=0,
-        item=None,
-    )
-
-    path = KB_RESOURCE_FIELD.format(kbid=kbid, uuid=rid, field=field)
-    await file_storage_manager.start(dm, path=path, kbid=kbid)
-
-    async def generate():
-        yield example
-
-    size = await file_storage_manager.append(dm, generate(), 0)
-    await dm.update(offset=size)
-    assert size == len(example)
-    await file_storage_manager.finish(dm)
-
-    async for data in file_storage_manager.read_range(path, kbid, 1, size):
-        assert data == example[1:]
-
-    await file_storage_manager.delete_upload(path, kbid)
-
-    with pytest.raises(CloudFileNotFound):
-        async for data in file_storage_manager.read_range(path, kbid, 1, size):
-            assert data == example[1:]

nucliadb/writer/tests/utils.py

@@ -1,35 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import hashlib
-from base64 import b64encode
-from os.path import dirname
-
-
-def load_file_as_FileB64_payload(f: str, content_type: str) -> dict:
-    file_location = f"{dirname(__file__)}/{f}"
-    filename = f.split("/")[-1]
-    data = b64encode(open(file_location, "rb").read())
-
-    return {
-        "filename": filename,
-        "content_type": content_type,
-        "payload": data.decode("utf-8"),
-        "md5": hashlib.md5(data).hexdigest(),
-    }

nucliadb/writer/tus/pg.py

@@ -1,125 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-from __future__ import annotations
-
-from typing import AsyncIterator
-
-import asyncpg
-from nucliadb_protos.resources_pb2 import CloudFile
-
-from nucliadb.writer.tus.dm import FileDataManager
-from nucliadb.writer.tus.exceptions import CloudFileNotFound
-from nucliadb.writer.tus.storage import BlobStore, FileStorageManager
-from nucliadb_utils.storages import CHUNK_SIZE
-from nucliadb_utils.storages.pg import PostgresFileDataLayer
-
-
-class PGFileStorageManager(FileStorageManager):
-    _handler = None
-    storage: PGBlobStore
-    chunk_size = min_upload_size = CHUNK_SIZE
-
-    async def start(self, dm: FileDataManager, path: str, kbid: str):
-        bucket = self.storage.get_bucket_name(kbid)
-
-        async with self.storage.pool.acquire() as conn:
-            async with conn.transaction():
-                dl = PostgresFileDataLayer(conn)
-                if path is not None:
-                    await dl.delete_file(kbid, path)
-
-                await dl.create_file(
-                    kb_id=bucket,
-                    file_id=path,
-                    filename=dm.filename,
-                    size=dm.size,
-                    content_type=dm.content_type,
-                )
-
-        await dm.update(upload_file_id=path, path=path, bucket=bucket)
-
-    async def iter_data(self, uri, kbid: str, headers=None):
-        bucket = self.storage.get_bucket_name(kbid)
-
-        async with self.storage.pool.acquire() as conn:
-            dl = PostgresFileDataLayer(conn)
-            async for chunk in dl.iterate_chunks(bucket, uri):
-                yield chunk["data"]
-
-    async def read_range(
-        self, uri: str, kbid: str, start: int, end: int
-    ) -> AsyncIterator[bytes]:
-        """
-        Iterate through ranges of data
-        """
-        bucket = self.storage.get_bucket_name(kbid)
-
-        async with self.storage.pool.acquire() as conn:
-            dl = PostgresFileDataLayer(conn)
-            file_info = await dl.get_file_info(kbid, uri)
-            if file_info is None:
-                raise CloudFileNotFound()
-            async for data in dl.iterate_range(
-                kb_id=bucket, file_id=uri, start=start, end=end
-            ):
-                yield data
-
-    async def append(self, dm: FileDataManager, iterable, offset) -> int:
-        bucket = dm.get("bucket")
-        path = dm.get("path")
-        count = 0
-        async with self.storage.pool.acquire() as conn:
-            dl = PostgresFileDataLayer(conn)
-            async for chunk in iterable:
-                await dl.append_chunk(kb_id=bucket, file_id=path, data=chunk)
-                size = len(chunk)
-                count += size
-                offset += len(chunk)
-        return count
-
-    async def finish(self, dm: FileDataManager):
-        path = dm.get("path")
-        await dm.finish()
-        return path
-
-    async def delete_upload(self, uri: str, kbid: str):
-        async with self.storage.pool.acquire() as conn:
-            async with conn.transaction():
-                dl = PostgresFileDataLayer(conn)
-                await dl.delete_file(kbid, uri)
-
-
-class PGBlobStore(BlobStore):
-    def __init__(self, dsn: str):
-        self.dsn = dsn
-        self.source = CloudFile.POSTGRES
-
-    async def initialize(self):
-        self.pool = await asyncpg.create_pool(self.dsn)
-
-    async def finalize(self):
-        await self.pool.close()
-        self.initialized = False
-
-    async def check_exists(self, bucket_name: str) -> bool:
-        return True
-
-    def get_bucket_name(self, kbid: str) -> str:
-        return kbid

nucliadb-2.46.1.post382.dist-info/METADATA

@@ -1,134 +0,0 @@
-Metadata-Version: 2.1
-Name: nucliadb
-Version: 2.46.1.post382
-Home-page: https://docs.nuclia.dev/docs/guides/nucliadb/intro
-Author: NucliaDB Community
-Author-email: nucliadb@nuclia.com
-License: BSD
-Project-URL: Nuclia, https://nuclia.com
-Project-URL: Github, https://github.com/nuclia/nucliadb
-Project-URL: Discord, https://discord.gg/8EvQwmsbzf
-Project-URL: API Reference, https://docs.nuclia.dev/docs/api
-Keywords: search,semantic,AI
-Classifier: Development Status :: 4 - Beta
-Classifier: Intended Audience :: Developers
-Classifier: Intended Audience :: Information Technology
-Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3 :: Only
-Requires-Python: >=3.9, <4
-Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all] >=2.46.1.post382
-Requires-Dist: nucliadb-utils[cache,fastapi,storages] >=2.46.1.post382
-Requires-Dist: nucliadb-protos >=2.46.1.post382
-Requires-Dist: nucliadb-models >=2.46.1.post382
-Requires-Dist: nucliadb-admin-assets >=1.0.0.post1224
-Requires-Dist: nucliadb-node-binding >=2.26.0
-Requires-Dist: uvicorn <0.19.0
-Requires-Dist: pydantic-argparse
-Requires-Dist: aiohttp >=3.9.3
-Requires-Dist: lru-dict >=1.1.7
-Requires-Dist: backoff
-Requires-Dist: aiofiles >=0.8.0
-Requires-Dist: psutil >=5.9.7
-Requires-Dist: types-psutil >=5.9.5.17
-Requires-Dist: types-aiofiles >=0.8.3
-Requires-Dist: protobuf >=4.22.3
-Requires-Dist: types-protobuf <4.0,>=3.19.20
-Requires-Dist: grpcio >=1.44.0
-Requires-Dist: grpcio-health-checking >=1.44.0
-Requires-Dist: grpcio-channelz >=1.44.0
-Requires-Dist: grpcio-status >=1.44.0
-Requires-Dist: grpcio-tools >=1.44.0
-Requires-Dist: grpcio-testing >=1.44.0
-Requires-Dist: grpcio-reflection >=1.44.0
-Requires-Dist: orjson >=3.6.7
-Requires-Dist: types-setuptools
-Requires-Dist: pydantic <2.0,>=1.9.0
-Requires-Dist: aiobotocore >=2.9.0
-Requires-Dist: botocore >=1.34.0
-Requires-Dist: google-cloud-storage
-Requires-Dist: gcloud
-Requires-Dist: oauth2client
-Requires-Dist: jwcrypto >=1.5.6
-Requires-Dist: fastapi-versioning >=0.10.0
-Requires-Dist: fastapi >=0.95.2
-Requires-Dist: sentry-sdk >=1.5.12
-Requires-Dist: pyjwt >=2.4.0
-Requires-Dist: mmh3 >=3.0.0
-Requires-Dist: httpx >=0.23.0
-Requires-Dist: types-pkg-resources >=0.1.3
-Requires-Dist: grpc-stubs >=1.44.0
-Requires-Dist: aiodns >=3.0.0
-Requires-Dist: types-orjson
-Requires-Dist: asyncpg >=0.27.0
-Requires-Dist: tikv-client >=0.0.3
-Requires-Dist: multidict >=6.0.4
-Requires-Dist: deprecated >=1.2.12
-Requires-Dist: asgiref >=3.3.2
-Requires-Dist: jmespath >=1.0.0
-Requires-Dist: idna >=3.3
-Requires-Dist: sniffio >=1.2.0
-Requires-Dist: async-lru >=2.0.4
-Requires-Dist: async-timeout >=4.0.3
-Requires-Dist: cachetools >=5.3.2
-Requires-Dist: types-cachetools >=5.3.0.5
-Requires-Dist: kubernetes-asyncio
-Provides-Extra: redis
-Requires-Dist: redis >=4.3.4 ; extra == 'redis'
-
-# nucliadb
-
-This module contains most of the Python components for NucliaDB:
-
-- ingest
-- reader
-- writer
-- search
-- train
-
-# NucliaDB Migrations
-
-This module is used to manage NucliaDB Migrations.
-
-All migrations will be provided in the `migrations` folder and have a filename
-that follows the structure: `[sequence]_[migration name].py`.
-Where `sequence` is the order the migration should be run in with zero padding.
-Example: `0001_migrate_data.py`.
-
-Each migration should have the following:
-
-```python
-from nucliadb.migrator.context import ExecutionContext
-
-
-async def migrate(context: ExecutionContext) -> None:
-    """
-    Non-kb type of migration. Migrate global data.
-    """
-
-
-async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
-    """
-    Migrate kb.
-
-    Must have both types of migrations.
-    """
-```
-
-
-## How migrations are managed
-
-- All migrations utilize a distributed lock to prevent simulateously running jobs
-- Global migration state:
-    - current version
-    - target version
-    - KBs to migrate
-- KB Migration State:
-    - current version
-
-- Migrations are currently run with a deployment and will be continuously retried on failure.
-- Running migrations in a deployment is to make sure a migration does not prevent code deployment.