nucliadb 4.0.0.post542__py3-none-any.whl → 6.2.1.post2777__py3-none-any.whl

nucliadb/reader/api/v1/learning_collector.py

@@ -1,63 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-from fastapi import Request
-from fastapi_versioning import version
-
-from nucliadb.learning_proxy import learning_collector_proxy
-from nucliadb.reader.api.v1.router import KB_PREFIX, api
-from nucliadb_models.resource import NucliaDBRoles
-from nucliadb_utils.authentication import requires
-
-
-@api.get(
-    path=f"/{KB_PREFIX}/{{kbid}}/feedback/{{month}}",
-    status_code=200,
-    summary="Download feedback of a Knowledge Box",
-    description="Download the feedback of a particular month in a Knowledge Box",  # noqa
-    response_model=None,
-    tags=["Knowledge Boxes"],
-)
-@requires(NucliaDBRoles.READER)
-@version(1)
-async def feedback_download(
-    request: Request,
-    kbid: str,
-    month: str,
-):
-    return await learning_collector_proxy(
-        request, "GET", f"/collect/feedback/{kbid}/{month}"
-    )
-
-
-@api.get(
-    path=f"/{KB_PREFIX}/{{kbid}}/feedback",
-    status_code=200,
-    summary="Feedback avalaible months",
-    description="List of months within the last year with feedback data",
-    response_model=None,
-    tags=["Knowledge Boxes"],
-)
-@requires(NucliaDBRoles.READER)
-@version(1)
-async def feedback_list_months(
-    request: Request,
-    kbid: str,
-):
-    return await learning_collector_proxy(request, "GET", f"/collect/feedback/{kbid}")

nucliadb/reader/tests/__init__.py

@@ -1,19 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#

nucliadb/reader/tests/conftest.py

@@ -1,31 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-pytest_plugins = [
-    "pytest_docker_fixtures",
-    "nucliadb.tests.fixtures",
-    "nucliadb.tests.tikv",
-    "nucliadb.ingest.tests.fixtures",  # should be refactored out
-    "nucliadb.reader.tests.fixtures",
-    "nucliadb_utils.tests.nats",
-    "nucliadb_utils.tests.conftest",
-    "nucliadb_utils.tests.gcs",
-    "nucliadb_utils.tests.s3",
-    "nucliadb_utils.tests.asyncbenchmark",
-]

nucliadb/reader/tests/fixtures.py

@@ -1,136 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import uuid
-from datetime import datetime
-from enum import Enum
-from typing import Optional
-
-import pytest
-from httpx import AsyncClient
-from nucliadb_protos.writer_pb2 import BrokerMessage
-
-from nucliadb.ingest.orm.resource import KB_RESOURCE_SLUG_BASE
-from nucliadb.reader import API_PREFIX
-from nucliadb_utils.utilities import clear_global_cache
-
-
-@pytest.fixture(scope="function")
-def test_settings_reader(cache, gcs, fake_node, maindb_driver):  # type: ignore
-    from nucliadb_utils.settings import (
-        FileBackendConfig,
-        running_settings,
-        storage_settings,
-    )
-
-    running_settings.debug = False
-    print(f"Driver ready at {maindb_driver.url}")
-
-    storage_settings.gcs_endpoint_url = gcs
-    storage_settings.file_backend = FileBackendConfig.GCS
-    storage_settings.gcs_bucket = "test"
-
-    yield
-
-
-@pytest.fixture(scope="function")
-async def reader_api(test_settings_reader: None, local_files):  # type: ignore
-    from nucliadb.reader.app import create_application
-
-    application = create_application()
-
-    def make_client_fixture(
-        roles: Optional[list[Enum]] = None,
-        user: str = "",
-        version: str = "1",
-    ) -> AsyncClient:
-        roles = roles or []
-        client_base_url = "http://test"
-        client_base_url = f"{client_base_url}/{API_PREFIX}/v{version}"
-
-        client = AsyncClient(app=application, base_url=client_base_url)  # type: ignore
-        client.headers["X-NUCLIADB-ROLES"] = ";".join([role.value for role in roles])
-        client.headers["X-NUCLIADB-USER"] = user
-
-        return client
-
-    await application.router.startup()
-    yield make_client_fixture
-    await application.router.shutdown()
-    clear_global_cache()
-
-
-def broker_simple_resource(knowledgebox: str, number: int) -> BrokerMessage:
-    rid = str(uuid.uuid4())
-    message1: BrokerMessage = BrokerMessage(
-        kbid=knowledgebox,
-        uuid=rid,
-        slug=str(number),
-        type=BrokerMessage.AUTOCOMMIT,
-    )
-
-    message1.basic.icon = "text/plain"
-    message1.basic.title = str(number)
-    message1.basic.summary = "Summary of document"
-    message1.basic.thumbnail = "doc"
-    message1.basic.layout = "default"
-    message1.basic.metadata.useful = True
-    message1.basic.metadata.language = "es"
-    message1.basic.created.FromDatetime(datetime.utcnow())
-    message1.basic.modified.FromDatetime(datetime.utcnow())
-    message1.source = BrokerMessage.MessageSource.WRITER
-
-    return message1
-
-
-@pytest.fixture(scope="function")
-async def test_resources(processor, knowledgebox_ingest, test_settings_reader):
-    """
-    Create a set of resources with only basic information to test pagination
-    """
-    resources = []
-    amount = 10
-    for i in range(1, 10 + 1):
-        message = broker_simple_resource(knowledgebox_ingest, i)
-        await processor.process(message=message, seqid=i)
-        resources.append(message.uuid)
-        # Give processed data some time to reach the node
-
-    from time import time
-
-    from nucliadb.common.maindb.utils import get_driver
-
-    driver = get_driver()
-
-    t0 = time()
-
-    while time() - t0 < 30:  # wait max 30 seconds for it
-        txn = await driver.begin()
-        count = 0
-        async for key in txn.keys(
-            match=KB_RESOURCE_SLUG_BASE.format(kbid=knowledgebox_ingest), count=-1
-        ):
-            count += 1
-
-        await txn.abort()
-        if count == amount:
-            break
-        print(f"got {count}, retrying")
-
-    yield knowledgebox_ingest, resources

nucliadb/reader/tests/test_list_resources.py

@@ -1,75 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-from typing import Callable, Optional
-
-import pytest
-from httpx import AsyncClient
-
-from nucliadb.reader.api import DEFAULT_RESOURCE_LIST_PAGE_SIZE
-from nucliadb.reader.api.v1.router import KB_PREFIX
-from nucliadb_models.resource import NucliaDBRoles
-
-# All this scenarios are meant to match a total of 10 resources
-# coming from test_pagination_resources. Tests uses redis so order
-# is not guaranteed
-PAGINATION_TEST_SCENARIOS = [
-    (None, None, 10, True),  # Get first (also last) page with default values
-    (0, 20, 10, True),  # Get first (also last)page explicitly
-    (1, 20, 0, True),  # Get invalid page
-    (0, 5, 5, False),  # Get first non-last page
-    (1, 5, 5, True),  # Get last full page
-    (1, 6, 4, True),  # Get last partial page
-]
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize(
-    "page, size, expected_resources_count, expected_is_last_page",
-    PAGINATION_TEST_SCENARIOS,
-)
-async def test_list_resources(
-    reader_api: Callable[..., AsyncClient],
-    test_resources: tuple[str, list[str]],
-    page: Optional[int],
-    size: Optional[int],
-    expected_resources_count: int,
-    expected_is_last_page: bool,
-) -> None:
-    kbid = test_resources[0]
-
-    query_params = {}
-    if page is not None:
-        query_params["page"] = page
-
-    if size is not None:
-        query_params["size"] = size
-
-    async with reader_api(roles=[NucliaDBRoles.READER]) as client:
-        resp = await client.get(f"/{KB_PREFIX}/{kbid}/resources", params=query_params)
-        assert resp.status_code == 200
-        resources = resp.json()["resources"]
-        pagination = resp.json()["pagination"]
-
-        assert len(resources) == expected_resources_count
-        assert pagination["last"] == expected_is_last_page
-        assert pagination["page"] == query_params.get("page", 0)
-        assert pagination["size"] == query_params.get(
-            "size", DEFAULT_RESOURCE_LIST_PAGE_SIZE
-        )

nucliadb/reader/tests/test_reader_file_download.py

@@ -1,273 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import os
-from typing import Callable
-
-import pytest
-from httpx import AsyncClient
-from nucliadb_protos.resources_pb2 import FieldType
-
-import nucliadb.ingest.tests.fixtures
-from nucliadb.ingest.orm.resource import Resource
-from nucliadb.ingest.tests.fixtures import TEST_CLOUDFILE, THUMBNAIL
-from nucliadb.reader.api.v1.download import parse_media_range, safe_http_header_encode
-from nucliadb.reader.api.v1.router import KB_PREFIX, RESOURCE_PREFIX, RSLUG_PREFIX
-from nucliadb_models.resource import NucliaDBRoles
-
-BASE = ("field_id", "field_type")
-VALUE = ("value",)
-EXTRACTED = ("extracted",)
-
-
-@pytest.mark.asyncio
-async def test_resource_download_extracted_file(
-    reader_api: Callable[..., AsyncClient], test_resource: Resource
-) -> None:
-    rsc = test_resource
-    kbid = rsc.kb.kbid
-    rid = rsc.uuid
-    field_type = "text"
-    field_id = "text1"
-    download_type = "extracted"
-    download_field = "thumbnail"
-
-    async with reader_api(roles=[NucliaDBRoles.READER]) as client:
-        resp = await client.get(
-            f"/{KB_PREFIX}/{kbid}/{RESOURCE_PREFIX}/{rid}/{field_type}/{field_id}/download/{download_type}/{download_field}",  # noqa
-        )
-        assert resp.status_code == 200
-        filename = f"{os.path.dirname(nucliadb.ingest.tests.fixtures.__file__)}{THUMBNAIL.bucket_name}/{THUMBNAIL.uri}"
-
-        open(filename, "rb").read() == resp.content
-
-
-@pytest.mark.asyncio
-async def test_resource_download_field_file(
-    reader_api: Callable[..., AsyncClient], test_resource: Resource
-) -> None:
-    rsc = test_resource
-    kbid = rsc.kb.kbid
-    rid = rsc.uuid
-    field_id = "file1"
-
-    async with reader_api(roles=[NucliaDBRoles.READER]) as client:
-        resp = await client.get(
-            f"/{KB_PREFIX}/{kbid}/{RESOURCE_PREFIX}/{rid}?show=values",
-        )
-        assert (
-            resp.json()["data"]["files"]["file1"]["value"]["file"]["filename"]
-            == "text.pb"
-        )
-
-        # Check that invalid range is handled
-        resp = await client.get(
-            f"/{KB_PREFIX}/{kbid}/{RESOURCE_PREFIX}/{rid}/file/{field_id}/download/field",
-            headers={"range": "bytes=invalid-range"},
-        )
-        assert resp.status_code == 416
-        assert resp.json()["detail"]["reason"] == "rangeNotParsable"
-
-        # Check that multipart ranges not implemented is handled
-        resp = await client.get(
-            f"/{KB_PREFIX}/{kbid}/{RESOURCE_PREFIX}/{rid}/file/{field_id}/download/field",
-            headers={"range": "bytes=0-50, 100-150"},
-        )
-        assert resp.status_code == 416
-        assert resp.json()["detail"]["reason"] == "rangeNotSupported"
-
-        resp = await client.get(
-            f"/{KB_PREFIX}/{kbid}/{RESOURCE_PREFIX}/{rid}/file/{field_id}/download/field",
-            headers={"range": "bytes=0-"},
-        )
-        assert resp.status_code == 206
-        assert resp.headers["Content-Disposition"]
-
-        filename = f"{os.path.dirname(nucliadb.ingest.tests.fixtures.__file__)}/{TEST_CLOUDFILE.bucket_name}/{TEST_CLOUDFILE.uri}"  # noqa
-
-        open(filename, "rb").read() == resp.content
-
-        resp = await client.get(
-            f"/{KB_PREFIX}/{kbid}/{RESOURCE_PREFIX}/{rid}?show=values",
-        )
-        assert resp.status_code == 200
-
-        assert (
-            resp.json()["data"]["texts"]["text1"]["value"]["md5"]
-            == "74a3187271f1d526b1f6271bfb7df52e"
-        )
-        assert (
-            resp.json()["data"]["files"]["file1"]["value"]["file"]["md5"]
-            == "01cca3f53edb934a445a3112c6caa652"
-        )
-
-
-@pytest.mark.asyncio
-async def test_resource_download_field_layout(
-    reader_api: Callable[..., AsyncClient], test_resource: Resource
-) -> None:
-    rsc = test_resource
-    kbid = rsc.kb.kbid
-    rid = rsc.uuid
-    field_id = "layout1"
-    download_field = "field1"
-
-    async with reader_api(roles=[NucliaDBRoles.READER]) as client:
-        resp = await client.get(
-            f"/{KB_PREFIX}/{kbid}/{RESOURCE_PREFIX}/{rid}/layout/{field_id}/download/field/{download_field}",
-        )
-        assert resp.status_code == 200
-        filename = f"{os.path.dirname(nucliadb.ingest.tests.fixtures.__file__)}/{TEST_CLOUDFILE.bucket_name}/{TEST_CLOUDFILE.uri}"  # noqa
-
-        open(filename, "rb").read() == resp.content
-
-
-@pytest.mark.asyncio
-async def test_resource_download_field_conversation(
-    reader_api: Callable[..., AsyncClient], test_resource: Resource
-) -> None:
-    rsc = test_resource
-    kbid = rsc.kb.kbid
-    rid = rsc.uuid
-    field_id = "conv1"
-
-    msg_id, file_id = await _get_message_with_file(test_resource)
-
-    async with reader_api(roles=[NucliaDBRoles.READER]) as client:
-        resp = await client.get(
-            f"/{KB_PREFIX}/{kbid}/{RESOURCE_PREFIX}/{rid}/conversation/{field_id}/download/field/{msg_id}/{file_id}",
-        )
-        assert resp.status_code == 200
-        filename = f"{os.path.dirname(nucliadb.ingest.tests.fixtures.__file__)}/{THUMBNAIL.bucket_name}/{THUMBNAIL.uri}"  # noqa
-        assert open(filename, "rb").read() == resp.content
-
-
-@pytest.mark.parametrize(
-    "endpoint_part,endpoint_params",
-    [
-        [
-            "{field_type}/{field_id}/download/extracted/{download_field}",
-            {"field_type": "text", "field_id": "text1", "download_field": "thumbnail"},
-        ],  # noqa
-        ["file/{field_id}/download/field", {"field_id": "file1"}],
-        [
-            "layout/{field_id}/download/field/{download_field}",
-            {"field_id": "layout1", "download_field": "field1"},
-        ],
-        [
-            "conversation/{field_id}/download/field/{message_id}/{file_num}",
-            {"field_id": "conv1"},
-        ],
-    ],
-)
-@pytest.mark.asyncio
-async def test_download_fields_by_resource_slug(
-    reader_api, test_resource, endpoint_part, endpoint_params
-):
-    rsc = test_resource
-    kbid = rsc.kb.kbid
-    slug = rsc.basic.slug
-    if endpoint_part.startswith("conversation"):
-        # For conversations, we need to get a message id and a file number
-        msg_id, file_num = await _get_message_with_file(test_resource)
-        endpoint_params["message_id"] = msg_id
-        endpoint_params["file_num"] = file_num
-
-    async with reader_api(roles=[NucliaDBRoles.READER]) as client:
-        resource_path = f"/{KB_PREFIX}/{kbid}/{RSLUG_PREFIX}/{slug}"
-        endpoint = endpoint_part.format(**endpoint_params)
-        resp = await client.get(
-            f"{resource_path}/{endpoint}",
-        )
-        assert resp.status_code == 200
-
-        # Check that 404 is returned when a slug does not exist
-        unexisting_resource_path = f"/{KB_PREFIX}/{kbid}/{RSLUG_PREFIX}/idonotexist"
-        resp = await client.get(
-            f"{unexisting_resource_path}/{endpoint}",
-        )
-        assert resp.status_code == 404
-        assert resp.json()["detail"] == "Resource does not exist"
-
-
-async def _get_message_with_file(test_resource):
-    conversation_field = await test_resource.get_field("conv1", FieldType.CONVERSATION)
-    conversations = await conversation_field.get_value(page=1)
-    message_with_files = conversations.messages[33]
-    msg_id, file_num = message_with_files.content.attachments[1].uri.split("/")[-2:]
-    return msg_id, file_num
-
-
-@pytest.mark.parametrize(
-    "range_request,filesize,start,end,range_size,exception",
-    [
-        # No end range specified
-        ("bytes=0-", 10, 0, 9, 10, None),
-        # End range == file size
-        (f"bytes=0-10", 10, 0, 9, 10, None),
-        # End range < file size
-        (f"bytes=0-5", 10, 0, 5, 6, None),
-        # End range > file size
-        (f"bytes=0-11", 10, 0, 9, 10, None),
-        # Starting at a middle point until the end
-        (f"bytes=2-", 10, 2, 9, 8, None),
-        # A slice of bytes in the middle of the file
-        (f"bytes=2-8", 10, 2, 8, 7, None),
-        # Invalid range
-        ("bytes=something", 10, None, None, None, ValueError),
-        # Multi-part ranges not supported yet
-        ("bytes=0-50, 100-150", 10, None, None, None, NotImplementedError),
-    ],
-)
-def test_parse_media_range(range_request, filesize, start, end, range_size, exception):
-    if not exception:
-        result = parse_media_range(range_request, filesize)
-        assert result == (start, end, range_size)
-    else:
-        with pytest.raises(exception):
-            parse_media_range(range_request, filesize)
-
-
-@pytest.mark.asyncio
-async def test_resource_download_field_file_content_disposition(
-    reader_api: Callable[..., AsyncClient], test_resource: Resource
-) -> None:
-    rsc = test_resource
-    kbid = rsc.kb.kbid
-    rid = rsc.uuid
-    field_id = "file1"
-    download_url = (
-        f"/{KB_PREFIX}/{kbid}/{RESOURCE_PREFIX}/{rid}/file/{field_id}/download/field"
-    )
-    async with reader_api(roles=[NucliaDBRoles.READER]) as client:
-        # Defaults to attachment
-        resp = await client.get(download_url)
-        assert resp.status_code == 200
-        assert resp.headers["Content-Disposition"].startswith("attachment; filename=")
-
-        resp = await client.get(f"{download_url}?inline=true")
-        assert resp.status_code == 200
-        assert resp.headers["Content-Disposition"] == "inline"
-
-
-@pytest.mark.parametrize("text", ["ÇŞĞIİÖÜ"])
-def test_safe_http_header_encode(text):
-    safe_text = safe_http_header_encode(text)
-    # This is how startette encodes the headers
-    safe_text.lower().encode("latin-1")