nucliadb 4.0.0.post542__py3-none-any.whl → 6.2.1.post2777__py3-none-any.whl

nucliadb/writer/tests/test_knowledgebox.py

@@ -1,49 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import pytest
-
-from nucliadb.writer.api.v1.router import KB_PREFIX, KBS_PREFIX
-from nucliadb_models.resource import NucliaDBRoles
-
-
-@pytest.mark.asyncio
-async def test_knowledgebox_lifecycle(writer_api):
-    async with writer_api(roles=[NucliaDBRoles.MANAGER]) as client:
-        resp = await client.post(
-            f"/{KBS_PREFIX}",
-            json={
-                "slug": "kbid1",
-                "title": "My Knowledge Box",
-                "description": "My lovely knowledgebox",
-            },
-        )
-        assert resp.status_code == 201
-        data = resp.json()
-        assert data["slug"] == "kbid1"
-        kbid = data["uuid"]
-
-        resp = await client.patch(
-            f"/{KB_PREFIX}/{kbid}",
-            json={
-                "slug": "kbid2",
-                "description": "My lovely knowledgebox2",
-            },
-        )
-        assert resp.status_code == 200

nucliadb/writer/tests/test_reprocess_file_field.py

@@ -1,133 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-from typing import AsyncIterator
-from unittest.mock import AsyncMock
-
-import pytest
-
-from nucliadb.common import datamanagers
-from nucliadb.ingest.processing import ProcessingInfo
-from nucliadb.writer.api.v1.router import KB_PREFIX, RESOURCE_PREFIX, RESOURCES_PREFIX
-from nucliadb.writer.tests.utils import load_file_as_FileB64_payload
-from nucliadb.writer.utilities import get_processing
-from nucliadb_models.resource import NucliaDBRoles, QueueType
-
-
-@pytest.fixture(scope="function")
-def processing_mock(mocker):
-    processing = get_processing()
-    mocker.patch.object(
-        processing,
-        "send_to_process",
-        AsyncMock(
-            return_value=ProcessingInfo(seqid=0, account_seq=0, queue=QueueType.SHARED)
-        ),
-    )
-    yield processing
-
-
-@pytest.fixture(scope="function")
-@pytest.mark.asyncio
-async def file_field(
-    writer_api, knowledgebox_writer: str
-) -> AsyncIterator[tuple[str, str, str]]:
-    kbid = knowledgebox_writer
-    field_id = "myfile"
-
-    async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
-        resp = await client.post(
-            f"/{KB_PREFIX}/{kbid}/{RESOURCES_PREFIX}",
-            json={
-                "slug": "resource",
-                "title": "My resource",
-                "files": {
-                    field_id: {
-                        "language": "en",
-                        "password": "xxxxxx",
-                        "file": load_file_as_FileB64_payload(
-                            "assets/text001.txt", "text/plain"
-                        ),
-                    }
-                },
-            },
-        )
-        assert resp.status_code == 201
-        rid = resp.json()["uuid"]
-
-        assert (
-            await datamanagers.atomic.resources.resource_exists(kbid=kbid, rid=rid)
-        ) is True
-
-    yield kbid, rid, field_id
-
-    async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
-        resp = await client.delete(
-            f"/{KB_PREFIX}/{kbid}/{RESOURCE_PREFIX}/{rid}",
-        )
-        assert resp.status_code == 204
-
-
-@pytest.mark.asyncio
-async def test_reprocess_nonexistent_file_field(
-    writer_api, knowledgebox_writer: str, resource: str
-):
-    kbid = knowledgebox_writer
-    rid = resource
-    field_id = "nonexistent-field"
-
-    async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
-        resp = await client.post(
-            f"/{KB_PREFIX}/{kbid}/{RESOURCE_PREFIX}/{rid}/file/{field_id}/reprocess",
-        )
-        assert resp.status_code == 404
-
-
-@pytest.mark.asyncio
-async def test_reprocess_file_field_with_password(
-    writer_api, file_field: tuple[str, str, str], processing_mock
-):
-    kbid, rid, field_id = file_field
-    password = "secret-password"
-
-    async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
-        resp = await client.post(
-            f"/{KB_PREFIX}/{kbid}/{RESOURCE_PREFIX}/{rid}/file/{field_id}/reprocess",
-            headers={
-                "X-FILE-PASSWORD": password,
-            },
-        )
-        assert resp.status_code == 202
-
-    assert processing_mock.send_to_process.await_count == 1
-
-
-@pytest.mark.asyncio
-async def test_reprocess_file_field_without_password(
-    writer_api, file_field: tuple[str, str, str], processing_mock
-):
-    kbid, rid, field_id = file_field
-
-    async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
-        resp = await client.post(
-            f"/{KB_PREFIX}/{kbid}/{RESOURCE_PREFIX}/{rid}/file/{field_id}/reprocess",
-        )
-        assert resp.status_code == 202
-
-    assert processing_mock.send_to_process.await_count == 1

nucliadb/writer/tests/test_resources.py

@@ -1,476 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-from datetime import datetime
-from typing import Any, Callable, Optional
-from unittest.mock import AsyncMock  # type: ignore
-
-import pytest
-from httpx import AsyncClient
-
-import nucliadb_models
-from nucliadb.common import datamanagers
-from nucliadb.common.maindb.local import LocalDriver
-from nucliadb.common.maindb.redis import RedisDriver
-from nucliadb.ingest.orm.resource import Resource
-from nucliadb.ingest.processing import PushPayload
-from nucliadb.writer.api.v1.router import (
-    KB_PREFIX,
-    RESOURCE_PREFIX,
-    RESOURCES_PREFIX,
-    RSLUG_PREFIX,
-)
-from nucliadb.writer.tests.test_fields import (
-    TEST_CONVERSATION_PAYLOAD,
-    TEST_DATETIMES_PAYLOAD,
-    TEST_EXTERNAL_FILE_PAYLOAD,
-    TEST_FILE_PAYLOAD,
-    TEST_KEYWORDSETS_PAYLOAD,
-    TEST_LAYOUT_PAYLOAD,
-    TEST_LINK_PAYLOAD,
-    TEST_TEXT_PAYLOAD,
-)
-from nucliadb_models.resource import NucliaDBRoles
-
-
-@pytest.mark.asyncio
-async def test_resource_crud(
-    writer_api: Callable[[list[str]], AsyncClient], knowledgebox_writer: str
-):
-    knowledgebox_id = knowledgebox_writer
-    async with writer_api([NucliaDBRoles.WRITER]) as client:
-        # Test create resource
-        resp = await client.post(
-            f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCES_PREFIX}",
-            json={
-                "slug": "resource1",
-                "title": "My resource",
-                "summary": "Some summary",
-                "icon": "image/png",
-                "layout": "layout",
-                "metadata": {
-                    "language": "en",
-                    "metadata": {"key1": "value1", "key2": "value2"},
-                },
-                "fieldmetadata": [
-                    {
-                        "paragraphs": [
-                            {
-                                "key": "paragraph1",
-                                "classifications": [
-                                    {"labelset": "ls1", "label": "label1"}
-                                ],
-                            }
-                        ],
-                        "token": [
-                            {"token": "token1", "klass": "klass1", "start": 1, "end": 2}
-                        ],
-                        "field": {"field": "text1", "field_type": "text"},
-                    }
-                ],
-                "usermetadata": {
-                    "classifications": [{"labelset": "ls1", "label": "label1"}],
-                    "relations": [
-                        {
-                            "relation": "CHILD",
-                            "to": {
-                                "type": "resource",
-                                "value": "resource_uuid",
-                            },
-                        }
-                    ],
-                },
-                "origin": {
-                    "source_id": "source_id",
-                    "url": "http://some_source",
-                    "created": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
-                    "modified": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
-                    "metadata": {"key1": "value1", "key2": "value2"},
-                    "tags": ["tag1", "tag2"],
-                    "collaborators": ["col1", "col2"],
-                    "filename": "file.pdf",
-                    "related": ["related1"],
-                },
-                "texts": {"text1": TEST_TEXT_PAYLOAD},
-                "links": {"link1": TEST_LINK_PAYLOAD},
-                "files": {
-                    "file1": TEST_FILE_PAYLOAD,
-                    "external1": TEST_EXTERNAL_FILE_PAYLOAD,
-                },
-                "layouts": {"layout1": TEST_LAYOUT_PAYLOAD},
-                "conversations": {"conv1": TEST_CONVERSATION_PAYLOAD},
-                "keywordsets": {"keywordset1": TEST_KEYWORDSETS_PAYLOAD},
-                "datetimes": {"datetime1": TEST_DATETIMES_PAYLOAD},
-            },
-        )
-
-        assert resp.status_code == 201
-        data = resp.json()
-        assert "uuid" in data
-        assert "seqid" in data
-        rid = data["uuid"]
-
-        # Test update resource
-        resp = await client.patch(
-            f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}",
-            json={},
-        )
-        assert resp.status_code == 200
-
-        data = resp.json()
-
-        assert "seqid" in data
-
-        # Test delete resource
-        resp = await client.delete(
-            f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}",
-        )
-        assert resp.status_code == 204
-
-
-@pytest.mark.asyncio
-async def test_resource_crud_sync(
-    writer_api: Callable[[list[str]], AsyncClient], knowledgebox_writer: str
-):
-    knowledgebox_id = knowledgebox_writer
-    async with writer_api([NucliaDBRoles.WRITER]) as client:
-        # Test create resource
-        resp = await client.post(
-            f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCES_PREFIX}",
-            json={
-                "slug": "resource1",
-                "title": "My resource",
-                "summary": "Some summary",
-                "icon": "image/png",
-                "layout": "layout",
-                "metadata": {
-                    "language": "en",
-                    "metadata": {"key1": "value1", "key2": "value2"},
-                },
-                "fieldmetadata": [
-                    {
-                        "paragraphs": [
-                            {
-                                "key": "paragraph1",
-                                "classifications": [
-                                    {"labelset": "ls1", "label": "label1"}
-                                ],
-                            }
-                        ],
-                        "token": [
-                            {"token": "token1", "klass": "klass1", "start": 1, "end": 2}
-                        ],
-                        "field": {"field": "text1", "field_type": "text"},
-                    }
-                ],
-                "usermetadata": {
-                    "classifications": [{"labelset": "ls1", "label": "label1"}],
-                    "relations": [
-                        {
-                            "relation": "CHILD",
-                            "to": {
-                                "type": "resource",
-                                "value": "resource_uuid",
-                            },
-                        }
-                    ],
-                },
-                "origin": {
-                    "source_id": "source_id",
-                    "url": "http://some_source",
-                    "created": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
-                    "modified": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
-                    "metadata": {"key1": "value1", "key2": "value2"},
-                    "tags": ["tag1", "tag2"],
-                    "collaborators": ["col1", "col2"],
-                    "filename": "file.pdf",
-                    "related": ["related1"],
-                },
-                "texts": {"text1": TEST_TEXT_PAYLOAD},
-                "links": {"link1": TEST_LINK_PAYLOAD},
-                "files": {"file1": TEST_FILE_PAYLOAD},
-                "layouts": {"layout1": TEST_LAYOUT_PAYLOAD},
-                "conversations": {"conv1": TEST_CONVERSATION_PAYLOAD},
-                "keywordsets": {"keywordset1": TEST_KEYWORDSETS_PAYLOAD},
-                "datetimes": {"datetime1": TEST_DATETIMES_PAYLOAD},
-            },
-        )
-
-        assert resp.status_code == 201
-        data = resp.json()
-        assert "uuid" in data
-        assert "seqid" in data
-        assert "elapsed" in data
-        rid = data["uuid"]
-
-        assert (
-            await datamanagers.atomic.resources.resource_exists(
-                kbid=knowledgebox_id, rid=rid
-            )
-        ) is True
-
-        # Test update resource
-        resp = await client.patch(
-            f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}",
-            json={},
-        )
-        assert resp.status_code == 200
-
-        # Test delete resource
-
-        resp = await client.delete(
-            f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/resource1",
-        )
-
-        assert resp.status_code == 404
-
-        resp = await client.delete(
-            f"/{KB_PREFIX}/{knowledgebox_id}/{RESOURCE_PREFIX}/{rid}",
-        )
-        assert resp.status_code == 204
-
-        assert (
-            await datamanagers.atomic.resources.resource_exists(
-                kbid=knowledgebox_id, rid=rid
-            )
-        ) is False
-
-
-@pytest.mark.asyncio
-async def test_reprocess_resource(
-    writer_api: Callable[..., AsyncClient],
-    test_resource: Resource,
-    mocker,
-    maindb_driver,
-) -> None:
-    if isinstance(maindb_driver, (LocalDriver, RedisDriver)):
-        pytest.skip("Keys might not be ordered correctly in this driver")
-
-    rsc = test_resource
-    kbid = rsc.kb.kbid
-    rid = rsc.uuid
-
-    from nucliadb.writer.utilities import get_processing
-
-    processing = get_processing()
-    processing.values.clear()  # type: ignore
-
-    original = processing.send_to_process
-    mocker.patch.object(processing, "send_to_process", AsyncMock(side_effect=original))
-
-    async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
-        resp = await client.post(
-            f"/{KB_PREFIX}/{kbid}/resource/{rid}/reprocess",
-        )
-        assert resp.status_code == 202
-
-        assert processing.send_to_process.call_count == 1  # type: ignore
-        payload = processing.send_to_process.call_args[0][0]  # type: ignore
-        assert isinstance(payload, PushPayload)
-        assert payload.uuid == rid
-        assert payload.kbid == kbid
-
-        assert isinstance(payload.filefield.get("file1"), str)
-        assert payload.filefield["file1"] == "convert_internal_filefield_to_str,0"
-        assert isinstance(payload.linkfield.get("link1"), nucliadb_models.LinkUpload)
-        assert isinstance(payload.textfield.get("text1"), nucliadb_models.Text)
-        assert isinstance(
-            payload.layoutfield.get("layout1"), nucliadb_models.LayoutDiff
-        )
-        assert (
-            payload.layoutfield["layout1"].blocks["field1"].file
-            == "convert_internal_cf_to_str,2"
-        )
-        assert isinstance(
-            payload.conversationfield.get("conv1"), nucliadb_models.PushConversation
-        )
-        assert (
-            payload.conversationfield["conv1"].messages[33].content.attachments[0]
-            == "convert_internal_cf_to_str,0"
-        )
-        assert (
-            payload.conversationfield["conv1"].messages[33].content.attachments[1]
-            == "convert_internal_cf_to_str,1"
-        )
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize(
-    "method,endpoint,payload",
-    [
-        ["patch", "/{KB_PREFIX}/{kb}/{RSLUG_PREFIX}/{slug}", {}],
-        ["post", "/{KB_PREFIX}/{kb}/{RSLUG_PREFIX}/{slug}/reprocess", None],
-        ["post", "/{KB_PREFIX}/{kb}/{RSLUG_PREFIX}/{slug}/reindex", None],
-        ["delete", "/{KB_PREFIX}/{kb}/{RSLUG_PREFIX}/{slug}", None],
-    ],
-)
-async def test_resource_endpoints_by_slug(
-    writer_api: Callable[[list[str]], AsyncClient],
-    knowledgebox_ingest: str,
-    method: str,
-    endpoint: str,
-    payload: Optional[dict[Any, Any]],
-):
-    async with writer_api([NucliaDBRoles.WRITER]) as client:
-        slug = "my-resource"
-        resp = await client.post(
-            f"/{KB_PREFIX}/{knowledgebox_ingest}/{RESOURCES_PREFIX}",
-            json={
-                "slug": slug,
-                "texts": {"text1": {"body": "test1", "format": "PLAIN"}},
-            },
-        )
-        assert resp.status_code == 201
-
-        endpoint = endpoint.format(
-            KB_PREFIX=KB_PREFIX,
-            kb=knowledgebox_ingest,
-            RSLUG_PREFIX=RSLUG_PREFIX,
-            slug=slug,
-        )
-        extra_params = {}
-        if payload is not None:
-            extra_params["json"] = payload
-
-        op = getattr(client, method)
-        resp = await op(endpoint, **extra_params)
-
-        assert resp.status_code in (200, 202, 204)
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize(
-    "method,endpoint,payload",
-    [
-        ["patch", "/{KB_PREFIX}/{kb}/{RSLUG_PREFIX}/{slug}", {}],
-        ["post", "/{KB_PREFIX}/{kb}/{RSLUG_PREFIX}/{slug}/reprocess", None],
-        ["post", "/{KB_PREFIX}/{kb}/{RSLUG_PREFIX}/{slug}/reindex", None],
-        ["delete", "/{KB_PREFIX}/{kb}/{RSLUG_PREFIX}/{slug}", None],
-    ],
-)
-async def test_resource_endpoints_by_slug_404(
-    writer_api,
-    knowledgebox_ingest,
-    method,
-    endpoint,
-    payload,
-):
-    async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
-        endpoint = endpoint.format(
-            KB_PREFIX=KB_PREFIX,
-            kb=knowledgebox_ingest,
-            RSLUG_PREFIX=RSLUG_PREFIX,
-            slug="idonotexist",
-        )
-        extra_params = {}
-        if payload is not None:
-            extra_params["json"] = payload
-
-        op = getattr(client, method)
-        resp = await op(endpoint, **extra_params)
-
-        assert resp.status_code == 404
-        assert resp.json()["detail"] == "Resource does not exist"
-
-
-@pytest.mark.asyncio
-async def test_reindex(writer_api, test_resource):
-    rsc = test_resource
-    kbid = rsc.kb.kbid
-    rid = rsc.uuid
-    async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
-        resp = await client.post(
-            f"/{KB_PREFIX}/{kbid}/resource/{rid}/reindex",
-        )
-        assert resp.status_code == 200
-
-        resp = await client.post(
-            f"/{KB_PREFIX}/{kbid}/resource/{rid}/reindex?reindex_vectors=True",
-        )
-        assert resp.status_code == 200
-
-
-@pytest.mark.asyncio
-async def test_paragraph_annotations(writer_api, knowledgebox_writer):
-    kbid = knowledgebox_writer
-    async with writer_api(roles=[NucliaDBRoles.WRITER]) as client:
-        # Must have at least one classification
-        resp = await client.post(
-            f"/{KB_PREFIX}/{kbid}/resources",
-            json={
-                "texts": {"text1": TEST_TEXT_PAYLOAD},
-                "fieldmetadata": [
-                    {
-                        "paragraphs": [
-                            {
-                                "key": "paragraph1",
-                                "classifications": [],
-                            }
-                        ],
-                        "field": {"field": "text1", "field_type": "text"},
-                    }
-                ],
-            },
-        )
-        assert resp.status_code == 422
-        body = resp.json()
-        assert body["detail"] == "ensure classifications has at least 1 items"
-
-        classification = {"label": "label", "labelset": "ls"}
-
-        resp = await client.post(
-            f"/{KB_PREFIX}/{kbid}/resources",
-            json={
-                "texts": {"text1": TEST_TEXT_PAYLOAD},
-                "fieldmetadata": [
-                    {
-                        "paragraphs": [
-                            {
-                                "key": "paragraph1",
-                                "classifications": [classification],
-                            }
-                        ],
-                        "field": {"field": "text1", "field_type": "text"},
-                    }
-                ],
-            },
-        )
-        assert resp.status_code == 201
-        rid = resp.json()["uuid"]
-
-        # Classifications need to be unique
-        resp = await client.patch(
-            f"/{KB_PREFIX}/{kbid}/resource/{rid}",
-            json={
-                "fieldmetadata": [
-                    {
-                        "paragraphs": [
-                            {
-                                "key": "paragraph1",
-                                "classifications": [classification, classification],
-                            }
-                        ],
-                        "field": {"field": "text1", "field_type": "text"},
-                    }
-                ],
-            },
-        )
-        assert resp.status_code == 422
-        body = resp.json()
-        assert body["detail"] == "Paragraph classifications need to be unique"