nucliadb 4.0.0.post542__py3-none-any.whl → 6.2.1.post2777__py3-none-any.whl

nucliadb/search/tests/unit/search/test_paragraphs.py

@@ -1,157 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-import asyncio
-import random
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-from nucliadb_protos.utils_pb2 import ExtractedText
-
-from nucliadb.search.search import paragraphs
-
-
-@pytest.fixture()
-def extracted_text():
-    yield ExtractedText(
-        text=b"Hello World!",
-        split_text={"1": b"Hello", "2": b"World!"},
-    )
-
-
-@pytest.fixture()
-def storage_field(extracted_text):
-    mock = MagicMock()
-
-    data = extracted_text.SerializeToString()
-
-    async def _read_range(start, end):
-        yield data[start:end]
-
-    mock.read_range = _read_range
-    yield mock
-
-
-@pytest.fixture()
-def field(storage_field, extracted_text):
-    mock = MagicMock()
-    mock.get_storage_field.return_value = storage_field
-    mock.get_extracted_text = AsyncMock(return_value=extracted_text)
-    yield mock
-
-
-async def test_get_paragraph_from_full_text(field, extracted_text: ExtractedText):
-    assert (
-        await paragraphs.get_paragraph_from_full_text(
-            field=field, start=0, end=12, split=None
-        )
-        == extracted_text.text
-    )
-
-
-async def test_get_paragraph_from_full_text_with_split(
-    field, extracted_text: ExtractedText
-):
-    assert (
-        await paragraphs.get_paragraph_from_full_text(
-            field=field, start=0, end=6, split="1"
-        )
-        == extracted_text.split_text["1"]
-    )
-
-
-class TestGetParagraphText:
-    @pytest.fixture()
-    def orm_resource(self, field):
-        mock = AsyncMock()
-        mock.get_field.return_value = field
-        with patch(
-            "nucliadb.search.search.paragraphs.get_resource_from_cache",
-            return_value=mock,
-        ):
-            yield mock
-
-    async def test_get_paragraph_text(self, orm_resource):
-        assert (
-            await paragraphs.get_paragraph_text(
-                kbid="kbid",
-                rid="rid",
-                field="/t/text",
-                start=0,
-                end=12,
-                split=None,
-                highlight=True,
-                ematches=None,
-                matches=None,
-            )
-            == "Hello World!"
-        )
-
-        orm_resource.get_field.assert_called_once_with("text", 4, load=False)
-
-
-async def fake_get_extracted_text_from_gcloud(*args, **kwargs):
-    await asyncio.sleep(random.uniform(0, 1))
-    return ExtractedText(text=b"Hello World!")
-
-
-async def test_get_field_extracted_text_is_cached(field):
-    field.kbid = "kbid"
-    field.uuid = "rid"
-    field.id = "fid"
-    # Simulate a slow response from GCloud
-    field.get_extracted_text = AsyncMock(
-        side_effect=fake_get_extracted_text_from_gcloud
-    )
-
-    # Run 10 times in parallel to check that the cache is working
-    etcache = paragraphs.ExtractedTextCache()
-    futures = [
-        paragraphs.get_field_extracted_text(field, cache=etcache) for _ in range(10)
-    ]
-    await asyncio.gather(*futures)
-
-    field.get_extracted_text.assert_awaited_once()
-
-
-async def test_get_field_extracted_text_is_not_cached_when_none(field):
-    field.get_extracted_text = AsyncMock(return_value=None)
-
-    await paragraphs.get_field_extracted_text(field)
-    await paragraphs.get_field_extracted_text(field)
-
-    assert field.get_extracted_text.await_count == 2
-
-
-def test_extracted_text_cache():
-    etcache = paragraphs.ExtractedTextCache()
-    assert etcache.get_value("foo") is None
-
-    assert isinstance(etcache.get_lock("foo"), asyncio.Lock)
-    assert len(etcache.locks) == 1
-
-    etcache.set_value("foo", "bar")
-    assert len(etcache.values) == 1
-
-    assert etcache.get_value("foo") == "bar"
-
-    etcache.clear()
-
-    assert len(etcache.values) == 0
-    assert len(etcache.locks) == 0

nucliadb/search/tests/unit/search/test_predict_proxy.py

@@ -1,106 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-
-from unittest.mock import AsyncMock, Mock, patch
-
-import pytest
-from fastapi.datastructures import QueryParams
-from fastapi.responses import JSONResponse, StreamingResponse
-
-from nucliadb.common.datamanagers.exceptions import KnowledgeBoxNotFound
-from nucliadb.search.search.predict_proxy import PredictProxiedEndpoints, predict_proxy
-
-MODULE = "nucliadb.search.search.predict_proxy"
-
-
-@pytest.fixture(scope="function")
-def exists_kb():
-    with patch(f"{MODULE}.exists_kb", return_value=True) as mock:
-        yield mock
-
-
-@pytest.fixture(scope="function")
-def predict_response():
-    async def iter_any():
-        for i in range(3):
-            yield i.to_bytes(i, "big")
-
-    resp = Mock()
-    resp.status = 200
-    resp.headers = {}
-    resp.content = Mock(iter_any=iter_any)
-    resp.json = AsyncMock(return_value={"answer": "foo"})
-    yield resp
-
-
-@pytest.fixture(scope="function")
-def predict(predict_response):
-    predict_engine = Mock()
-    predict_engine.get_predict_headers = Mock(return_value={})
-    predict_engine.make_request = AsyncMock(return_value=predict_response)
-    with patch(f"{MODULE}.get_predict", return_value=predict_engine):
-        yield predict_engine
-
-
-async def test_raises_error_on_non_existing_kb(exists_kb):
-    exists_kb.return_value = False
-    with pytest.raises(KnowledgeBoxNotFound):
-        await predict_proxy(
-            "foo",
-            PredictProxiedEndpoints.CHAT,
-            "GET",
-            QueryParams(),
-        )
-
-
-async def test_stream_response(exists_kb, predict, predict_response):
-    predict_response.headers["Transfer-Encoding"] = "chunked"
-    predict_response.headers["NUCLIA-LEARNING-ID"] = "foo"
-
-    resp = await predict_proxy(
-        "foo",
-        PredictProxiedEndpoints.CHAT,
-        "GET",
-        QueryParams(),
-    )
-
-    assert isinstance(resp, StreamingResponse)
-    assert resp.status_code == 200
-    assert resp.headers["NUCLIA-LEARNING-ID"] == "foo"
-    assert resp.headers["Access-Control-Expose-Headers"] == "NUCLIA-LEARNING-ID"
-    body = [chunk async for chunk in resp.body_iterator]
-    assert list(map(lambda x: x.to_bytes(x, "big"), range(3))) == body
-
-
-async def test_json_response(exists_kb, predict, predict_response):
-    predict_response.headers["NUCLIA-LEARNING-ID"] = "foo"
-
-    resp = await predict_proxy(
-        "foo",
-        PredictProxiedEndpoints.CHAT,
-        "GET",
-        QueryParams(),
-    )
-
-    assert isinstance(resp, JSONResponse)
-    assert resp.status_code == 200
-    assert resp.headers["NUCLIA-LEARNING-ID"] == "foo"
-    assert resp.headers["Access-Control-Expose-Headers"] == "NUCLIA-LEARNING-ID"
-    assert resp.body == b'{"answer":"foo"}'

nucliadb/search/tests/unit/search/test_query.py

@@ -1,153 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-import json
-import unittest
-from unittest.mock import AsyncMock, Mock, patch
-
-import pytest
-from nucliadb_protos.knowledgebox_pb2 import Synonyms
-from nucliadb_protos.nodereader_pb2 import SearchRequest
-from nucliadb_protos.utils_pb2 import RelationNode
-
-from nucliadb.search.search.exceptions import InvalidQueryError
-from nucliadb.search.search.query import (
-    QueryParser,
-    check_supported_filters,
-    parse_entities_to_filters,
-)
-from nucliadb_models.search import MinScore
-
-QUERY_MODULE = "nucliadb.search.search.query"
-
-
-def test_parse_entities_to_filters():
-    detected_entities = [
-        RelationNode(value="John", ntype=RelationNode.NodeType.ENTITY, subtype="person")
-    ]
-
-    request = SearchRequest()
-    request.filter.field_labels.append("/e/person/Austin")
-    request.filter.expression = json.dumps({"and": [{"literal": "/e/person/Austin"}]})
-    assert parse_entities_to_filters(request, detected_entities) == ["/e/person/John"]
-    assert request.filter.field_labels == ["/e/person/Austin", "/e/person/John"]
-    assert json.loads(request.filter.expression) == {
-        "and": [
-            {"literal": "/e/person/John"},
-            {"and": [{"literal": "/e/person/Austin"}]},
-        ]
-    }
-
-    assert parse_entities_to_filters(request, detected_entities) == []
-    assert request.filter.field_labels == ["/e/person/Austin", "/e/person/John"]
-
-
-@pytest.fixture()
-def read_only_txn():
-    txn = unittest.mock.AsyncMock()
-    with unittest.mock.patch(
-        f"{QUERY_MODULE}.get_read_only_transaction", return_value=txn
-    ):
-        yield txn
-
-
-@pytest.fixture()
-def kbdm(read_only_txn):
-    kbdm = unittest.mock.AsyncMock()
-    with unittest.mock.patch(f"{QUERY_MODULE}.datamanagers.kb", kbdm):
-        yield kbdm
-
-
-class TestApplySynonymsToRequest:
-    @pytest.fixture
-    def get_synonyms(self):
-        get_kb_synonyms = AsyncMock()
-        synonyms = Synonyms()
-        synonyms.terms["planet"].synonyms.extend(["earth", "globe"])
-        get_kb_synonyms.return_value = synonyms
-        yield get_kb_synonyms
-
-    @pytest.fixture
-    def query_parser(self, get_synonyms):
-        qp = QueryParser(
-            kbid="kbid",
-            features=[],
-            query="query",
-            filters=[],
-            faceted=[],
-            page_number=0,
-            page_size=10,
-            min_score=MinScore(semantic=0.5),
-            with_synonyms=True,
-        )
-        with patch("nucliadb.search.search.query.get_kb_synonyms", get_synonyms):
-            yield qp
-
-    @pytest.mark.asyncio
-    async def test_not_applies_if_empty_body(
-        self, query_parser: QueryParser, get_synonyms
-    ):
-        query_parser.query = ""
-        search_request = Mock()
-        await query_parser.parse_synonyms(search_request)
-
-        get_synonyms.assert_not_awaited()
-        search_request.ClearField.assert_not_called()
-
-    @pytest.mark.asyncio
-    async def test_not_applies_if_synonyms_object_not_found(
-        self, query_parser: QueryParser, get_synonyms
-    ):
-        query_parser.query = "planet"
-        get_synonyms.return_value = None
-        request = Mock()
-
-        await query_parser.parse_synonyms(Mock())
-
-        request.ClearField.assert_not_called()
-        get_synonyms.assert_awaited_once_with("kbid")
-
-    @pytest.mark.asyncio
-    async def test_not_applies_if_synonyms_not_found_for_query(
-        self, query_parser: QueryParser, get_synonyms
-    ):
-        query_parser.query = "foobar"
-        request = Mock()
-
-        await query_parser.parse_synonyms(request)
-
-        request.ClearField.assert_not_called()
-
-        query_parser.query = "planet"
-        await query_parser.parse_synonyms(request)
-
-        request.ClearField.assert_called_once_with("body")
-        assert request.advanced_query == "planet OR earth OR globe"
-
-
-def test_check_supported_filters():
-    check_supported_filters({"literal": "a"}, ["a"])
-    check_supported_filters({"or": [{"literal": "a"}, {"literal": "b"}]}, [])
-    with pytest.raises(InvalidQueryError):
-        check_supported_filters({"or": [{"literal": "a"}, {"literal": "b"}]}, ["b"])
-    with pytest.raises(InvalidQueryError):
-        check_supported_filters(
-            {"and": [{"literal": "a"}, {"and": [{"literal": "c"}, {"literal": "b"}]}]},
-            ["b"],
-        )

nucliadb/search/tests/unit/test_app.py

@@ -1,79 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-import json
-from unittest.mock import patch
-
-import pytest
-
-from nucliadb.common.cluster.index_node import IndexNode
-from nucliadb.search import app
-
-pytestmark = pytest.mark.asyncio
-
-
-async def test_alive():
-    with patch.object(app.manager, "get_index_nodes", return_value=[{"id": "node1"}]):
-        resp = await app.alive(None)
-        assert resp.status_code == 200
-
-
-async def test_not_alive():
-    with patch.object(app.manager, "get_index_nodes", return_value=[]):
-        resp = await app.alive(None)
-        assert resp.status_code == 503
-
-
-async def test_ready():
-    with patch.object(app.manager, "get_index_nodes", return_value=[{"id": "node1"}]):
-        resp = await app.ready(None)
-        assert resp.status_code == 200
-
-
-async def test_not_ready():
-    with patch.object(app.manager, "get_index_nodes", return_value=[]):
-        resp = await app.ready(None)
-        assert resp.status_code == 503
-
-
-async def test_node_members():
-    nodes = [
-        IndexNode(
-            id="node1", address="node1", shard_count=0, available_disk=100, dummy=True
-        ),
-        IndexNode(
-            id="node2",
-            address="node2",
-            shard_count=0,
-            available_disk=50,
-            dummy=True,
-            primary_id="node1",
-        ),
-    ]
-    with patch.object(app.manager, "get_index_nodes", return_value=nodes):
-        resp = await app.node_members(None)
-        assert resp.status_code == 200
-        members = json.loads(resp.body)
-        sorted(members, key=lambda x: x["id"])
-        assert members[0]["id"] == "node1"
-        assert members[0]["primary_id"] is None
-        assert members[0]["available_disk"] == 100
-        assert members[1]["id"] == "node2"
-        assert members[1]["primary_id"] == "node1"
-        assert members[1]["available_disk"] == 50

nucliadb/search/tests/unit/test_find_merge.py

@@ -1,112 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-
-import random
-
-from nucliadb_protos.nodereader_pb2 import DocumentScored, ParagraphResult
-
-from nucliadb.search.search.find_merge import Orderer, merge_paragraphs_vectors
-from nucliadb_models.search import SCORE_TYPE
-
-
-def test_orderer():
-    orderer = Orderer()
-
-    items = {}
-    for i in range(30):
-        key = str(i)
-        score = random.random() * 25
-        items[key] = score
-
-    boosted = {4, 10, 28}
-
-    boosted_items = []
-    regular_items = []
-
-    for i, (key, score) in enumerate(items.items()):
-        if i in boosted:
-            boosted_items.append(key)
-            orderer.add_boosted(key)
-        else:
-            regular_items.append(key)
-            orderer.add(key)
-
-    sorted_items = list(orderer.sorted_by_insertion())
-    assert sorted_items == boosted_items + regular_items
-
-
-def test_orderer_handles_duplicate_insertions():
-    orderer = Orderer()
-    orderer.add_boosted("a")
-    orderer.add_boosted("b")
-    orderer.add_boosted("a")
-    orderer.add_boosted("c")
-    orderer.add("a")
-    assert list(orderer.sorted_by_insertion()) == ["a", "b", "c"]
-
-
-def test_merge_paragraphs_vectors():
-    paragraphs = []
-    for i in range(5):
-        pr = ParagraphResult()
-        pr.uuid = "foo"
-        pr.score.bm25 = i
-        pr.score.booster = 0
-        pr.paragraph = f"id/text/paragraph/{i}/0-10"
-        pr.start = 0
-        pr.end = 10
-        pr.field = "/a/title"
-        paragraphs.append(pr)
-
-    vectors = []
-    for i in range(5):
-        score = max(5 / float(i + 1), 1)
-        vr = DocumentScored()
-        vr.doc_id.id = f"id/vector/paragraph/{i}/0-2"
-        vr.score = score
-        vr.metadata.position.start = 0
-        vr.metadata.position.start = 2
-        vectors.append(vr)
-
-    paragraphs, next_page = merge_paragraphs_vectors(
-        [paragraphs], [vectors], 20, 0, min_score=1
-    )
-    assert not next_page
-    assert len(paragraphs) == 10
-
-    vector_scores = set()
-    for index, score_type in [
-        (0, SCORE_TYPE.BM25),
-        (1, SCORE_TYPE.VECTOR),
-        (2, SCORE_TYPE.BM25),
-        (3, SCORE_TYPE.BM25),
-        (4, SCORE_TYPE.VECTOR),
-        (5, SCORE_TYPE.BM25),
-        (6, SCORE_TYPE.BM25),
-        (7, SCORE_TYPE.VECTOR),
-        (8, SCORE_TYPE.VECTOR),
-        (9, SCORE_TYPE.VECTOR),
-    ]:
-        assert paragraphs[index].paragraph.score_type == score_type
-        if score_type == SCORE_TYPE.VECTOR:
-            vector_scores.add(paragraphs[index].paragraph.score)
-
-    # Check that the vector scores are different
-    assert len(vector_scores) == 5

nucliadb/search/tests/unit/test_merge.py

@@ -1,34 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-from unittest.mock import patch
-
-from nucliadb.search.search.merge import ResourceSearchResults, merge_paragraphs_results
-
-
-async def test_str_model():
-    # make sure __str__ works as advertised
-    res = await merge_paragraphs_results([], 1, 1, "kbid", [], [], [], False, 1)
-    assert str(res) == res.json()
-
-
-async def test_str_model_fallback():
-    with patch.object(ResourceSearchResults, "json", side_effect=Exception("ERROR")):
-        res = await merge_paragraphs_results([], 1, 1, "kbid", [], [], [], False, 1)
-        assert "sentences=None" in str(res)