nucliadb 4.0.0.post542__py3-none-any.whl → 6.2.1.post2777__py3-none-any.whl

nucliadb/search/tests/unit/search/search/test_shards.py

@@ -1,45 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import asyncio
-from unittest.mock import AsyncMock, Mock
-
-import pytest
-from nucliadb_protos.nodereader_pb2 import SearchRequest
-
-from nucliadb.search.search.shards import node_observer, query_shard
-
-
-async def test_node_observer_records_timeout_errors():
-    node = Mock(id="node-1")
-    # When waiting for a task to finish with asyncio, if it times out asyncio will
-    # cancell the task throwing a CancelledError on that task
-    node.reader.Search = AsyncMock(side_effect=asyncio.CancelledError)
-    query = SearchRequest(body="foo")
-
-    node_observer.counter.clear()
-
-    with pytest.raises(asyncio.CancelledError):
-        await query_shard(node, "shard", query)
-
-    sample = node_observer.counter.collect()[0].samples[0]
-    assert sample.name == "node_client_count_total"
-    assert sample.labels["type"] == "search"
-    assert sample.labels["node_id"] == "node-1"
-    assert sample.labels["status"] == "timeout"

nucliadb/search/tests/unit/search/search/test_utils.py

@@ -1,82 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import pytest
-
-from nucliadb.search.search.utils import (
-    has_user_vectors,
-    is_empty_query,
-    is_exact_match_only_query,
-    should_disable_vector_search,
-)
-from nucliadb_models.search import SearchRequest
-
-
-@pytest.mark.parametrize(
-    "item,empty",
-    [
-        (SearchRequest(query=""), True),
-        (SearchRequest(query="foo"), False),
-    ],
-)
-def test_is_empty_query(item, empty):
-    assert is_empty_query(item) is empty
-
-
-@pytest.mark.parametrize(
-    "query,exact_match",
-    [
-        ("some", False),
-        ("some query terms", False),
-        ('"something"', True),
-        ('   "something"', True),
-        ('"something"   ', True),
-        ('"something exact"', True),
-        ('"something exact" and something else', False),
-    ],
-)
-def test_is_exact_match_only_query(query, exact_match):
-    item = SearchRequest(query=query)
-    assert is_exact_match_only_query(item) is exact_match
-
-
-@pytest.mark.parametrize(
-    "item,has_vectors",
-    [
-        (SearchRequest(query=""), False),
-        (SearchRequest(vector=[]), False),
-        (SearchRequest(vector=[1.0]), True),
-    ],
-)
-def test_has_user_vectors(item, has_vectors):
-    assert has_user_vectors(item) is has_vectors
-
-
-@pytest.mark.parametrize(
-    "item,disable_vectors",
-    [
-        (SearchRequest(query=""), True),
-        (SearchRequest(query='"exact match"'), True),
-        (SearchRequest(query="foo"), False),
-        (SearchRequest(query="", vector=[1.0, 2.0]), False),
-        (SearchRequest(query='"exact match"', vector=[1.0, 2.0]), False),
-    ],
-)
-def test_should_disable_vectors(item, disable_vectors):
-    assert should_disable_vector_search(item) is disable_vectors

nucliadb/search/tests/unit/search/test_chat_prompt.py

@@ -1,270 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-from unittest import mock
-from unittest.mock import AsyncMock, patch
-
-import pytest
-
-from nucliadb.ingest.orm.resource import KB_REVERSE
-from nucliadb.search.search.chat import prompt as chat_prompt
-from nucliadb_models.search import (
-    SCORE_TYPE,
-    FindField,
-    FindParagraph,
-    FindResource,
-    KnowledgeboxFindResults,
-    MinScore,
-)
-from nucliadb_protos import resources_pb2
-
-
-@pytest.fixture()
-def messages():
-    msgs = [
-        resources_pb2.Message(
-            ident="1", content=resources_pb2.MessageContent(text="Message 1")
-        ),
-        resources_pb2.Message(
-            ident="2", content=resources_pb2.MessageContent(text="Message 2")
-        ),
-        resources_pb2.Message(
-            ident="3",
-            who="1",
-            content=resources_pb2.MessageContent(text="Message 3"),
-            type=resources_pb2.Message.MessageType.QUESTION,
-        ),
-        resources_pb2.Message(
-            ident="4",
-            content=resources_pb2.MessageContent(text="Message 4"),
-            type=resources_pb2.Message.MessageType.ANSWER,
-            to=["1"],
-        ),
-        resources_pb2.Message(
-            ident="5", content=resources_pb2.MessageContent(text="Message 5")
-        ),
-    ]
-    yield msgs
-
-
-@pytest.fixture()
-def field_obj(messages):
-    mock = AsyncMock()
-    mock.get_metadata.return_value = resources_pb2.FieldConversation(pages=1, total=5)
-    mock.db_get_value.return_value = resources_pb2.Conversation(messages=messages)
-
-    yield mock
-
-
-@pytest.fixture()
-def kb(field_obj):
-    mock = AsyncMock()
-    mock.get.return_value.get_field.return_value = field_obj
-    yield mock
-
-
-@pytest.mark.asyncio
-async def test_get_next_conversation_messages(field_obj, messages):
-    assert (
-        len(
-            await chat_prompt.get_next_conversation_messages(
-                field_obj=field_obj, page=1, start_idx=0, num_messages=5
-            )
-        )
-        == 5
-    )
-    assert (
-        len(
-            await chat_prompt.get_next_conversation_messages(
-                field_obj=field_obj, page=1, start_idx=0, num_messages=1
-            )
-        )
-        == 1
-    )
-
-    assert await chat_prompt.get_next_conversation_messages(
-        field_obj=field_obj,
-        page=1,
-        start_idx=0,
-        num_messages=1,
-        message_type=resources_pb2.Message.MessageType.ANSWER,
-        msg_to="1",
-    ) == [messages[3]]
-
-
-@pytest.mark.asyncio
-async def test_find_conversation_message(field_obj, messages):
-    assert await chat_prompt.find_conversation_message(
-        field_obj=field_obj, mident="3"
-    ) == (messages[2], 1, 2)
-
-
-@pytest.mark.asyncio
-async def test_get_expanded_conversation_messages(kb, messages):
-    assert await chat_prompt.get_expanded_conversation_messages(
-        kb=kb, rid="rid", field_id="field_id", mident="3"
-    ) == [messages[3]]
-
-
-@pytest.mark.asyncio
-async def test_get_expanded_conversation_messages_question(kb, messages):
-    assert (
-        await chat_prompt.get_expanded_conversation_messages(
-            kb=kb, rid="rid", field_id="field_id", mident="1"
-        )
-        == messages[1:]
-    )
-
-    kb.get.assert_called_with("rid")
-    kb.get.return_value.get_field.assert_called_with(
-        "field_id", KB_REVERSE["c"], load=True
-    )
-
-
-@pytest.mark.asyncio
-async def test_get_expanded_conversation_messages_missing(kb, messages):
-    assert (
-        await chat_prompt.get_expanded_conversation_messages(
-            kb=kb, rid="rid", field_id="field_id", mident="missing"
-        )
-        == []
-    )
-
-
-def _create_find_result(
-    _id: str, result_text: str, score_type: SCORE_TYPE = SCORE_TYPE.BM25, order=1
-):
-    return FindResource(
-        id=_id.split("/")[0],
-        fields={
-            "c/conv": FindField(
-                paragraphs={
-                    _id: FindParagraph(
-                        id=_id,
-                        score=1.0,
-                        score_type=score_type,
-                        order=order,
-                        text=result_text,
-                    )
-                }
-            )
-        },
-    )
-
-
-@pytest.mark.asyncio
-async def test_default_prompt_context(kb):
-    result_text = " ".join(["text"] * 10)
-    with (
-        patch("nucliadb.search.search.chat.prompt.get_read_only_transaction"),
-        patch("nucliadb.search.search.chat.prompt.get_storage"),
-        patch("nucliadb.search.search.chat.prompt.KnowledgeBoxORM", return_value=kb),
-    ):
-        context = chat_prompt.CappedPromptContext(max_size=int(1e6))
-        find_results = KnowledgeboxFindResults(
-            facets={},
-            resources={
-                "bmid": _create_find_result(
-                    "bmid/c/conv/ident", result_text, SCORE_TYPE.BM25, order=1
-                ),
-                "vecid": _create_find_result(
-                    "vecid/c/conv/ident", result_text, SCORE_TYPE.VECTOR, order=2
-                ),
-                "both_id": _create_find_result(
-                    "both_id/c/conv/ident", result_text, SCORE_TYPE.BOTH, order=0
-                ),
-            },
-        )
-        ordered_paragraphs = chat_prompt.get_ordered_paragraphs(find_results)
-
-        await chat_prompt.default_prompt_context(
-            context,
-            "kbid",
-            ordered_paragraphs,
-        )
-        prompt_result = context.output
-        # Check that the results are sorted by increasing order and that the extra
-        # context is added at the beginning, indicating that it has the most priority
-        paragraph_ids = [pid for pid in prompt_result.keys()]
-        assert paragraph_ids == [
-            "both_id/c/conv/ident",
-            "bmid/c/conv/ident",
-            "vecid/c/conv/ident",
-        ]
-
-
-@pytest.fixture(scope="function")
-def find_results():
-    return KnowledgeboxFindResults(
-        facets={},
-        resources={
-            "resource1": _create_find_result(
-                "resource1/a/title", "Resource 1", SCORE_TYPE.BOTH, order=1
-            ),
-            "resource2": _create_find_result(
-                "resource2/a/title", "Resource 2", SCORE_TYPE.VECTOR, order=2
-            ),
-        },
-        min_score=MinScore(semantic=-1),
-    )
-
-
-@pytest.mark.asyncio
-async def test_prompt_context_builder_prepends_user_context(
-    find_results: KnowledgeboxFindResults,
-):
-    builder = chat_prompt.PromptContextBuilder(
-        kbid="kbid", find_results=find_results, user_context=["Carrots are orange"]
-    )
-
-    async def _mock_build_context(context, *args, **kwargs):
-        context["resource1/a/title"] = "Resource 1"
-        context["resource2/a/title"] = "Resource 2"
-
-    with mock.patch.object(builder, "_build_context", new=_mock_build_context):
-        context, context_order, image_context = await builder.build()
-        assert len(context) == 3
-        assert len(context_order) == 3
-        assert len(image_context) == 0
-        assert context["USER_CONTEXT_0"] == "Carrots are orange"
-        assert context["resource1/a/title"] == "Resource 1"
-        assert context["resource2/a/title"] == "Resource 2"
-        assert context_order["USER_CONTEXT_0"] == 0
-        assert context_order["resource1/a/title"] == 1
-        assert context_order["resource2/a/title"] == 2
-
-
-def test_capped_prompt_context():
-    context = chat_prompt.CappedPromptContext(max_size=2)
-
-    # Check that output is trimmed
-    context["key1"] = "123"
-
-    assert context.output == {"key1": "12"}
-    assert context.size == 2
-
-    # Update existing value
-    context["key1"] = "foobar"
-    assert context.output == {"key1": "fo"}
-    assert context.size == 2
-
-    # Check without limits
-    context = chat_prompt.CappedPromptContext(max_size=None)
-    context["key1"] = "foo" * int(1e6)
-
-    assert context.output == {"key1": "foo" * int(1e6)}

nucliadb/search/tests/unit/search/test_fetch.py

@@ -1,108 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-import time
-
-import pytest
-from pytest_benchmark.fixture import BenchmarkFixture  # type: ignore
-
-from nucliadb.search.search.paragraphs import highlight_paragraph as highlight
-
-
-@pytest.mark.benchmark(
-    group="highlight",
-    min_time=0.1,
-    max_time=0.5,
-    min_rounds=5,
-    timer=time.time,
-    disable_gc=True,
-    warmup=False,
-)
-def test_highligh_error(benchmark: BenchmarkFixture):
-    text = "bu kimlik belgelerinin geçerlilik sürelerinin standartlara aykırı olmadığını, fotoğraftaki yakın alan iletişim çipindeki bilgilerin tutarlı ve geçerli olmadığını ve İçişleri Bakanlığı'nın ortasında kimlik değişimine erişebilenleri onaylar. sistem"  # noqa
-    ematch = ["kimlik", "sistem"]
-    res = benchmark(highlight, text, [], ematch)
-    assert res.count("mark") == 6
-    assert (
-        res
-        == "bu <mark>kimlik</mark> belgelerinin geçerlilik sürelerinin standartlara aykırı olmadığını, fotoğraftaki yakın alan iletişim çipindeki bilgilerin tutarlı ve geçerli olmadığını ve İçişleri Bakanlığı'nın ortasında <mark>kimlik</mark> değişimine erişebilenleri onaylar. <mark>sistem</mark>"  # noqa
-    )
-
-
-def test_highlight():
-    res = highlight(
-        "Query whatever you want my to make it work my query with this",
-        ["this", "is", "my", "query"],
-    )
-    assert (
-        res
-        == "<mark>Query</mark> whatever you want <mark>my</mark> to make it work <mark>my</mark> <mark>query</mark> with <mark>this</mark>"  # noqa
-    )
-
-    res = highlight(
-        "Query whatever you want to make it work my query with this",
-        ["this", "is"],
-        ["my query"],
-    )
-
-    assert (
-        res
-        == "Query whatever you want to make it work <mark>my query</mark> with <mark>this</mark>"
-    )
-
-    res = highlight(
-        "Query whatever you redis want to make it work my query with this",
-        ["this", "is"],
-        ["my query"],
-    )
-
-    assert (
-        res
-        == "Query whatever you redis want to make it work <mark>my query</mark> with <mark>this</mark>"
-    )
-
-    res = highlight(
-        "Plone offers superior security controls, often without cost, of course!",
-        ["use", "cases", "of", "plone"],
-    )
-
-    assert (
-        res
-        == "<mark>Plone</mark> offers superior security controls, often without cost, <mark>of</mark> course!"
-    )
-
-    res = highlight(
-        "In contrast, traditional companies often make it impossible",
-        ["of", "market"],
-        ["of", "market"],
-    )
-    assert res == "In contrast, traditional companies often make it impossible"
-
-    # sc-3067: Unbalanced parenthesis or brackets in query should not make highlight fail
-    res = highlight(
-        "Some sentence here",
-        [
-            "Some).",
-        ],
-        [
-            "sent)ence",
-            "(here",
-        ],
-    )
-    assert res == "Some sentence here"

nucliadb/search/tests/unit/search/test_filters.py

@@ -1,125 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-from unittest import mock
-
-import jsonschema  # type: ignore
-import pytest
-
-from nucliadb.search.search.filters import (
-    INDEX_NODE_FILTERS_SCHEMA,
-    convert_filter_to_node_schema,
-    convert_to_node_filters,
-    iter_filter_labels_expression,
-    translate_label_filters,
-)
-from nucliadb_models.search import Filter
-
-
-@pytest.fixture(scope="function")
-def is_paragraph_labelset_kind_mock():
-    with mock.patch(
-        "nucliadb.search.search.filters.is_paragraph_labelset_kind"
-    ) as mocked:
-        yield mocked
-
-
-@pytest.mark.parametrize(
-    "original,converted",
-    [
-        ("foo", {"literal": "foo"}),
-        (Filter(all=["foo"]), {"literal": "foo"}),
-        (Filter(all=["foo", "bar"]), {"and": [{"literal": "foo"}, {"literal": "bar"}]}),
-        (Filter(any=["foo"]), {"literal": "foo"}),
-        (Filter(any=["foo", "bar"]), {"or": [{"literal": "foo"}, {"literal": "bar"}]}),
-        (Filter(none=["foo"]), {"not": {"literal": "foo"}}),
-        (
-            Filter(none=["foo", "bar"]),
-            {"not": {"or": [{"literal": "foo"}, {"literal": "bar"}]}},
-        ),
-        (Filter(not_all=["foo"]), {"not": {"literal": "foo"}}),
-        (
-            Filter(not_all=["foo", "bar"]),
-            {"not": {"and": [{"literal": "foo"}, {"literal": "bar"}]}},
-        ),
-    ],
-)
-def test_convert_filter_to_node_schema(original, converted):
-    assert convert_filter_to_node_schema(original) == converted
-    jsonschema.validate(converted, INDEX_NODE_FILTERS_SCHEMA)
-
-
-def test_convert_to_node_filters():
-    assert convert_to_node_filters([]) == {}
-    assert convert_to_node_filters(["foo"]) == {"literal": "foo"}
-    assert convert_to_node_filters(["foo", "bar"]) == {
-        "and": [{"literal": "foo"}, {"literal": "bar"}]
-    }
-    assert convert_to_node_filters([Filter(all=["foo"])]) == {"literal": "foo"}
-    assert convert_to_node_filters([Filter(all=["foo"]), Filter(any=["bar"])]) == {
-        "and": [{"literal": "foo"}, {"literal": "bar"}]
-    }
-
-
-def test_translate_label_filters():
-    literal = {"literal": "/classification.labels/foo/bar"}
-    translated = {"literal": "/l/foo/bar"}
-
-    assert translate_label_filters(literal) == translated
-    assert translate_label_filters({"not": literal}) == {"not": translated}
-    assert translate_label_filters({"and": [literal, literal]}) == {
-        "and": [translated, translated]
-    }
-    assert translate_label_filters({"or": [literal, literal]}) == {
-        "or": [translated, translated]
-    }
-    assert translate_label_filters(
-        {"and": [{"or": [literal, literal]}, {"not": literal}]}
-    ) == {
-        "and": [
-            {"or": [translated, translated]},
-            {"not": translated},
-        ]
-    }
-
-
-def test_iter_filter_labels_expression():
-    literal = {"literal": "foo"}
-    assert list(iter_filter_labels_expression(literal)) == ["foo"]
-    assert list(iter_filter_labels_expression({"and": [literal, literal]})) == [
-        "foo",
-        "foo",
-    ]
-    assert list(iter_filter_labels_expression({"or": [literal, literal]})) == [
-        "foo",
-        "foo",
-    ]
-    assert list(
-        iter_filter_labels_expression({"not": {"and": [literal, literal]}})
-    ) == ["foo", "foo"]
-
-
-def test_filters_model():
-    f = Filter(all=["foo", "bar"], any=None)
-    assert f.all == ["foo", "bar"]
-    assert f.any is None
-    assert f.none is None
-    assert f.not_all is None
-
-    with pytest.raises(ValueError):
-        Filter(all=["foo"], any=["bar"])