nucliadb 4.0.0.post542__py3-none-any.whl → 6.2.1.post2798__py3-none-any.whl

nucliadb/tests/unit/tasks/test_tasks.py

@@ -1,58 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import pydantic
-import pytest
-
-from nucliadb import tasks
-from nucliadb_utils import const
-
-
-class StreamTest(const.Streams):
-    name = "test"
-    group = "test"
-    subject = "test"
-
-
-class Message(pydantic.BaseModel):
-    kbid: str
-
-
-async def test_start_consumer(context):
-    with pytest.raises(ValueError):
-        await tasks.start_consumer("foo", context)
-
-    @tasks.register_task("foo", StreamTest, msg_type=Message)
-    async def some_test_work(context, msg: Message): ...
-
-    consumer = await tasks.start_consumer("foo", context)
-    assert consumer.initialized
-
-
-async def test_get_producer(context):
-    with pytest.raises(ValueError):
-        await tasks.get_producer("bar", context)
-
-    @tasks.register_task("bar", StreamTest, msg_type=Message)
-    async def some_test_work(context, msg: Message): ...
-
-    producer = await tasks.get_producer("bar", context)
-    assert producer.initialized
-    assert producer.msg_type == Message
-    assert producer.name == "bar_producer"

nucliadb/tests/unit/test_field_ids.py

@@ -1,49 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import pytest
-from pydantic import BaseModel, ValidationError
-
-from nucliadb_models.utils import FieldIdString
-
-
-class DummyFieldIdModel(BaseModel):
-    field_id: FieldIdString
-
-
-def test_field_ids():
-    """Test some examples of valid fields and exhaustively test invalid
-    fields.
-
-    """
-    valid_field_ids = [
-        "foo",
-        "foo_bar",
-        "foo-bar_123",
-    ]
-    for valid in valid_field_ids:
-        DummyFieldIdModel(field_id=valid)
-
-    invalid_field_ids = [
-        "",
-        "foo/bar",
-    ]
-    for invalid in invalid_field_ids:
-        with pytest.raises(ValidationError):
-            DummyFieldIdModel(field_id=invalid)

nucliadb/tests/unit/test_health.py

@@ -1,86 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-import asyncio
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-from grpc_health.v1 import health_pb2
-
-from nucliadb import health
-from nucliadb.common.cluster import manager
-from nucliadb.ingest.settings import DriverConfig, settings
-
-pytestmark = pytest.mark.asyncio
-
-
-@pytest.fixture(autouse=True)
-def register_checks():
-    health.register_health_checks(
-        [health.nats_manager_healthy, health.nodes_health_check, health.pubsub_check]
-    )
-    yield
-    health.unregister_all_checks()
-
-
-@pytest.fixture(autouse=True)
-def nats_manager():
-    mock = MagicMock()
-    with patch("nucliadb.health.get_nats_manager", return_value=mock):
-        yield mock
-
-
-async def test_grpc_health_check():
-    servicer = AsyncMock()
-    with (
-        patch.object(manager, "INDEX_NODES", {"node1": "node1"}),
-        patch.object(settings, "driver", DriverConfig.PG),
-    ):
-        task = asyncio.create_task(health.grpc_health_check(servicer))
-        await asyncio.sleep(0.05)
-
-        servicer.set.assert_called_with("", health_pb2.HealthCheckResponse.SERVING)
-
-        task.cancel()
-
-
-async def test_health_check_fail():
-    servicer = AsyncMock()
-    with (
-        patch.object(manager, "INDEX_NODES", {}),
-        patch.object(settings, "driver", DriverConfig.PG),
-    ):
-        task = asyncio.create_task(health.grpc_health_check(servicer))
-        await asyncio.sleep(0.05)
-
-        servicer.set.assert_called_with("", health_pb2.HealthCheckResponse.NOT_SERVING)
-
-        task.cancel()
-
-
-async def test_health_check_fail_unhealthy_nats(nats_manager):
-    nats_manager.healthy.return_value = False
-    servicer = AsyncMock()
-    with patch.object(manager, "INDEX_NODES", {"node1": "node1"}):  # has nodes
-        task = asyncio.create_task(health.grpc_health_check(servicer))
-        await asyncio.sleep(0.05)
-
-        servicer.set.assert_called_with("", health_pb2.HealthCheckResponse.NOT_SERVING)
-
-        task.cancel()

nucliadb/tests/unit/test_kb_slugs.py

@@ -1,54 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import pytest
-from pydantic import BaseModel, ValidationError
-
-from nucliadb_models.utils import SlugString
-
-
-class DummyModel(BaseModel):
-    slug: SlugString
-
-
-def test_kb_slugs():
-    """Test some examples of valid slugs and exhaustively test invalid
-    fields.
-
-    """
-    valid_slugs = [
-        "foo",
-        "foo_bar",
-        "foo-bar_123",
-        "my-kbis:my-kb-slug",
-        "SomeUpperCase",
-    ]
-    for valid in valid_slugs:
-        DummyModel(slug=valid)
-
-    invalid_slugs = [
-        "",
-        "foo/bar",
-        "foo.bar",
-        "@myslug",
-        "&invalid",
-    ]
-    for invalid in invalid_slugs:
-        with pytest.raises(ValidationError):
-            DummyModel(slug=invalid)

nucliadb/tests/unit/test_learning_proxy.py

@@ -1,252 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-from io import BytesIO
-from unittest import mock
-
-import pytest
-
-from nucliadb.learning_proxy import (
-    LearningConfiguration,
-    LearningService,
-    delete_configuration,
-    get_configuration,
-    learning_config_client,
-    proxy,
-    set_configuration,
-)
-
-MODULE = "nucliadb.learning_proxy"
-
-
-@pytest.fixture()
-def config_response():
-    resp = mock.Mock(
-        status_code=200,
-        headers={
-            "x-foo": "bar",
-            "Content-Type": "application/json",
-        },
-        content=b"some data",
-    )
-    resp.raise_for_status.return_value = None
-    yield resp
-
-
-@pytest.fixture()
-def config_stream_response():
-    resp = mock.Mock(
-        status_code=200,
-        headers={
-            "Transfer-Encoding": "chunked",
-        },
-        content=b"some data",
-    )
-
-    async def aiter_bytes():
-        yield b"some data"
-
-    resp.aiter_bytes = aiter_bytes
-    resp.raise_for_status.return_value = None
-    yield resp
-
-
-@pytest.fixture()
-def async_client(config_response):
-    client = mock.AsyncMock()
-    client.is_closed.return_value = False
-    client.request = mock.AsyncMock(return_value=config_response)
-    client.get = mock.AsyncMock(return_value=config_response)
-    client.post = mock.AsyncMock(return_value=config_response)
-    client.patch = mock.AsyncMock(return_value=config_response)
-    client.delete = mock.AsyncMock(return_value=config_response)
-    with mock.patch(f"{MODULE}.service_client") as mocked:
-        mocked.return_value.__aenter__.return_value = client
-        mocked.return_value.__aexit__.return_value = None
-        yield client
-
-
-@pytest.fixture()
-def is_onprem_nucliadb_mock():
-    with mock.patch(f"{MODULE}.is_onprem_nucliadb", return_value=False) as mocked:
-        yield mocked
-
-
-@pytest.fixture()
-def settings():
-    with mock.patch(f"{MODULE}.nuclia_settings") as settings:
-        settings.learning_internal_svc_base_url = (
-            "http://{service}.learning.svc.cluster.local:8080"
-        )
-        settings.nuclia_service_account = "service-account"
-        settings.nuclia_zone = "europe-1"
-        settings.nuclia_public_url = "http://{zone}.public-url"
-        yield settings
-
-
-async def test_get_learning_config_client(settings, is_onprem_nucliadb_mock):
-    is_onprem_nucliadb_mock.return_value = True
-    async with learning_config_client() as client:
-        assert str(client.base_url) == "http://europe-1.public-url/api/v1/"
-        assert client.headers["X-NUCLIA-NUAKEY"] == f"Bearer service-account"
-
-    is_onprem_nucliadb_mock.return_value = False
-    async with learning_config_client() as client:
-        assert (
-            str(client.base_url)
-            == "http://config.learning.svc.cluster.local:8080/api/v1/internal/"
-        )
-        assert "X-NUCLIA-NUAKEY" not in client.headers
-
-
-async def test_get_configuration(async_client):
-    lconfig = LearningConfiguration(
-        semantic_model="multilingual",
-        semantic_threshold=1.5,
-        semantic_vector_size=222,
-        semantic_vector_similarity="cosine",
-    )
-    resp = mock.Mock()
-    resp.raise_for_status.return_value = None
-    resp.json.return_value = lconfig.dict()
-    async_client.get.return_value = resp
-
-    resp = await get_configuration("kbid")
-
-    assert resp is not None
-    assert resp == lconfig
-    async_client.get.assert_called_once_with("config/kbid")
-
-
-async def test_set_configuration(config_response, async_client):
-    config_response.json.return_value = {
-        "semantic_model": "multilingual",
-        "semantic_threshold": 1.5,
-        "semantic_vector_size": 222,
-        "semantic_vector_similarity": "cosine",
-    }
-    resp = await set_configuration("kbid", {"some": "data"})
-    assert resp is not None
-    async_client.post.assert_called_once_with("config/kbid", json={"some": "data"})
-
-
-async def test_delete_configuration(async_client):
-    resp = await delete_configuration("kbid")
-    assert resp is None
-    async_client.delete.assert_called_once_with("config/kbid")
-
-
-async def test_proxy(async_client):
-    request = mock.Mock(
-        query_params={"some": "data"},
-        body=mock.AsyncMock(return_value=b"some data"),
-        headers={"x-nucliadb-user": "user", "x-nucliadb-roles": "roles"},
-    )
-    response = await proxy(
-        LearningService.CONFIG, request, "GET", "url", extra_headers={"foo": "bar"}
-    )
-
-    assert response.status_code == 200
-    assert response.body == b"some data"
-    assert response.media_type == "application/json"
-    assert response.headers["x-foo"] == "bar"
-
-    async_client.request.assert_called_once_with(
-        method="GET",
-        url="url",
-        params={"some": "data"},
-        content=b"some data",
-        headers={"foo": "bar", "x-nucliadb-user": "user", "x-nucliadb-roles": "roles"},
-    )
-
-
-async def test_proxy_stream_response(async_client, config_stream_response):
-    async_client.request.return_value = config_stream_response
-
-    request = mock.Mock(
-        query_params={"some": "data"},
-        body=mock.AsyncMock(return_value=b"some data"),
-        headers={"x-nucliadb-user": "user", "x-nucliadb-roles": "roles"},
-    )
-    response = await proxy(LearningService.CONFIG, request, "GET", "url")
-
-    assert response.status_code == 200
-    data = BytesIO()
-    async for chunk in response.body_iterator:
-        data.write(chunk)
-    assert data.getvalue() == b"some data"
-    assert response.headers["Transfer-Encoding"] == "chunked"
-
-    async_client.request.assert_called_once_with(
-        method="GET",
-        url="url",
-        params={"some": "data"},
-        content=b"some data",
-        headers={"x-nucliadb-user": "user", "x-nucliadb-roles": "roles"},
-    )
-
-
-async def test_proxy_error(async_client):
-    async_client.request.side_effect = Exception("some error")
-
-    request = mock.Mock(
-        query_params={"some": "data"},
-        body=mock.AsyncMock(return_value=b"some data"),
-        headers={},
-    )
-    response = await proxy(LearningService.CONFIG, request, "GET", "url")
-
-    assert response.status_code == 503
-    assert (
-        response.body
-        == b"Unexpected error while trying to proxy the request to the learning config API."
-    )
-    assert response.media_type == "text/plain"
-
-    assert len(async_client.request.mock_calls) == 3
-    async_client.request.assert_called_with(
-        method="GET",
-        url="url",
-        params={"some": "data"},
-        content=b"some data",
-        headers={},
-    )
-
-
-async def test_proxy_headers(async_client):
-    request = mock.Mock(
-        query_params={},
-        body=mock.AsyncMock(return_value=b""),
-        headers={
-            "x-nucliadb-user": "user",
-            "x-nucliadb-roles": "roles",
-            "host": "localhost",
-        },
-    )
-    response = await proxy(LearningService.CONFIG, request, "GET", "url")
-
-    assert response.status_code == 200
-
-    async_client.request.assert_called_once_with(
-        method="GET",
-        url="url",
-        params={},
-        content=b"",
-        headers={"x-nucliadb-user": "user", "x-nucliadb-roles": "roles"},
-    )

nucliadb/tests/unit/test_metrics_exporter.py

@@ -1,77 +0,0 @@
-# Copyright (C) 2021 Bosutech XXI S.L.
-#
-# nucliadb is offered under the AGPL v3.0 and as commercial software.
-# For commercial licensing, contact us at info@nuclia.com.
-#
-# AGPL:
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import asyncio
-from unittest import mock
-from unittest.mock import AsyncMock, Mock
-
-from nucliadb.metrics_exporter import (
-    run_exporter,
-    run_exporter_task,
-    update_migration_metrics,
-)
-
-
-async def test_run_exporter_task():
-    coro = AsyncMock()
-    context = Mock()
-    task = asyncio.create_task(run_exporter_task(context, coro, interval=0.5))
-    await asyncio.sleep(1)
-    task.cancel()
-    assert coro.await_count == 2
-    assert coro.call_args_list[0] == mock.call(context)
-
-
-async def test_run_exporter():
-    with mock.patch(
-        "nucliadb.metrics_exporter.update_migration_metrics"
-    ) as update_migration_metrics:
-        with mock.patch(
-            "nucliadb.metrics_exporter.update_node_metrics"
-        ) as update_node_metrics:
-            context = Mock()
-            task = asyncio.create_task(run_exporter(context))
-
-            await asyncio.sleep(1)
-
-            update_node_metrics.assert_called()
-            update_migration_metrics.assert_called()
-
-            task.cancel()
-
-
-async def test_update_migration_metrics():
-    async def iter_kbids(context):
-        yield "foo"
-        yield "bar"
-
-    with mock.patch("nucliadb.metrics_exporter.iter_kbids", new=iter_kbids):
-        with mock.patch("nucliadb.metrics_exporter.MIGRATION_COUNT") as migration_count:
-            with mock.patch(
-                "nucliadb.metrics_exporter.MigrationsDataManager", autospec=True
-            ) as mdm:
-                mdm.get_global_info = AsyncMock(return_value=Mock(current_version="1"))
-                mdm.get_kb_info = AsyncMock(return_value=Mock(current_version="2"))
-
-                driver = Mock()
-                context = Mock(kv_driver=driver)
-
-                await update_migration_metrics(context)
-
-                assert migration_count.set.call_count == 2