udata 12.0.2.dev16__py3-none-any.whl → 12.0.2.dev18__py3-none-any.whl

udata/harvest/tests/test_base_backend.py

@@ -10,6 +10,7 @@ from udata.core.dataset import tasks
 from udata.core.dataset.factories import DatasetFactory
 from udata.harvest.models import HarvestItem
 from udata.models import Dataset
+from udata.tests.api import PytestOnlyDBTestCase
 from udata.tests.helpers import assert_equal_dates
 from udata.utils import faker
 
@@ -93,8 +94,7 @@ class HarvestFilterTest:
             HarvestFilter(faker.word(), faker.word(), type, faker.sentence())
 
 
-@pytest.mark.usefixtures("clean_db")
-class BaseBackendTest:
+class BaseBackendTest(PytestOnlyDBTestCase):
     def test_simple_harvest(self):
         now = datetime.utcnow()
         nb_datasets = 3
@@ -420,8 +420,7 @@ class BaseBackendTest:
         assert dataset_reused_uri.harvest.source_id == str(source.id)
 
 
-@pytest.mark.usefixtures("clean_db")
-class BaseBackendValidateTest:
+class BaseBackendValidateTest(PytestOnlyDBTestCase):
     @pytest.fixture
     def validate(self):
         return FakeBackend(HarvestSourceFactory()).validate

udata/harvest/tests/test_dcat_backend.py

@@ -18,6 +18,7 @@ from udata.harvest.models import HarvestJob
 from udata.models import Dataset
 from udata.rdf import DCAT, RDF, namespace_manager
 from udata.storage.s3 import get_from_json
+from udata.tests.api import PytestOnlyDBTestCase
 
 from .. import actions
 from ..backends.dcat import URIS_TO_REPLACE
@@ -67,9 +68,8 @@ def mock_csw_pagination(rmock, path, pattern):
     return url
 
 
-@pytest.mark.usefixtures("clean_db")
 @pytest.mark.options(PLUGINS=["dcat"])
-class DcatBackendTest:
+class DcatBackendTest(PytestOnlyDBTestCase):
     def test_simple_flat(self, rmock):
         filename = "flat.jsonld"
         url = mock_dcat(rmock, filename)
@@ -191,7 +191,6 @@ class DcatBackendTest:
 
     def test_harvest_dataservices_keep_attached_associated_datasets(self, rmock):
         """It should update the existing list of dataservice.datasets and not overwrite existing ones"""
-        rmock.get("https://example.com/schemas", json=ResourceSchemaMockData.get_mock_data())
 
         filename = "bnodes.xml"
         url = mock_dcat(rmock, filename)
@@ -359,10 +358,8 @@ class DcatBackendTest:
             is None
         )
 
-    @pytest.mark.options(SCHEMA_CATALOG_URL="https://example.com/schemas", HARVEST_MAX_ITEMS=2)
+    @pytest.mark.options(HARVEST_MAX_ITEMS=2)
     def test_harvest_max_items(self, rmock):
-        rmock.get("https://example.com/schemas", json=ResourceSchemaMockData.get_mock_data())
-
         filename = "bnodes.xml"
         url = mock_dcat(rmock, filename)
         org = OrganizationFactory()
@@ -373,10 +370,7 @@ class DcatBackendTest:
         assert Dataset.objects.count() == 2
         assert HarvestJob.objects.first().status == "done"
 
-    @pytest.mark.options(SCHEMA_CATALOG_URL="https://example.com/schemas")
     def test_harvest_spatial(self, rmock):
-        rmock.get("https://example.com/schemas", json=ResourceSchemaMockData.get_mock_data())
-
         filename = "bnodes.xml"
         url = mock_dcat(rmock, filename)
         org = OrganizationFactory()
@@ -445,10 +439,7 @@ class DcatBackendTest:
         assert resources_by_title["Resource 3-1"].schema.url is None
         assert resources_by_title["Resource 3-1"].schema.version == "2.2.0"
 
-    @pytest.mark.options(SCHEMA_CATALOG_URL="https://example.com/schemas")
     def test_harvest_inspire_themese(self, rmock):
-        rmock.get("https://example.com/schemas", json=ResourceSchemaMockData.get_mock_data())
-
         filename = "bnodes.xml"
         url = mock_dcat(rmock, filename)
         org = OrganizationFactory()
@@ -893,9 +884,8 @@ class DcatBackendTest:
         assert "404 Client Error" in job.errors[0].message
 
 
-@pytest.mark.usefixtures("clean_db")
 @pytest.mark.options(PLUGINS=["csw"])
-class CswDcatBackendTest:
+class CswDcatBackendTest(PytestOnlyDBTestCase):
     def test_geonetworkv4(self, rmock):
         url = mock_csw_pagination(rmock, "geonetwork/srv/eng/csw.rdf", "geonetworkv4-page-{}.xml")
         org = OrganizationFactory()
@@ -1044,9 +1034,8 @@ class CswDcatBackendTest:
         assert len(job.items) == 1
 
 
-@pytest.mark.usefixtures("clean_db")
 @pytest.mark.options(PLUGINS=["csw"])
-class CswIso19139DcatBackendTest:
+class CswIso19139DcatBackendTest(PytestOnlyDBTestCase):
     @pytest.mark.parametrize(
         "remote_url_prefix",
         [

udata/harvest/tests/test_models.py

@@ -1,7 +1,6 @@
 import logging
 
-import pytest
-
+from udata.tests.api import PytestOnlyDBTestCase
 from udata.utils import faker
 
 from ..models import HarvestSource
@@ -9,8 +8,7 @@ from ..models import HarvestSource
 log = logging.getLogger(__name__)
 
 
-@pytest.mark.usefixtures("clean_db")
-class HarvestSourceTest:
+class HarvestSourceTest(PytestOnlyDBTestCase):
     def test_defaults(self):
         source = HarvestSource.objects.create(name="Test", url=faker.url(), backend="factory")
         assert source.name == "Test"

udata/harvest/tests/test_notifications.py

@@ -1,14 +1,12 @@
-import pytest
-
 from udata.core.user.factories import AdminFactory, UserFactory
 from udata.harvest.notifications import validate_harvester_notifications
+from udata.tests.api import PytestOnlyDBTestCase
 from udata.tests.helpers import assert_equal_dates
 
 from .factories import HarvestSourceFactory
 
 
-@pytest.mark.usefixtures("clean_db")
-class HarvestNotificationsTest:
+class HarvestNotificationsTest(PytestOnlyDBTestCase):
     def test_pending_harvester_validations(self):
         source = HarvestSourceFactory()
         admin = AdminFactory()

udata/harvest/tests/test_tasks.py

@@ -1,14 +1,13 @@
 import logging
 
-import pytest
+from udata.tests.api import PytestOnlyDBTestCase
 
 from ..tasks import purge_harvest_jobs, purge_harvest_sources
 
 log = logging.getLogger(__name__)
 
 
-@pytest.mark.usefixtures("clean_db")
-class HarvestActionsTest:
+class HarvestActionsTest(PytestOnlyDBTestCase):
     def test_purge_sources(self, mocker):
         """It should purge from DB sources flagged as deleted"""
         mock = mocker.patch("udata.harvest.actions.purge_sources")

udata/mongo/taglist_field.py

@@ -32,12 +32,12 @@ class TagListField(ListField):
         super(TagListField, self).validate(values)
 
         for tag in values:
-            if not tags.MIN_TAG_LENGTH <= len(tag) <= tags.MAX_TAG_LENGTH:
+            if not tags.TAG_MIN_LENGTH <= len(tag) <= tags.TAG_MAX_LENGTH:
                 self.error(
                     _(
                         'Tag "%(tag)s" must be between %(min)d and %(max)d characters long.',
-                        min=tags.MIN_TAG_LENGTH,
-                        max=tags.MAX_TAG_LENGTH,
+                        min=tags.TAG_MIN_LENGTH,
+                        max=tags.TAG_MAX_LENGTH,
                         tag=tag,
                     )
                 )

udata/settings.py

@@ -658,6 +658,9 @@ class Testing(object):
     }  # Disables deliverability for email domain name
     PUBLISH_ON_RESOURCE_EVENTS = False
     HARVEST_ACTIVITY_USER_ID = None
+    SEARCH_SERVICE_API_URL = None
+    CDATA_BASE_URL = None
+    SCHEMA_CATALOG_URL = None
 
 
 class Debug(Defaults):

udata/tags.py

@@ -2,8 +2,8 @@ from flask import current_app
 from slugify import slugify
 from werkzeug.local import LocalProxy
 
-MIN_TAG_LENGTH = LocalProxy(lambda: current_app.config["TAG_MIN_LENGTH"])
-MAX_TAG_LENGTH = LocalProxy(lambda: current_app.config["TAG_MAX_LENGTH"])
+TAG_MIN_LENGTH = LocalProxy(lambda: current_app.config["TAG_MIN_LENGTH"])
+TAG_MAX_LENGTH = LocalProxy(lambda: current_app.config["TAG_MAX_LENGTH"])
 
 
 def slug(value: str) -> str:
@@ -12,10 +12,10 @@ def slug(value: str) -> str:
 
 def normalize(value: str) -> str:
     value = slug(value)
-    if len(value) < MIN_TAG_LENGTH:
+    if len(value) < TAG_MIN_LENGTH:
         value = ""
-    elif len(value) > MAX_TAG_LENGTH:
-        value = value[:MAX_TAG_LENGTH]
+    elif len(value) > TAG_MAX_LENGTH:
+        value = value[:TAG_MAX_LENGTH]
     return value
 
 

udata/tests/__init__.py

@@ -1,24 +1,48 @@
 import unittest
 
 import pytest
+from werkzeug import Response
 
 from udata import settings
+from udata.app import UDataApp, create_app
+from udata.tests.plugin import TestClient
 
 from . import helpers
 
 
-class TestCase(unittest.TestCase):
-    settings = settings.Testing
+class TestCaseMixin:
+    app: UDataApp
 
-    @pytest.fixture(autouse=True)
-    def inject_app(self, app):
-        self.app = app
-        return self.create_app()
-
-    def create_app(self):
+    def get_settings(self, request):
         """
-        Here for compatibility legacy test classes
+        This seems really complicated for what it's doing. I think we want to create
+        an app with the default setting being `settings.Testing` and some overrides
+        from the "options" markers. But to do that `settings.Testing` should inherit from
+        `settings.Default`?
+
+        We may also want to prevent loading `udata.cfg` for testing to avoid failing tests
+        locally because some config is changed on our computer. Tests should work only with
+        default settings (or overrides for a specific test) but not with a local `udata.cfg`.
+
+        Not sure if the plugin situation is still relevant now that plugins are integrated
+        into udata.
         """
+        _settings = settings.Testing
+        # apply the options(plugins) marker from pytest_flask as soon as app is created
+        # https://github.com/pytest-dev/pytest-flask/blob/a62ea18cb0fe89e3f3911192ab9ea4f9b12f8a16/pytest_flask/plugin.py#L126
+        # this lets us have default settings for plugins applied while testing
+        plugins = getattr(_settings, "PLUGINS", [])
+        for options in request.node.iter_markers("options"):
+            option = options.kwargs.get("plugins", []) or options.kwargs.get("PLUGINS", [])
+            plugins += option
+        setattr(_settings, "PLUGINS", plugins)
+        return _settings
+
+    @pytest.fixture(autouse=True, name="app")
+    def _app(self, request):
+        test_settings = self.get_settings(request)
+        self.app = create_app(settings.Defaults, override=test_settings)
+        self.app.test_client_class = TestClient
         return self.app
 
     def assertEqualDates(self, datetime1, datetime2, limit=1):  # Seconds.
@@ -26,58 +50,16 @@ class TestCase(unittest.TestCase):
         __tracebackhide__ = True
         helpers.assert_equal_dates(datetime1, datetime2, limit=1)
 
-
-class WebTestMixin(object):
-    user = None
-
-    @pytest.fixture(autouse=True)
-    def inject_client(self, client):
-        """
-        Inject test client for compatibility with Flask-Testing.
-        """
-        self.client = client
-
-    def get(self, url, **kwargs):
-        return self.client.get(url, **kwargs)
-
-    def post(self, url, data=None, **kwargs):
-        return self.client.post(url, data=data, **kwargs)
-
-    def put(self, url, data=None, **kwargs):
-        return self.client.put(url, data=data, **kwargs)
-
-    def delete(self, url, data=None, **kwargs):
-        return self.client.delete(url, data=data, **kwargs)
-
-    def assertRedirects(self, response, location, message=None):
-        """
-        Checks if response is an HTTP redirect to the
-        given location.
-        :param response: Flask response
-        :param location: relative URL path to SERVER_NAME or an absolute URL
-        """
-        __tracebackhide__ = True
-        helpers.assert_redirects(response, location, message=message)
-
-    def assertStatus(self, response, status_code, message=None):
-        __tracebackhide__ = True
-        helpers.assert_status(response, status_code, message=message)
-
-    def full_url(self, *args, **kwargs):
+    def assertStreamEqual(self, response1: Response, response2: Response):
         __tracebackhide__ = True
-        return helpers.full_url(*args, **kwargs)
+        stream1 = list(response1.iter_encoded())
+        stream2 = list(response2.iter_encoded())
+        assert stream1 == stream2
 
-    def login(self, user=None):
-        self.user = self.client.login(user)
-        return self.user
 
-
-for code in 200, 201, 204, 400, 401, 403, 404, 410, 500:
-    name = "assert{0}".format(code)
-    helper = getattr(helpers, name)
-    setattr(WebTestMixin, name, lambda s, r, h=helper: h(r))
+class TestCase(TestCaseMixin, unittest.TestCase):
+    pass
 
 
-@pytest.mark.usefixtures("clean_db")
-class DBTestMixin(object):
+class PytestOnlyTestCase(TestCaseMixin):
     pass

udata/tests/api/__init__.py

@@ -1,12 +1,30 @@
 from contextlib import contextmanager
+from urllib.parse import urlparse
 
 import pytest
 
-from ..frontend import FrontTestCase
+from udata.mongo import db
+from udata.mongo.document import get_all_models
+from udata.tests import PytestOnlyTestCase, TestCase, helpers
 
 
 @pytest.mark.usefixtures("instance_path")
-class APITestCase(FrontTestCase):
+class APITestCaseMixin:
+    """
+    See explanation about `get`, `post` overrides in :TestClientOverride
+
+    (switch from `data` in kwargs to `data` in args to avoid doing `data=data` and default to `json=True`)
+    """
+
+    user = None
+
+    @pytest.fixture(autouse=True)
+    def load_api_routes(self, app):
+        from udata import api, frontend
+
+        api.init_app(app)
+        frontend.init_app(app)
+
     @pytest.fixture(autouse=True)
     def inject_api(self, api):
         """
@@ -14,11 +32,22 @@ class APITestCase(FrontTestCase):
         """
         self.api = api
 
+    @pytest.fixture(autouse=True)
+    def inject_client(self, client):
+        """
+        Inject test client for compatibility with Flask-Testing.
+        """
+        self.client = client
+
     @contextmanager
     def api_user(self, user=None):
         with self.api.user(user) as user:
             yield user
 
+    def login(self, user=None):
+        self.user = self.client.login(user)
+        return self.user
+
     def get(self, url, *args, **kwargs):
         return self.api.get(url, *args, **kwargs)
 
@@ -36,3 +65,59 @@ class APITestCase(FrontTestCase):
 
     def options(self, url, data=None, *args, **kwargs):
         return self.api.options(url, data=data, *args, **kwargs)
+
+    def assertStatus(self, response, status_code, message=None):
+        __tracebackhide__ = True
+        helpers.assert_status(response, status_code, message=message)
+
+
+for code in 200, 201, 204, 400, 401, 403, 404, 410, 500:
+    name = "assert{0}".format(code)
+    helper = getattr(helpers, name)
+    setattr(APITestCaseMixin, name, lambda s, r, h=helper: h(r))
+
+
+class _CleanDBMixin:
+    """
+    This is only for internal use. We shouldn't inherit from this mixin but
+    from `DBTestCase` or `PytestOnlyDBTestCase` (or `*APITestCase`)
+    This is temporary while we have two hierarchies.
+    """
+
+    def drop_db(self, app):
+        """Clear the database"""
+        parsed_url = urlparse(app.config["MONGODB_HOST"])
+
+        # drop the leading /
+        db_name = parsed_url.path[1:]
+        db.connection.drop_database(db_name)
+
+    @pytest.fixture(autouse=True)
+    def _clean_db(self, app):
+        self.drop_db(app)
+        for model in get_all_models():
+            # When dropping the database, MongoEngine will keep the collection cached inside
+            # `_collection` (in memory). This cache is used to call `ensure_indexes` only on the
+            # first call to `_get_collection()`, on subsequent calls the value inside `_collection`
+            # is returned without calling `ensure_indexes`.
+            # In tests, the first test will have a clean memory state, so MongoEngine will initialise
+            # the collection and create the indexes, then the following test, with a clean database (no indexes)
+            # will have the collection cached, so MongoEngine will never create the indexes (except if `auto_create_index_on_save`
+            # is set on the model, which may be the reason it is present on most of the big models, we may remove it?)
+            model._collection = None
+
+
+class DBTestCase(_CleanDBMixin, TestCase):
+    pass
+
+
+class PytestOnlyDBTestCase(_CleanDBMixin, PytestOnlyTestCase):
+    pass
+
+
+class APITestCase(APITestCaseMixin, DBTestCase):
+    pass
+
+
+class PytestOnlyAPITestCase(APITestCaseMixin, PytestOnlyDBTestCase):
+    pass

udata/tests/api/test_activities_api.py

@@ -1,4 +1,3 @@
-import pytest
 from flask import url_for
 from werkzeug.test import TestResponse
 
@@ -11,12 +10,9 @@ from udata.core.topic.factories import TopicFactory
 from udata.core.topic.models import Topic
 from udata.core.user.factories import AdminFactory, UserFactory
 from udata.mongo import db
+from udata.tests.api import APITestCase
 from udata.tests.helpers import assert200, assert400
 
-pytestmark = [
-    pytest.mark.usefixtures("clean_db"),
-]
-
 
 class FakeDatasetActivity(Activity):
     key = "fakeDataset"
@@ -33,26 +29,24 @@ class FakeTopicActivity(Activity):
     related_to = db.ReferenceField(Topic, required=True)
 
 
-class ActivityAPITest:
-    modules = []
-
-    def test_activity_api_list(self, api) -> None:
+class ActivityAPITest(APITestCase):
+    def test_activity_api_list(self) -> None:
         """It should fetch an activity list from the API"""
         activities: list[Activity] = [
             FakeDatasetActivity.objects.create(actor=UserFactory(), related_to=DatasetFactory()),
             FakeReuseActivity.objects.create(actor=UserFactory(), related_to=ReuseFactory()),
         ]
 
-        response: TestResponse = api.get(url_for("api.activity"))
+        response: TestResponse = self.get(url_for("api.activity"))
         assert200(response)
         assert len(response.json["data"]) == len(activities)
 
-    def test_activity_api_list_filter_by_bogus_related_to(self, api) -> None:
+    def test_activity_api_list_filter_by_bogus_related_to(self) -> None:
         """It should return a 400 error if the `related_to` parameter isn't a valid ObjectId."""
-        response: TestResponse = api.get(url_for("api.activity", related_to="foobar"))
+        response: TestResponse = self.get(url_for("api.activity", related_to="foobar"))
         assert400(response)
 
-    def test_activity_api_list_filtered_by_related_to(self, api) -> None:
+    def test_activity_api_list_filtered_by_related_to(self) -> None:
         """It should only return activities that correspond to the `related_to` parameter."""
         dataset1: Dataset = DatasetFactory()
         dataset2: Dataset = DatasetFactory()
@@ -64,18 +58,18 @@ class ActivityAPITest:
             FakeReuseActivity.objects.create(actor=UserFactory(), related_to=reuse),
         ]
 
-        response: TestResponse = api.get(url_for("api.activity", related_to=dataset1.id))
+        response: TestResponse = self.get(url_for("api.activity", related_to=dataset1.id))
         assert200(response)
         len(response.json["data"]) == 2
         assert response.json["data"][0]["related_to"] == dataset1.title
         assert response.json["data"][1]["related_to"] == dataset1.title
 
-        response: TestResponse = api.get(url_for("api.activity", related_to=reuse.id))
+        response: TestResponse = self.get(url_for("api.activity", related_to=reuse.id))
         assert200(response)
         len(response.json["data"]) == 1
         assert response.json["data"][0]["related_to"] == reuse.title
 
-    def test_activity_api_list_with_private(self, api) -> None:
+    def test_activity_api_list_with_private(self) -> None:
         """It should fetch an activity list from the API"""
         activities: list[Activity] = [
             FakeDatasetActivity.objects.create(
@@ -87,28 +81,28 @@ class ActivityAPITest:
         ]
 
         # Anonymised user won't see activities about private documents
-        response: TestResponse = api.get(url_for("api.activity"))
+        response: TestResponse = self.get(url_for("api.activity"))
         assert200(response)
         assert len(response.json["data"]) == 0
 
         # Lambda user won't see activities about private documents
-        api.login()
-        response: TestResponse = api.get(url_for("api.activity"))
+        self.login()
+        response: TestResponse = self.get(url_for("api.activity"))
         assert200(response)
         assert len(response.json["data"]) == 0
 
         # Sysadmin user will see activities about private documents
-        api.login(AdminFactory())
-        response: TestResponse = api.get(url_for("api.activity"))
+        self.login(AdminFactory())
+        response: TestResponse = self.get(url_for("api.activity"))
         assert200(response)
         assert len(response.json["data"]) == len(activities)
 
-    def test_activity_api_with_topic(self, api) -> None:
+    def test_activity_api_with_topic(self) -> None:
         """It should fetch topic activities from the API"""
         topic: Topic = TopicFactory()
         FakeTopicActivity.objects.create(actor=UserFactory(), related_to=topic)
 
-        response: TestResponse = api.get(url_for("api.activity"))
+        response: TestResponse = self.get(url_for("api.activity"))
         assert200(response)
         assert len(response.json["data"]) == 1
 

udata/tests/api/test_auth_api.py

@@ -14,6 +14,7 @@ from udata.api.oauth2 import OAuth2Client, OAuth2Token
 from udata.auth import PermissionDenied
 from udata.core.user.factories import UserFactory
 from udata.forms import Form, fields, validators
+from udata.tests.api import PytestOnlyAPITestCase
 from udata.tests.helpers import (
     assert200,
     assert400,
@@ -65,10 +66,7 @@ def oauth(app, request):
     return OAuth2Client.objects.create(**kwargs)
 
 
-@pytest.mark.usefixtures("clean_db")
-class APIAuthTest:
-    modules = []
-
+class APIAuthTest(PytestOnlyAPITestCase):
     def test_no_auth(self, api):
         """Should not return a content type if there is no content on delete"""
         response = api.get(url_for("api.fake"))