udata 10.9.1.dev37462__py2.py3-none-any.whl → 10.9.1.dev37604__py2.py3-none-any.whl

udata/api/__init__.py

@@ -352,7 +352,6 @@ def init_app(app):
     import udata.core.reports.api  # noqa
     import udata.core.site.api  # noqa
     import udata.core.tags.api  # noqa
-    import udata.core.topic.api  # noqa
     import udata.core.topic.apiv2  # noqa
     import udata.core.post.api  # noqa
     import udata.core.contact_point.api  # noqa

udata/core/dataset/api.py

@@ -213,7 +213,7 @@ class DatasetApiParser(ModelApiParser):
             except Topic.DoesNotExist:
                 pass
             else:
-                datasets = datasets.filter(id__in=[d.id for d in topic.datasets])
+                datasets = datasets.filter(id__in=topic.get_nested_elements_ids("Dataset"))
         if args.get("dataservice"):
             if not ObjectId.is_valid(args["dataservice"]):
                 api.abort(400, "Dataservice arg must be an identifier")

udata/core/dataset/search.py

@@ -3,6 +3,7 @@ import datetime
 from udata.core.dataset.api import DEFAULT_SORTING, DatasetApiParser
 from udata.core.spatial.constants import ADMIN_LEVEL_MAX
 from udata.core.spatial.models import admin_levels
+from udata.core.topic.models import TopicElement
 from udata.models import Dataset, GeoZone, License, Organization, Topic, User
 from udata.search import (
     BoolFilter,
@@ -73,7 +74,9 @@ class DatasetSearch(ModelSearchAdapter):
         organization = None
         owner = None
 
-        topics = Topic.objects(datasets=dataset).only("id")
+        topic_ids = list(
+            set(te.topic.id for te in TopicElement.objects(element=dataset) if te.topic)
+        )
 
         if dataset.organization:
             org = Organization.objects(id=dataset.organization.id).first()
@@ -112,7 +115,7 @@ class DatasetSearch(ModelSearchAdapter):
             "owner": str(owner.id) if owner else None,
             "format": [r.format.lower() for r in dataset.resources if r.format],
             "schema": [r.schema.name for r in dataset.resources if r.schema],
-            "topics": [str(t.id) for t in topics if topics],
+            "topics": [str(tid) for tid in topic_ids],
         }
         extras = {}
         for key, value in dataset.extras.items():

udata/core/dataset/tasks.py

@@ -13,7 +13,7 @@ from udata.core import csv, storages
 from udata.core.dataservices.models import Dataservice
 from udata.harvest.models import HarvestJob
 from udata.i18n import lazy_gettext as _
-from udata.models import Activity, Discussion, Follow, Organization, Topic, Transfer, db
+from udata.models import Activity, Discussion, Follow, Organization, TopicElement, Transfer, db
 from udata.tasks import job
 
 from .constants import UPDATE_FREQUENCIES
@@ -43,10 +43,7 @@ def purge_datasets(self):
         # Remove activity
         Activity.objects(related_to=dataset).delete()
         # Remove topics' related dataset
-        for topic in Topic.objects(datasets=dataset):
-            datasets = topic.datasets
-            datasets.remove(dataset)
-            topic.update(datasets=datasets)
+        TopicElement.objects(element=dataset).update(element=None)
         # Remove dataservices related dataset
         for dataservice in Dataservice.objects(datasets=dataset):
             datasets = dataservice.datasets

udata/core/reuse/tasks.py

@@ -1,5 +1,6 @@
 from udata import mail
 from udata.core import storages
+from udata.core.topic.models import TopicElement
 from udata.i18n import lazy_gettext as _
 from udata.models import Activity, Discussion, Follow, Transfer
 from udata.tasks import get_logger, job, task
@@ -21,6 +22,8 @@ def purge_reuses(self) -> None:
         Activity.objects(related_to=reuse).delete()
         # Remove transfers
         Transfer.objects(subject=reuse).delete()
+        # Remove reuses references in Topics
+        TopicElement.objects(element=reuse).update(element=None)
         # Remove reuse's logo in all sizes
         if reuse.image.filename is not None:
             storage = storages.images

udata/core/topic/__init__.py

@@ -0,0 +1 @@
+DEFAULT_PAGE_SIZE = 20

udata/core/topic/api_fields.py

@@ -0,0 +1,87 @@
+from flask import url_for
+
+from udata.api import api, apiv2, fields
+from udata.core.organization.api_fields import org_ref_fields
+from udata.core.spatial.api_fields import spatial_coverage_fields
+from udata.core.topic import DEFAULT_PAGE_SIZE
+from udata.core.user.api_fields import user_ref_fields
+
+topic_fields = apiv2.model(
+    "Topic",
+    {
+        "id": fields.String(description="The topic identifier"),
+        "name": fields.String(description="The topic name", required=True),
+        "slug": fields.String(description="The topic permalink string", readonly=True),
+        "description": fields.Markdown(
+            description="The topic description in Markdown", required=True
+        ),
+        "tags": fields.List(
+            fields.String, description="Some keywords to help in search", required=True
+        ),
+        "elements": fields.Raw(
+            attribute=lambda o: {
+                "rel": "subsection",
+                "href": url_for(
+                    "apiv2.topic_elements",
+                    topic=o.id,
+                    page=1,
+                    page_size=DEFAULT_PAGE_SIZE,
+                    _external=True,
+                ),
+                "type": "GET",
+                "total": len(o.elements),
+            },
+            description="Link to the topic elements",
+        ),
+        "featured": fields.Boolean(description="Is the topic featured"),
+        "private": fields.Boolean(description="Is the topic private"),
+        "created_at": fields.ISODateTime(description="The topic creation date", readonly=True),
+        "spatial": fields.Nested(
+            spatial_coverage_fields, allow_null=True, description="The spatial coverage"
+        ),
+        "last_modified": fields.ISODateTime(
+            description="The topic last modification date", readonly=True
+        ),
+        "organization": fields.Nested(
+            org_ref_fields,
+            allow_null=True,
+            description="The publishing organization",
+            readonly=True,
+        ),
+        "owner": fields.Nested(
+            user_ref_fields, description="The owner user", readonly=True, allow_null=True
+        ),
+        "uri": fields.String(
+            attribute=lambda t: url_for("apiv2.topic", topic=t),
+            description="The topic API URI",
+            readonly=True,
+        ),
+        "extras": fields.Raw(description="Extras attributes as key-value pairs"),
+    },
+)
+
+topic_page_fields = apiv2.model("TopicPage", fields.pager(topic_fields))
+
+element_fields = apiv2.model(
+    "TopicElement",
+    {
+        "id": fields.String(description="The element id"),
+        "title": fields.String(description="The element title"),
+        "description": fields.String(description="The element description"),
+        "tags": fields.List(fields.String, description="The element tags"),
+        "extras": fields.Raw(description="Extras attributes as key-value pairs"),
+        "element": fields.Nested(
+            api.model_reference, description="The element target object", allow_null=True
+        ),
+    },
+)
+
+element_page_fields = apiv2.model("TopicElementPage", fields.pager(element_fields))
+
+topic_input_fields = apiv2.clone(
+    "TopicInput",
+    topic_fields,
+    {
+        "elements": fields.List(fields.Nested(element_fields, description="The topic elements")),
+    },
+)

udata/core/topic/apiv2.py

@@ -1,110 +1,37 @@
 import logging
 
 import mongoengine
-from bson import ObjectId
-from flask import request, url_for
+from flask import request
 from flask_security import current_user
 
-from udata.api import API, apiv2, fields
-from udata.core.dataset.api import DatasetApiParser
-from udata.core.dataset.apiv2 import dataset_page_fields
-from udata.core.dataset.models import Dataset
-from udata.core.organization.api_fields import org_ref_fields
-from udata.core.reuse.api import ReuseApiParser
-from udata.core.reuse.models import Reuse
-from udata.core.spatial.api_fields import spatial_coverage_fields
-from udata.core.topic.models import Topic
-from udata.core.topic.parsers import TopicApiParser
+from udata.api import API, api, apiv2
+from udata.core.discussions.models import Discussion
+from udata.core.topic.api_fields import (
+    element_fields,
+    element_page_fields,
+    topic_fields,
+    topic_input_fields,
+    topic_page_fields,
+)
+from udata.core.topic.forms import TopicElementForm, TopicForm
+from udata.core.topic.models import Topic, TopicElement
+from udata.core.topic.parsers import TopicApiParser, TopicElementsParser
 from udata.core.topic.permissions import TopicEditPermission
-from udata.core.user.api_fields import user_ref_fields
+
+apiv2.inherit("ModelReference", api.model_reference)
 
 DEFAULT_SORTING = "-created_at"
-DEFAULT_PAGE_SIZE = 20
 
 log = logging.getLogger(__name__)
 
 ns = apiv2.namespace("topics", "Topics related operations")
 
 topic_parser = TopicApiParser()
-generic_parser = apiv2.page_parser()
-dataset_parser = DatasetApiParser()
-reuse_parser = ReuseApiParser()
+elements_parser = TopicElementsParser()
 
 common_doc = {"params": {"topic": "The topic ID"}}
 
 
-topic_fields = apiv2.model(
-    "Topic",
-    {
-        "id": fields.String(description="The topic identifier"),
-        "name": fields.String(description="The topic name", required=True),
-        "slug": fields.String(description="The topic permalink string", readonly=True),
-        "description": fields.Markdown(
-            description="The topic description in Markdown", required=True
-        ),
-        "tags": fields.List(
-            fields.String, description="Some keywords to help in search", required=True
-        ),
-        "datasets": fields.Raw(
-            attribute=lambda o: {
-                "rel": "subsection",
-                "href": url_for(
-                    "apiv2.topic_datasets",
-                    topic=o.id,
-                    page=1,
-                    page_size=DEFAULT_PAGE_SIZE,
-                    _external=True,
-                ),
-                "type": "GET",
-                "total": len(o.datasets),
-            },
-            description="Link to the topic datasets",
-        ),
-        "reuses": fields.Raw(
-            attribute=lambda o: {
-                "rel": "subsection",
-                "href": url_for(
-                    "apiv2.topic_reuses",
-                    topic=o.id,
-                    page=1,
-                    page_size=DEFAULT_PAGE_SIZE,
-                    _external=True,
-                ),
-                "type": "GET",
-                "total": len(o.reuses),
-            },
-            description="Link to the topic reuses",
-        ),
-        "featured": fields.Boolean(description="Is the topic featured"),
-        "private": fields.Boolean(description="Is the topic private"),
-        "created_at": fields.ISODateTime(description="The topic creation date", readonly=True),
-        "spatial": fields.Nested(
-            spatial_coverage_fields, allow_null=True, description="The spatial coverage"
-        ),
-        "last_modified": fields.ISODateTime(
-            description="The topic last modification date", readonly=True
-        ),
-        "organization": fields.Nested(
-            org_ref_fields,
-            allow_null=True,
-            description="The publishing organization",
-            readonly=True,
-        ),
-        "owner": fields.Nested(
-            user_ref_fields, description="The owner user", readonly=True, allow_null=True
-        ),
-        "uri": fields.String(
-            attribute=lambda t: url_for("api.topic", topic=t),
-            description="The topic API URI",
-            readonly=True,
-        ),
-        "extras": fields.Raw(description="Extras attributes as key-value pairs"),
-    },
-)
-
-topic_page_fields = apiv2.model("TopicPage", fields.pager(topic_fields))
-
-
 @ns.route("/", endpoint="topics_list")
 class TopicsAPI(API):
     @apiv2.expect(topic_parser.parser)
@@ -117,6 +44,16 @@ class TopicsAPI(API):
         sort = args["sort"] or ("$text_score" if args["q"] else None) or DEFAULT_SORTING
         return topics.order_by(sort).paginate(args["page"], args["page_size"])
 
+    @apiv2.secure
+    @apiv2.doc("create_topic")
+    @apiv2.expect(topic_input_fields)
+    @apiv2.marshal_with(topic_fields)
+    @apiv2.response(400, "Validation error")
+    def post(self):
+        """Create a topic"""
+        form = apiv2.validate(TopicForm)
+        return form.save(), 201
+
 
 @ns.route("/<topic:topic>/", endpoint="topic", doc=common_doc)
 @apiv2.response(404, "Topic not found")
@@ -127,37 +64,52 @@ class TopicAPI(API):
         """Get a given topic"""
         return topic
 
+    @apiv2.secure
+    @apiv2.doc("update_topic")
+    @apiv2.expect(topic_input_fields)
+    @apiv2.marshal_with(topic_fields)
+    @apiv2.response(400, "Validation error")
+    @apiv2.response(403, "Forbidden")
+    def put(self, topic):
+        """Update a given topic"""
+        if not TopicEditPermission(topic).can():
+            apiv2.abort(403, "Forbidden")
+        form = apiv2.validate(TopicForm, topic)
+        return form.save()
 
-topic_add_items_fields = apiv2.model(
-    "TopicItemsAdd",
-    {
-        "id": fields.String(description="Id of the item to add", required=True),
-    },
-    location="json",
-)
+    @apiv2.secure
+    @apiv2.doc("delete_topic")
+    @apiv2.response(204, "Object deleted")
+    @apiv2.response(403, "Forbidden")
+    def delete(self, topic):
+        """Delete a given topic"""
+        if not TopicEditPermission(topic).can():
+            apiv2.abort(403, "Forbidden")
+        # Remove discussions linked to the topic
+        Discussion.objects(subject=topic).delete()
+        topic.delete()
+        return "", 204
 
 
-@ns.route("/<topic:topic>/datasets/", endpoint="topic_datasets", doc=common_doc)
-class TopicDatasetsAPI(API):
-    @apiv2.doc("topic_datasets")
-    @apiv2.expect(dataset_parser.parser)
-    @apiv2.marshal_with(dataset_page_fields)
+@ns.route("/<topic:topic>/elements/", endpoint="topic_elements", doc=common_doc)
+class TopicElementsAPI(API):
+    @apiv2.doc("topic_elements")
+    @apiv2.expect(elements_parser.parser)
+    @apiv2.marshal_with(element_page_fields)
     def get(self, topic):
-        """Get a given topic datasets, with filters"""
-        args = dataset_parser.parse()
-        args["topic"] = topic.id
-        datasets = Dataset.objects(archived=None, deleted=None, private=False)
-        datasets = dataset_parser.parse_filters(datasets, args)
-        sort = args["sort"] or ("$text_score" if args["q"] else None) or "-created_at_internal"
-        return datasets.order_by(sort).paginate(args["page"], args["page_size"])
+        """Get a given topic's elements with pagination."""
+        args = elements_parser.parse()
+        elements = elements_parser.parse_filters(
+            topic.elements,
+            args,
+        )
+        return elements.paginate(args["page"], args["page_size"])
 
     @apiv2.secure
-    @apiv2.doc("topic_datasets_create")
-    @apiv2.expect([topic_add_items_fields])
+    @apiv2.doc("topic_elements_create")
+    @apiv2.expect([api.model_reference])
     @apiv2.marshal_with(topic_fields)
-    @apiv2.response(400, "Malformed object id(s) in request")
     @apiv2.response(400, "Expecting a list")
-    @apiv2.response(400, "Expecting a list of dicts with id attribute")
     @apiv2.response(404, "Topic not found")
     @apiv2.response(403, "Forbidden")
     def post(self, topic):
@@ -168,113 +120,83 @@ class TopicDatasetsAPI(API):
 
         if not isinstance(data, list):
             apiv2.abort(400, "Expecting a list")
-        if not all(isinstance(d, dict) and d.get("id") for d in data):
-            apiv2.abort(400, "Expecting a list of dicts with id attribute")
 
-        try:
-            datasets = Dataset.objects.filter(id__in=[d["id"] for d in data]).only("id")
-            diff = set(d.id for d in datasets) - set(d.id for d in topic.datasets)
-        except mongoengine.errors.ValidationError:
-            apiv2.abort(400, "Malformed object id(s) in request")
+        errors = []
+        elements = []
+        for element_data in data:
+            form = TopicElementForm.from_json(element_data, meta={"csrf": False})
+            if not form.validate():
+                errors.append(form.errors)
+            else:
+                element = TopicElement()
+                form.populate_obj(element)
+                element.save()
+                elements.append(element)
+
+        if errors:
+            apiv2.abort(400, errors=errors)
+
+        for element in elements:
+            element.topic = topic
+            element.save()
 
-        if diff:
-            topic.datasets += [ObjectId(did) for did in diff]
-            topic.save()
+        topic.save()
 
         return topic, 201
 
-
-@ns.route(
-    "/<topic:topic>/datasets/<dataset:dataset>/",
-    endpoint="topic_dataset",
-    doc={"params": {"topic": "The topic ID", "dataset": "The dataset ID"}},
-)
-class TopicDatasetAPI(API):
     @apiv2.secure
+    @apiv2.doc("topic_elements_delete")
     @apiv2.response(404, "Topic not found")
-    @apiv2.response(404, "Dataset not found in topic")
-    @apiv2.response(204, "Success")
-    def delete(self, topic, dataset):
-        """Delete a given dataset from the given topic"""
+    @apiv2.response(403, "Forbidden")
+    def delete(self, topic):
+        """Delete all elements from a Topic
+
+        This a workaround for https://github.com/kvesteri/wtforms-json/issues/43
+        -> we can't use PUT /api/2/topics/{topic}/ with an empty list of elements
+        """
         if not TopicEditPermission(topic).can():
             apiv2.abort(403, "Forbidden")
 
-        if dataset.id not in (d.id for d in topic.datasets):
-            apiv2.abort(404, "Dataset not found in topic")
-        topic.datasets = [d for d in topic.datasets if d.id != dataset.id]
-        topic.save()
+        topic.elements.delete()
 
         return None, 204
 
 
-@ns.route("/<topic:topic>/reuses/", endpoint="topic_reuses", doc=common_doc)
-class TopicReusesAPI(API):
-    @apiv2.doc("topic_reuses")
-    @apiv2.expect(reuse_parser.parser)
-    @apiv2.marshal_with(Reuse.__page_fields__)
-    def get(self, topic):
-        """Get a given topic reuses, with filters"""
-        args = reuse_parser.parse()
-        reuses = Reuse.objects(deleted=None, private__ne=True).filter(
-            id__in=[d.id for d in topic.reuses]
-        )
-        # warning: topic in reuse_parser is different from Topic
-        reuses = reuse_parser.parse_filters(reuses, args)
-        sort = args["sort"] or ("$text_score" if args["q"] else None) or DEFAULT_SORTING
-        return reuses.order_by(sort).paginate(args["page"], args["page_size"])
-
+@ns.route(
+    "/<topic:topic>/elements/<element_id>/",
+    endpoint="topic_element",
+    doc={"params": {"topic": "The topic ID", "element": "The element ID"}},
+)
+class TopicElementAPI(API):
     @apiv2.secure
-    @apiv2.doc("topic_reuses_create")
-    @apiv2.expect([topic_add_items_fields])
-    @apiv2.marshal_with(topic_fields)
-    @apiv2.response(400, "Malformed object id(s) in request")
-    @apiv2.response(400, "Expecting a list")
-    @apiv2.response(400, "Expecting a list of dicts with id attribute")
     @apiv2.response(404, "Topic not found")
-    @apiv2.response(403, "Forbidden")
-    def post(self, topic):
-        """Add reuses to a given topic from a list of reuses ids"""
+    @apiv2.response(404, "Element not found in topic")
+    @apiv2.response(204, "Success")
+    def delete(self, topic, element_id):
+        """Delete a given element from the given topic"""
         if not TopicEditPermission(topic).can():
             apiv2.abort(403, "Forbidden")
 
-        data = request.json
-
-        if not isinstance(data, list):
-            apiv2.abort(400, "Expecting a list")
-        if not all(isinstance(d, dict) and d.get("id") for d in data):
-            apiv2.abort(400, "Expecting a list of dicts with id attribute")
-
-        try:
-            reuses = Reuse.objects.filter(id__in=[r["id"] for r in data]).only("id")
-            diff = set(d.id for d in reuses) - set(d.id for d in topic.reuses)
-        except mongoengine.errors.ValidationError:
-            apiv2.abort(400, "Malformed object id(s) in request")
-
-        if diff:
-            topic.reuses += [ObjectId(rid) for rid in diff]
-            topic.save()
-
-        return topic, 201
+        element = TopicElement.objects.get_or_404(pk=element_id)
+        element.delete()
 
+        return None, 204
 
-@ns.route(
-    "/<topic:topic>/reuses/<reuse:reuse>/",
-    endpoint="topic_reuse",
-    doc={"params": {"topic": "The topic ID", "reuse": "The reuse ID"}},
-)
-class TopicReuseAPI(API):
     @apiv2.secure
+    @apiv2.doc("topic_element_update")
+    @apiv2.expect(element_fields)
+    @apiv2.marshal_with(element_fields)
     @apiv2.response(404, "Topic not found")
-    @apiv2.response(404, "Reuse not found in topic")
+    @apiv2.response(404, "Element not found in topic")
     @apiv2.response(204, "Success")
-    def delete(self, topic, reuse):
-        """Delete a given reuse from the given topic"""
+    def put(self, topic, element_id):
+        """Update a given element from the given topic"""
         if not TopicEditPermission(topic).can():
             apiv2.abort(403, "Forbidden")
 
-        if reuse.id not in (d.id for d in topic.reuses):
-            apiv2.abort(404, "Reuse not found in topic")
-        topic.reuses = [d for d in topic.reuses if d.id != reuse.id]
-        topic.save()
+        element = TopicElement.objects.get_or_404(pk=element_id)
+        form = apiv2.validate(TopicElementForm, element)
+        form.populate_obj(element)
+        element.save()
 
-        return None, 204
+        return element

udata/core/topic/factories.py

@@ -2,10 +2,40 @@ import factory
 
 from udata import utils
 from udata.core.dataset.factories import DatasetFactory
-from udata.core.reuse.factories import VisibleReuseFactory
+from udata.core.reuse.factories import ReuseFactory
 from udata.factories import ModelFactory
 
-from .models import Topic
+from .models import Topic, TopicElement
+
+
+class TopicElementFactory(ModelFactory):
+    class Meta:
+        model = TopicElement
+
+    title = factory.Faker("sentence")
+    description = factory.Faker("text")
+    topic = factory.SubFactory("udata.core.topic.factories.TopicFactory")
+
+    @classmethod
+    def element_as_payload(cls, elt) -> dict:
+        return {
+            "element": {"id": str(elt["element"].id), "class": elt["element"].__class__.__name__},
+            "title": elt["title"],
+            "description": elt["description"],
+        }
+
+    @classmethod
+    def as_payload(cls) -> dict:
+        elt = cls.as_dict()
+        return cls.element_as_payload(elt)
+
+
+class TopicElementDatasetFactory(TopicElementFactory):
+    element = factory.SubFactory(DatasetFactory)
+
+
+class TopicElementReuseFactory(TopicElementFactory):
+    element = factory.SubFactory(ReuseFactory)
 
 
 class TopicFactory(ModelFactory):
@@ -17,10 +47,41 @@ class TopicFactory(ModelFactory):
     tags = factory.LazyAttribute(lambda o: [utils.unique_string(16) for _ in range(3)])
     private = False
 
-    @factory.lazy_attribute
-    def datasets(self):
-        return DatasetFactory.create_batch(3)
 
-    @factory.lazy_attribute
-    def reuses(self):
-        return VisibleReuseFactory.create_batch(3)
+class TopicWithElementsFactory(TopicFactory):
+    """Factory that creates a topic with associated elements"""
+
+    @factory.post_generation
+    def elements(self, create, extracted, **kwargs):
+        if not create:
+            return
+        # Create associated elements
+        TopicElementDatasetFactory.create_batch(2, topic=self)
+        TopicElementReuseFactory.create(topic=self)
+
+    @classmethod
+    def elements_as_payload(cls, elements: list) -> dict:
+        return [
+            {
+                "element": {"id": str(elt.element.id), "class": elt.element.__class__.__name__},
+                "title": elt.title,
+                "description": elt.description,
+                "tags": elt.tags,
+                "extras": elt.extras,
+            }
+            for elt in elements
+        ]
+
+    @classmethod
+    def as_payload(cls) -> dict:
+        # Build topic without saving
+        topic = cls.build()
+        payload = topic.to_dict()
+        # Build elements without saving, but create datasets/reuses for valid references
+        elements = [
+            TopicElementDatasetFactory.build(topic=topic, element=DatasetFactory.create()),
+            TopicElementDatasetFactory.build(topic=topic, element=DatasetFactory.create()),
+            TopicElementReuseFactory.build(topic=topic, element=ReuseFactory.create()),
+        ]
+        payload["elements"] = cls.elements_as_payload(elements)
+        return payload