udata 6.2.1.dev26892__py2.py3-none-any.whl → 6.2.1.dev26916__py2.py3-none-any.whl

udata/core/dataset/rdf.py

@@ -424,6 +424,7 @@ def resource_from_rdf(graph_or_distrib, dataset=None):
         resource = Resource()
         if dataset:
             dataset.resources.append(resource)
+    resource.filetype = 'remote'
     resource.title = title_from_rdf(distrib, url)
     resource.url = url
     resource.description = sanitize_html(distrib.value(DCT.description))

udata/core/topic/api.py

@@ -23,9 +23,15 @@ topic_fields = api.model('Topic', {
     'tags': fields.List(
         fields.String, description='Some keywords to help in search', required=True),
     'datasets': fields.List(
-        fields.Nested(dataset_fields), description='The topic datasets'),
+        fields.Nested(dataset_fields),
+        description='The topic datasets',
+        attribute=lambda o: [d.fetch() for d in o.datasets],
+    ),
     'reuses': fields.List(
-        fields.Nested(reuse_fields), description='The topic reuses'),
+        fields.Nested(reuse_fields),
+        description='The topic reuses',
+        attribute=lambda o: [r.fetch() for r in o.reuses],
+    ),
     'featured': fields.Boolean(description='Is the topic featured'),
     'private': fields.Boolean(description='Is the topic private'),
     'created_at': fields.ISODateTime(
@@ -43,7 +49,7 @@ topic_fields = api.model('Topic', {
         'topics.display', lambda o: {'topic': o},
         description='The topic page URL', readonly=True, fallback_endpoint='api.topic'),
     'extras': fields.Raw(description='Extras attributes as key-value pairs'),
-}, mask='*,datasets{id,title,uri,page},reuses{id,title, image, image_thumbnail,uri,page}')
+}, mask='*,datasets{id,title,uri,page},reuses{id,title,image,image_thumbnail,uri,page}')
 
 topic_page_fields = api.model('TopicPage', fields.pager(topic_fields))
 
@@ -52,6 +58,10 @@ topic_parser = TopicApiParser()
 
 @ns.route('/', endpoint='topics')
 class TopicsAPI(API):
+    """
+    Warning: querying a list with a topic containing a lot of related objects (datasets, reuses)
+    will fail/take a lot of time because every object is dereferenced. Use api v2 if you can.
+    """
 
     @api.doc('list_topics')
     @api.expect(topic_parser.parser)
@@ -79,6 +89,11 @@ class TopicsAPI(API):
 @api.param('topic', 'The topic ID or slug')
 @api.response(404, 'Object not found')
 class TopicAPI(API):
+    """
+    Warning: querying a topic containing a lot of related objects (datasets, reuses)
+    will fail/take a lot of time because every object is dereferenced. Use api v2 if you can.
+    """
+
     @api.doc('get_topic')
     @api.marshal_with(topic_fields)
     def get(self, topic):

udata/core/topic/apiv2.py

@@ -2,6 +2,7 @@ import logging
 
 import mongoengine
 
+from bson import ObjectId
 from flask import url_for, request
 
 from udata.api import apiv2, API, fields
@@ -131,22 +132,8 @@ class TopicDatasetsAPI(API):
     @apiv2.response(400, 'Expecting a list')
     @apiv2.response(400, 'Expecting a list of dicts with id attribute')
     @apiv2.response(404, 'Topic not found')
-    @apiv2.response(404, 'Dataset(s) not found')
     @apiv2.response(403, 'Forbidden')
     def post(self, topic):
-        '''Add datasets to a given topic from a list of dataset ids'''
-        def add_dataset(topic, dataset):
-            if dataset not in topic.datasets:
-                topic.datasets.append(dataset)
-            return topic
-
-        def get_dataset(dataset_id):
-            try:
-                dataset = Dataset.objects.get_or_404(id=dataset_id)
-            except mongoengine.errors.ValidationError:
-                apiv2.abort(400, 'Malformed object id(s) in request')
-            return dataset
-
         if not TopicEditPermission(topic).can():
             apiv2.abort(403, 'Forbidden')
 
@@ -157,10 +144,15 @@ class TopicDatasetsAPI(API):
         if not all(isinstance(d, dict) and d.get('id') for d in data):
             apiv2.abort(400, 'Expecting a list of dicts with id attribute')
 
-        datasets = (get_dataset(d['id']) for d in data)
-        for dataset in datasets:
-            topic = add_dataset(topic, dataset)
-        topic.save()
+        try:
+            datasets = Dataset.objects.filter(id__in=[d['id'] for d in data]).only('id')
+            diff = set(d.id for d in datasets) - set(d.id for d in topic.datasets)
+        except mongoengine.errors.ValidationError:
+            apiv2.abort(400, 'Malformed object id(s) in request')
+
+        if diff:
+            topic.datasets += [ObjectId(did) for did in diff]
+            topic.save()
 
         return topic, 201
 
@@ -210,22 +202,9 @@ class TopicReusesAPI(API):
     @apiv2.response(400, 'Expecting a list')
     @apiv2.response(400, 'Expecting a list of dicts with id attribute')
     @apiv2.response(404, 'Topic not found')
-    @apiv2.response(404, 'Reuse(s) not found')
     @apiv2.response(403, 'Forbidden')
     def post(self, topic):
         '''Add reuses to a given topic from a list of reuses ids'''
-        def add_reuse(topic, reuse):
-            if reuse.id not in (r.id for r in topic.reuses):
-                topic.reuses.append(reuse)
-            return topic
-
-        def get_reuse(reuse_id):
-            try:
-                reuse = Reuse.objects.get_or_404(id=reuse_id)
-            except mongoengine.errors.ValidationError:
-                apiv2.abort(400, 'Malformed object id(s) in request')
-            return reuse
-
         if not TopicEditPermission(topic).can():
             apiv2.abort(403, 'Forbidden')
 
@@ -236,10 +215,15 @@ class TopicReusesAPI(API):
         if not all(isinstance(d, dict) and d.get('id') for d in data):
             apiv2.abort(400, 'Expecting a list of dicts with id attribute')
 
-        reuses = (get_reuse(d['id']) for d in data)
-        for reuse in reuses:
-            topic = add_reuse(topic, reuse)
-        topic.save()
+        try:
+            reuses = Reuse.objects.filter(id__in=[r['id'] for r in data]).only('id')
+            diff = set(d.id for d in reuses) - set(d.id for d in topic.reuses)
+        except mongoengine.errors.ValidationError:
+            apiv2.abort(400, 'Malformed object id(s) in request')
+
+        if diff:
+            topic.reuses += [ObjectId(rid) for rid in diff]
+            topic.save()
 
         return topic, 201
 

udata/core/topic/models.py

@@ -20,9 +20,9 @@ class Topic(db.Document, db.Owned):
 
     tags = db.ListField(db.StringField())
     datasets = db.ListField(
-        db.ReferenceField('Dataset', reverse_delete_rule=db.PULL))
+        db.LazyReferenceField('Dataset', reverse_delete_rule=db.PULL))
     reuses = db.ListField(
-        db.ReferenceField('Reuse', reverse_delete_rule=db.PULL))
+        db.LazyReferenceField('Reuse', reverse_delete_rule=db.PULL))
 
     featured = db.BooleanField()
     private = db.BooleanField()
@@ -54,7 +54,7 @@ class Topic(db.Document, db.Owned):
         except cls.DoesNotExist:
             datasets_list_dif = document.datasets
         for dataset in datasets_list_dif:
-            reindex.delay(*as_task_param(dataset))
+            reindex.delay(*as_task_param(dataset.fetch()))
 
     @property
     def display_url(self):

udata/harvest/tests/test_dcat_backend.py

@@ -300,6 +300,7 @@ class DcatBackendTest:
         assert len(dataset.resources) == 2
 
         resource_1 = next(res for res in dataset.resources if res.title == 'Resource 1-1')
+        assert resource_1.filetype == 'remote'
         # Format is a IANA URI
         assert resource_1.format == 'json'
         assert resource_1.mime == 'application/json'

udata/models/__init__.py

@@ -1,4 +1,3 @@
-import importlib
 import logging
 import warnings
 

udata/settings.py

@@ -461,6 +461,8 @@ class Testing(object):
     WTF_CSRF_ENABLED = False
     AUTO_INDEX = False
     CELERY_TASK_ALWAYS_EAGER = True
+    # TODO: ideally, this should be set to True in order to reveal exceptions in delayed tasks
+    CELERY_TASK_EAGER_PROPAGATES = False
     TEST_WITH_PLUGINS = False
     PLUGINS = []
     TEST_WITH_THEME = False