udata 7.0.4.dev27543__py2.py3-none-any.whl → 7.0.4.dev27593__py2.py3-none-any.whl

udata/core/dataset/api.py

@@ -19,6 +19,7 @@ These changes might lead to backward compatibility breakage meaning:
 
 import os
 import logging
+import mongoengine
 from datetime import datetime
 
 from bson.objectid import ObjectId
@@ -229,7 +230,12 @@ class DatasetAPI(API):
         DatasetEditPermission(dataset).test()
         dataset.last_modified_internal = datetime.utcnow()
         form = api.validate(DatasetForm, dataset)
-        return form.save()
+        # As validation for some fields (ie. extras) is at model
+        # level instead form level, we use mongoengine errors here.
+        try:
+            return form.save()
+        except mongoengine.errors.ValidationError as e:
+            api.abort(400, e.message)
 
     @api.secure
     @api.doc('delete_dataset')

udata/core/dataset/apiv2.py

@@ -1,4 +1,5 @@
 import logging
+import mongoengine
 
 from flask import url_for, request, abort
 from flask_restx import marshal
@@ -235,7 +236,10 @@ class DatasetExtrasAPI(API):
             data.pop(key)
         # then update the extras with the remaining payload
         dataset.extras.update(data)
-        dataset.save(signal_kwargs={'ignores': ['post_save']})
+        try:
+            dataset.save(signal_kwargs={'ignores': ['post_save']})
+        except mongoengine.errors.ValidationError as e:
+            apiv2.abort(400, e.message)
         return dataset.extras
 
     @apiv2.secure
@@ -248,11 +252,11 @@ class DatasetExtrasAPI(API):
         if dataset.deleted:
             apiv2.abort(410, 'Dataset has been deleted')
         DatasetEditPermission(dataset).test()
-        try:
-            for key in data:
+        for key in data:
+            try:
                 del dataset.extras[key]
-        except KeyError:
-            apiv2.abort(404, 'Key not found in existing extras')
+            except KeyError:
+                pass
         dataset.save(signal_kwargs={'ignores': ['post_save']})
         return dataset.extras, 204
 

udata/core/dataset/models.py

@@ -11,6 +11,7 @@ from flask import current_app
 from mongoengine import DynamicEmbeddedDocument, ValidationError as MongoEngineValidationError
 from mongoengine.signals import pre_save, post_save
 from mongoengine.fields import DateTimeField
+from pydoc import locate
 from stringdist import rdlevenshtein
 from werkzeug.utils import cached_property
 import requests
@@ -553,6 +554,31 @@ class Dataset(WithMetrics, BadgeMixin, db.Owned, db.Document):
         if self.frequency in LEGACY_FREQUENCIES:
             self.frequency = LEGACY_FREQUENCIES[self.frequency]
 
+        for key, value in self.extras.items():
+            if not key.startswith('custom:'):
+                continue
+            if not self.organization:
+                raise MongoEngineValidationError(
+                    'Custom metadatas are only accessible to dataset owned by on organization.')
+            custom_meta = key.split(':')[1]
+            org_custom = self.organization.extras.get('custom', [])
+            custom_present = False
+            for custom in org_custom:
+                if custom['title'] != custom_meta:
+                    continue
+                custom_present = True
+                if custom['type'] == 'choice':
+                    if value not in custom['choices']:
+                        raise MongoEngineValidationError(
+                            'Custom metadata choice is not defined by organization.')
+                else:
+                    if not isinstance(value, locate(custom['type'])):
+                        raise MongoEngineValidationError(
+                            'Custom metadata is not of the right type.')
+            if not custom_present:
+                raise MongoEngineValidationError(
+                    'Dataset\'s organization did not define the requested custom metadata.')
+
     def url_for(self, *args, **kwargs):
         return endpoint_for('datasets.show', 'api.dataset', dataset=self, *args, **kwargs)
 

udata/core/discussions/api.py

@@ -14,10 +14,8 @@ from udata.core.user.api_fields import user_ref_fields
 from .forms import DiscussionCreateForm, DiscussionCommentForm
 from .models import Message, Discussion
 from .permissions import CloseDiscussionPermission
-from .signals import (
-    on_new_discussion, on_new_discussion_comment, on_discussion_closed,
-    on_discussion_deleted
-)
+from .signals import on_discussion_deleted
+
 
 ns = api.namespace('discussions', 'Discussion related operations')
 

udata/core/discussions/tasks.py

@@ -16,8 +16,10 @@ log = get_logger(__name__)
 def owner_recipients(discussion):
     if getattr(discussion.subject, 'organization', None):
         return [m.user for m in discussion.subject.organization.members]
-    else:
+    elif getattr(discussion.subject, 'owner', None):
         return [discussion.subject.owner]
+    else:
+        return []
 
 
 @connect(on_new_discussion, by_id=True)

udata/core/organization/api_fields.py

@@ -87,6 +87,7 @@ org_fields = api.model('Organization', {
     'badges': fields.List(fields.Nested(badge_fields),
                           description='The organization badges',
                           readonly=True),
+    'extras': fields.Raw(description='Extras attributes as key-value pairs'),
 })
 
 org_page_fields = api.model('OrganizationPage', fields.pager(org_fields))

udata/core/organization/apiv2.py

@@ -1,11 +1,15 @@
+import mongoengine
 from flask import request
 
 from udata import search
 from udata.api import apiv2, API
 from udata.utils import multi_to_dict
+from udata.core.contact_point.api_fields import contact_point_fields
 from .search import OrganizationSearch
 from .api_fields import org_page_fields, org_fields, member_fields
-from udata.core.contact_point.api_fields import contact_point_fields
+from .permissions import (
+    EditOrganizationPermission, OrganizationPrivatePermission
+)
 
 apiv2.inherit('OrganizationPage', org_page_fields)
 apiv2.inherit('Organization', org_fields)
@@ -29,3 +33,58 @@ class OrganizationSearchAPI(API):
         '''Search all organizations'''
         search_parser.parse_args()
         return search.query(OrganizationSearch, **multi_to_dict(request.args))
+
+
+@ns.route('/<org:org>/extras/', endpoint='organization_extras')
+@apiv2.response(400, 'Wrong payload format, dict expected')
+@apiv2.response(400, 'Wrong payload format, list expected')
+@apiv2.response(404, 'Organization not found')
+@apiv2.response(410, 'Organization has been deleted')
+class OrganizationExtrasAPI(API):
+    @apiv2.doc('get_organization_extras')
+    def get(self, org):
+        '''Get an organization extras given its identifier'''
+        if org.deleted:
+            apiv2.abort(410, 'Organization has been deleted')
+        return org.extras
+
+    @apiv2.secure
+    @apiv2.doc('update_organization_extras')
+    def put(self, org):
+        '''Update a given organization extras'''
+        data = request.json
+        if not isinstance(data, dict):
+            apiv2.abort(400, 'Wrong payload format, dict expected')
+        if org.deleted:
+            apiv2.abort(410, 'Organization has been deleted')
+        EditOrganizationPermission(org).test()
+        # first remove extras key associated to a None value in payload
+        for key in [k for k in data if data[k] is None]:
+            org.extras.pop(key, None)
+            data.pop(key)
+
+        # then update the extras with the remaining payload
+        org.extras.update(data)
+        try:
+            org.save()
+        except mongoengine.errors.ValidationError as e:
+            apiv2.abort(400, e.message)
+        return org.extras
+
+    @apiv2.secure
+    @apiv2.doc('delete_organization_extras')
+    def delete(self, org):
+        '''Delete a given organization extras key on a given organization'''
+        data = request.json
+        if not isinstance(data, list):
+            apiv2.abort(400, 'Wrong payload format, list expected')
+        if org.deleted:
+            apiv2.abort(410, 'Organization has been deleted')
+        EditOrganizationPermission(org).test()
+        for key in data:
+            try:
+                del org.extras[key]
+            except KeyError:
+                pass
+        org.save()
+        return org.extras, 204

udata/core/organization/models.py

@@ -114,7 +114,7 @@ class Organization(WithMetrics, BadgeMixin, db.Datetimed, db.Document):
 
     ext = db.MapField(db.GenericEmbeddedDocumentField())
     zone = db.StringField()
-    extras = db.ExtrasField()
+    extras = db.OrganizationExtrasField()
 
     deleted = db.DateTimeField()
 

udata/harvest/backends/base.py

@@ -156,7 +156,7 @@ class BaseBackend(object):
             self.job.errors.append(error)
             self.job.status = 'failed'
             self.end()
-            return
+            return None
         except Exception as e:
             self.job.status = 'failed'
             error = HarvestError(message=safe_unicode(e))
@@ -164,7 +164,7 @@ class BaseBackend(object):
             self.end()
             msg = 'Initialization failed for "{0.name}" ({0.backend})'
             log.exception(msg.format(self.source))
-            return
+            return None
 
         if self.max_items:
             self.job.items = self.job.items[:self.max_items]

udata/harvest/models.py

@@ -94,7 +94,7 @@ class HarvestSource(db.Owned, db.Document):
                         populate_from='name', update=True)
     description = db.StringField()
     url = db.StringField(required=True)
-    backend = db.StringField()
+    backend = db.StringField(required=True)
     config = db.DictField()
     periodic_task = db.ReferenceField('PeriodicTask',
                                       reverse_delete_rule=db.NULLIFY)

udata/harvest/tasks.py

@@ -19,15 +19,18 @@ def harvest(self, ident):
     Backend = backends.get(current_app, source.backend)
     backend = Backend(source)
     items = backend.perform_initialization()
-    if items > 0:
+    if items is None:
+        pass
+    elif items == 0:
+        backend.finalize()
+    else:
         finalize = harvest_job_finalize.s(backend.job.id)
         items = [
             harvest_job_item.s(backend.job.id, item.remote_id)
             for item in backend.job.items
         ]
         chord(items)(finalize)
-    elif items == 0:
-        backend.finalize()
+    
 
 
 @task(ignore_result=False, route='low.harvest')

udata/harvest/tests/factories.py

@@ -22,6 +22,7 @@ class HarvestSourceFactory(ModelFactory):
     name = factory.Faker('name')
     url = factory.Faker('url')
     description = factory.Faker('text')
+    backend = 'factory'
 
 
 class HarvestJobFactory(ModelFactory):

udata/harvest/tests/test_models.py

@@ -12,7 +12,7 @@ log = logging.getLogger(__name__)
 @pytest.mark.usefixtures('clean_db')
 class HarvestSourceTest:
     def test_defaults(self):
-        source = HarvestSource.objects.create(name='Test', url=faker.url())
+        source = HarvestSource.objects.create(name='Test', url=faker.url(), backend='factory')
         assert source.name == 'Test'
         assert source.slug == 'test'
 

udata/models/__init__.py

@@ -17,7 +17,7 @@ from udata.errors import ConfigError
 from .badges_field import BadgesField
 from .taglist_field import TagListField
 from .datetime_fields import DateField, DateRange, Datetimed
-from .extras_fields import ExtrasField
+from .extras_fields import ExtrasField, OrganizationExtrasField
 from .slug_fields import SlugField
 from .url_field import URLField
 from .uuid_fields import AutoUUIDField
@@ -36,6 +36,7 @@ class UDataMongoEngine(MongoEngine):
         self.DateField = DateField
         self.Datetimed = Datetimed
         self.ExtrasField = ExtrasField
+        self.OrganizationExtrasField = OrganizationExtrasField
         self.SlugField = SlugField
         self.AutoUUIDField = AutoUUIDField
         self.Document = UDataDocument

udata/models/extras_fields.py

@@ -60,3 +60,44 @@ class ExtrasField(DictField):
         if isinstance(value, EmbeddedDocument):
             return value
         return super(ExtrasField, self).to_python(value)
+
+
+class OrganizationExtrasField(ExtrasField):
+    def __init__(self, **kwargs):
+        super(OrganizationExtrasField, self).__init__()
+
+    def validate(self, values):
+        super(ExtrasField, self).validate(values)
+
+        errors = {}
+
+        mandatory_keys = ["title", "description", "type"]
+        optional_keys = ["choices"]
+        valid_types = ["str", "int", "float", "bool", "datetime", "date", "choice"]
+
+        for elem in values.get('custom', []):
+            # Check if all mandatory keys are in the dictionary
+            if not all(key in elem for key in mandatory_keys):
+                errors['custom'] = 'The dictionary does not contain the mandatory keys: \'title\', \'description\', \'type\'.'
+
+            # Check if the dictionary contains only keys that are either mandatory or optional
+            if not all(key in mandatory_keys + optional_keys for key in elem):
+                errors['custom'] = 'The dictionary does contains extra keys than allowed ones.'
+
+            # Check if the "type" value is one of the valid types
+            if elem.get("type") not in valid_types:
+                errors['type'] = ('Type \'{type}\' of \'{title}\' should be one of: {types}'
+                                  .format(type=elem.get("type"), title=elem.get("title"), types=valid_types))
+
+            # Check if the "choices" key is present only if the type is "choice" and it's not an empty list
+            is_choices_valid = True
+            if elem.get("type") == "choice":
+                is_choices_valid = "choices" in elem and isinstance(elem["choices"], list) and len(
+                    elem["choices"]) > 0
+            elif "choices" in elem:
+                is_choices_valid = False
+            if not is_choices_valid:
+                errors['choices'] = 'The \'choices\' key must be an non empty list and can only be present when type \'choice\' is selected.'
+
+        if errors:
+            self.error('Custom extras error', errors=errors)

udata/settings.py

@@ -470,8 +470,7 @@ class Testing(object):
     WTF_CSRF_ENABLED = False
     AUTO_INDEX = False
     CELERY_TASK_ALWAYS_EAGER = True
-    # TODO: ideally, this should be set to True in order to reveal exceptions in delayed tasks
-    CELERY_TASK_EAGER_PROPAGATES = False
+    CELERY_TASK_EAGER_PROPAGATES = True
     TEST_WITH_PLUGINS = False
     PLUGINS = []
     TEST_WITH_THEME = False