udata 6.1.6.dev25761__py2.py3-none-any.whl → 6.1.7__py2.py3-none-any.whl

udata/__init__.py

@@ -4,5 +4,5 @@
 udata
 '''
 
-__version__ = '6.1.6.dev'
+__version__ = '6.1.7'
 __description__ = 'Open data portal'

udata/core/dataset/api_fields.py

@@ -73,6 +73,13 @@ frequency_fields = api.model('Frequency', {
     'label': fields.String(description='The frequency display name')
 })
 
+resource_internal_fields = api.model('ResourceInternals', {
+    'created_at_internal': fields.ISODateTime(
+        description='The resource\'s internal creation date on the site', required=True),
+    'last_modified_internal': fields.ISODateTime(
+        description='The resource\'s internal last modification date', required=True),
+})
+
 resource_fields = api.model('Resource', {
     'id': fields.String(description='The resource unique ID', readonly=True),
     'title': fields.String(description='The resource title', required=True),
@@ -112,6 +119,8 @@ resource_fields = api.model('Resource', {
                                  'new page)',
                                  readonly=True),
     'schema': fields.Raw(description='Reference to the associated schema', readonly=True),
+    'internal': fields.Nested(
+        resource_internal_fields, readonly=True, description='Site internal and specific object\'s data'),
 })
 
 upload_fields = api.inherit('UploadedResource', resource_fields, {
@@ -162,9 +171,16 @@ DEFAULT_MASK = ','.join((
     'id', 'title', 'acronym', 'slug', 'description', 'created_at', 'last_modified', 'deleted',
     'private', 'tags', 'badges', 'resources', 'frequency', 'frequency_date', 'extras', 'harvest',
     'metrics', 'organization', 'owner', 'temporal_coverage', 'spatial', 'license',
-    'uri', 'page', 'last_update', 'archived', 'quality'
+    'uri', 'page', 'last_update', 'archived', 'quality', 'internal'
 ))
 
+dataset_internal_fields = api.model('DatasetInternals', {
+    'created_at_internal': fields.ISODateTime(
+        description='The dataset\'s internal creation date on the site', required=True),
+    'last_modified_internal': fields.ISODateTime(
+        description='The dataset\'s internal last modification date', required=True),
+})
+
 dataset_fields = api.model('Dataset', {
     'id': fields.String(description='The dataset identifier', readonly=True),
     'title': fields.String(description='The dataset title', required=True),
@@ -174,7 +190,7 @@ dataset_fields = api.model('Dataset', {
     'description': fields.Markdown(
         description='The dataset description in markdown', required=True),
     'created_at': fields.ISODateTime(
-        description='The dataset creation date', required=True),
+        description='This date is computed between harvested creation date if any and site\'s internal creation date' , required=True),
     'last_modified': fields.ISODateTime(
         description='The dataset last modification date', required=True),
     'deleted': fields.ISODateTime(description='The deletion date if deleted'),
@@ -228,6 +244,8 @@ dataset_fields = api.model('Dataset', {
     'quality': fields.Raw(description='The dataset quality', readonly=True),
     'last_update': fields.ISODateTime(
         description='The resources last modification date', required=True),
+    'internal': fields.Nested(
+        dataset_internal_fields, readonly=True, description='Site internal and specific object\'s data'),
 }, mask=DEFAULT_MASK)
 
 dataset_page_fields = api.model('DatasetPage', fields.pager(dataset_fields),

udata/core/dataset/apiv2.py

@@ -16,7 +16,9 @@ from .api_fields import (
     user_ref_fields,
     checksum_fields,
     dataset_harvest_fields,
-    resource_harvest_fields
+    dataset_internal_fields,
+    resource_harvest_fields,
+    resource_internal_fields
 )
 from udata.core.spatial.api_fields import geojson
 from .models import (
@@ -33,7 +35,7 @@ DEFAULT_MASK_APIV2 = ','.join((
     'id', 'title', 'acronym', 'slug', 'description', 'created_at', 'last_modified', 'deleted',
     'private', 'tags', 'badges', 'resources', 'community_resources', 'frequency', 'frequency_date',
     'extras', 'metrics', 'organization', 'owner', 'temporal_coverage', 'spatial', 'license',
-    'uri', 'page', 'last_update', 'archived', 'quality', 'harvest'
+    'uri', 'page', 'last_update', 'archived', 'quality', 'harvest', 'internal'
 ))
 
 log = logging.getLogger(__name__)
@@ -129,6 +131,8 @@ dataset_fields = apiv2.model('Dataset', {
     'quality': fields.Raw(description='The dataset quality', readonly=True),
     'last_update': fields.ISODateTime(
         description='The resources last modification date', required=True),
+    'internal': fields.Nested(
+        dataset_internal_fields, readonly=True, description='Site internal and specific object\'s data')
 }, mask=DEFAULT_MASK_APIV2)
 
 
@@ -164,6 +168,8 @@ apiv2.inherit('GeoJSON', geojson)
 apiv2.inherit('Checksum', checksum_fields)
 apiv2.inherit('HarvestDatasetMetadata', dataset_harvest_fields)
 apiv2.inherit('HarvestResourceMetadata', resource_harvest_fields)
+apiv2.inherit('DatasetInternals', dataset_internal_fields)
+apiv2.inherit('ResourceInternals', resource_internal_fields)
 
 
 @ns.route('/search/', endpoint='dataset_search')

udata/core/dataset/csv.py

@@ -73,6 +73,7 @@ class ResourcesCsvAdapter(csv.NestedAdapter):
         'title',
         'description',
         'filetype',
+        'type',
         'format',
         'mime',
         'filesize',

udata/core/dataset/models.py

@@ -294,6 +294,13 @@ class ResourceMixin(object):
     last_modified_internal = db.DateTimeField(default=datetime.utcnow, required=True)
     deleted = db.DateTimeField()
 
+    @property
+    def internal(self):
+        return {
+            'created_at_internal': self.created_at_internal,
+            'last_modified_internal': self.last_modified_internal
+        }
+
     @property
     def created_at(self):
         return self.harvest.created_at if self.harvest and self.harvest.created_at else self.created_at_internal
@@ -827,6 +834,13 @@ class Dataset(WithMetrics, BadgeMixin, db.Owned, db.Document):
 
         return result
 
+    @property
+    def internal(self):
+        return {
+            'created_at_internal': self.created_at_internal,
+            'last_modified_internal': self.last_modified_internal
+        }
+
     @property
     def views_count(self):
         return self.metrics.get('views', 0)

udata/core/dataset/rdf.py

@@ -319,15 +319,15 @@ def frequency_from_rdf(term):
         except uris.ValidationError:
             pass
     if isinstance(term, Literal):
-        if term.toPython() in UPDATE_FREQUENCIES:
-            return term.toPython()
+        if term.toPython().lower() in UPDATE_FREQUENCIES:
+            return term.toPython().lower()
     if isinstance(term, RdfResource):
         term = term.identifier
     if isinstance(term, URIRef):
         if EUFREQ in term:
             return EU_RDF_REQUENCIES.get(term)
         _, _, freq = namespace_manager.compute_qname(term)
-        return freq
+        return freq.lower()
 
 
 def mime_from_rdf(resource):

udata/core/dataset/search.py

@@ -1,6 +1,6 @@
 import datetime
 from udata.models import (
-    Dataset, Organization, User, GeoZone, License
+    Dataset, Organization, User, GeoZone, License, Topic
 )
 from udata.search import (
     ModelSearchAdapter, register,
@@ -41,6 +41,7 @@ class DatasetSearch(ModelSearchAdapter):
         'schema': Filter(),
         'temporal_coverage': TemporalCoverageFilter(),
         'featured': BoolFilter(),
+        'topic': ModelTermsFilter(model=Topic),
     }
 
     @classmethod
@@ -63,6 +64,8 @@ class DatasetSearch(ModelSearchAdapter):
         organization = None
         owner = None
 
+        topics = Topic.objects(datasets=dataset)
+
         if dataset.organization:
             org = Organization.objects(id=dataset.organization.id).first()
             organization = {
@@ -94,7 +97,8 @@ class DatasetSearch(ModelSearchAdapter):
             'organization': organization,
             'owner': str(owner.id) if owner else None,
             'format': [r.format.lower() for r in dataset.resources if r.format],
-            'schema': [r.schema.get('name') for r in dataset.resources if r.schema]
+            'schema': [r.schema.get('name') for r in dataset.resources if r.schema],
+            'topics': [t.slug for t in topics if topics],
         }
         extras = {}
         for key, value in dataset.extras.items():

udata/core/discussions/api.py

@@ -101,11 +101,14 @@ class DiscussionAPI(API):
     @api.secure
     @api.doc('comment_discussion')
     @api.expect(comment_discussion_fields)
-    @api.response(403, 'Not allowed to close this discussion')
+    @api.response(403, 'Not allowed to close this discussion '
+                       "OR can't add comments on a closed discussion")
     @api.marshal_with(discussion_fields)
     def post(self, id):
         '''Add comment and optionally close a discussion given its ID'''
         discussion = Discussion.objects.get_or_404(id=id_or_404(id))
+        if discussion.closed:
+            api.abort(403, "Can't add comments on a closed discussion")
         form = api.validate(DiscussionCommentForm)
         message = Message(
             content=form.comment.data,

udata/core/discussions/tasks.py

@@ -41,7 +41,7 @@ def notify_new_discussion_comment(discussion_id, message=None):
     if isinstance(discussion.subject, (Dataset, Reuse, Post)):
         recipients = owner_recipients(discussion) + [
             m.posted_by for m in discussion.discussion]
-        recipients = [u for u in set(recipients) if u != message.posted_by]
+        recipients = list({u.id: u for u in recipients if u != message.posted_by}.values())
         subject = _('%(user)s commented your discussion',
                     user=message.posted_by.fullname)
 
@@ -59,7 +59,7 @@ def notify_discussion_closed(discussion_id, message=None):
     if isinstance(discussion.subject, (Dataset, Reuse, Post)):
         recipients = owner_recipients(discussion) + [
             m.posted_by for m in discussion.discussion]
-        recipients = [u for u in set(recipients) if u != message.posted_by]
+        recipients = list({u.id: u for u in recipients if u != message.posted_by}.values())
         subject = _('A discussion has been closed')
         mail.send(subject, recipients, 'discussion_closed',
                   discussion=discussion, message=message)

udata/core/organization/api_fields.py

@@ -61,6 +61,7 @@ org_fields = api.model('Organization', {
         required=True),
     'description': fields.Markdown(
         description='The organization description in Markdown', required=True),
+    'business_number_id': fields.String(description='The organization\'s business identification number.'),
     'created_at': fields.ISODateTime(
         description='The organization creation date', readonly=True),
     'last_modified': fields.ISODateTime(

udata/core/organization/forms.py

@@ -1,3 +1,5 @@
+from flask import current_app
+
 from udata.auth import current_user
 from udata.forms import Form, ModelForm, fields, validators
 from udata.i18n import lazy_gettext as _
@@ -15,6 +17,30 @@ __all__ = (
 )
 
 
+def org_bid_check(form, field):
+    if field.data:
+        # EID checks are country dependant. Following one is suitable for France.
+        if current_app.config.get('ORG_BID_FORMAT') == 'siret':
+            siret_number = str(field.data)
+            # Length control done here instead of using WTForm validator because field must remain optional.
+            if len(siret_number) != 14:
+                raise validators.ValidationError(_('A siret number is made of 14 digits'))
+            # Siret exception for the french postal service which business number id does not match the algorithm.
+            if siret_number == '35600000000048':
+                return
+            # Checksum verification on only the SIREN part, the 9 first digits.
+            try:
+                chiffres = [int(chiffre) for chiffre in siret_number[:9]]
+                chiffres[1::2] = [chiffre * 2 for chiffre in chiffres[1::2]]
+                chiffres = [chiffre - 9 if chiffre > 9 else chiffre for chiffre in chiffres]
+                total = sum(chiffres)
+            except ValueError:
+                raise validators.ValidationError(_('A siret number is only made of digits'))
+
+            if not total % 10 == 0:
+                raise validators.ValidationError(_('Invalid Siret number'))
+
+
 class OrganizationForm(ModelForm):
     model_class = Organization
 
@@ -27,6 +53,8 @@ class OrganizationForm(ModelForm):
     url = fields.URLField(
         _('Website'), description=_('The organization website URL'))
     logo = fields.ImageField(_('Logo'), sizes=LOGO_SIZES)
+    business_number_id = fields.StringField(_('Business id'), [org_bid_check],
+                                            description=_('Business identification number'))
 
     deleted = fields.DateTimeField()
     extras = fields.ExtrasField()

udata/core/organization/models.py

@@ -40,6 +40,8 @@ CERTIFIED = 'certified'
 TITLE_SIZE_LIMIT = 350
 DESCRIPTION_SIZE_LIMIT = 100000
 
+ORG_BID_SIZE_LIMIT = 14
+
 
 class Team(db.EmbeddedDocument):
     name = db.StringField(required=True)
@@ -104,6 +106,7 @@ class Organization(WithMetrics, BadgeMixin, db.Datetimed, db.Document):
     image_url = db.StringField()
     logo = db.ImageField(fs=avatars, basename=default_image_basename,
                          max_size=LOGO_MAX_SIZE, thumbnails=LOGO_SIZES)
+    business_number_id = db.StringField(max_length=ORG_BID_SIZE_LIMIT)
 
     members = db.ListField(db.EmbeddedDocumentField(Member))
     teams = db.ListField(db.EmbeddedDocumentField(Team))

udata/core/topic/api.py

@@ -41,6 +41,7 @@ topic_fields = api.model('Topic', {
     'page': fields.UrlFor(
         'topics.display', lambda o: {'topic': o},
         description='The topic page URL', readonly=True, fallback_endpoint='api.topic'),
+    'extras': fields.Raw(description='Extras attributes as key-value pairs'),
 }, mask='*,datasets{id,title,uri,page},reuses{id,title, image, image_thumbnail,uri,page}')
 
 

udata/core/topic/forms.py

@@ -22,3 +22,4 @@ class TopicForm(ModelForm):
     tags = fields.TagField(_('Tags'), [validators.DataRequired()])
     private = fields.BooleanField(_('Private'))
     featured = fields.BooleanField(_('Featured'))
+    extras = fields.ExtrasField()

udata/core/topic/models.py

@@ -1,6 +1,9 @@
 from flask import url_for
 
+from mongoengine.signals import pre_save
 from udata.models import db
+from udata.search import reindex
+from udata.tasks import as_task_param
 
 
 __all__ = ('Topic', )
@@ -23,10 +26,27 @@ class Topic(db.Document):
     owner = db.ReferenceField('User')
     featured = db.BooleanField()
     private = db.BooleanField()
+    extras = db.ExtrasField()
 
     def __str__(self):
         return self.name
 
+    @classmethod
+    def pre_save(cls, sender, document, **kwargs):
+        # Try catch is to prevent the mechanism to crash at the
+        # creation of the Topic, where an original state does not exist.
+        try:
+            original_doc = sender.objects.get(id=document.id)
+            # Get the diff between the original and current datasets
+            datasets_list_dif = set(original_doc.datasets) ^ set(document.datasets)
+        except cls.DoesNotExist:
+            datasets_list_dif = document.datasets
+        for dataset in datasets_list_dif:
+            reindex.delay(*as_task_param(dataset))
+
     @property
     def display_url(self):
         return url_for('topics.display', topic=self)
+
+
+pre_save.connect(Topic.pre_save, sender=Topic)

udata/harvest/backends/dcat.py

@@ -35,6 +35,12 @@ KNOWN_PAGINATION = (
     (HYDRA.PagedCollection, HYDRA.nextPage)
 )
 
+# Useful to patch essential failing URIs
+URIS_TO_REPLACE = {
+    # See https://github.com/etalab/data.gouv.fr/issues/1151
+    'https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld': 'https://gist.githubusercontent.com/maudetes/f019586185d6f59dcfb07f97148a1973/raw/585c3c7bf602b5a4e635b137257d0619792e2c1f/gistfile1.txt'  # noqa
+}
+
 
 def extract_graph(source, target, node, specs):
     for p, o in source.predicate_objects(node):
@@ -80,7 +86,10 @@ class DcatBackend(BaseBackend):
         page = 0
         while url:
             subgraph = Graph(namespace_manager=namespace_manager)
-            subgraph.parse(data=requests.get(url).text, format=fmt)
+            data = requests.get(url).text
+            for old_uri, new_uri in URIS_TO_REPLACE.items():
+                data = data.replace(old_uri, new_uri)
+            subgraph.parse(data=data, format=fmt)
 
             url = None
             for cls, prop in KNOWN_PAGINATION:

udata/harvest/tests/test_dcat_backend.py

@@ -10,6 +10,7 @@ from udata.core.organization.factories import OrganizationFactory
 from udata.core.dataset.factories import LicenseFactory
 
 from .factories import HarvestSourceFactory
+from ..backends.dcat import URIS_TO_REPLACE
 from .. import actions
 
 log = logging.getLogger(__name__)
@@ -333,7 +334,7 @@ class DcatBackendTest:
         assert 'geodesy' in dataset.tags  # support dcat:theme
         assert dataset.license.id == 'fr-lo'
         assert len(dataset.resources) == 1
-        assert dataset.description.startswith('Data from the \'National network')
+        assert dataset.description.startswith("Data from the 'National network")
         assert dataset.harvest is not None
         assert dataset.harvest.dct_identifier == '0437a976-cff1-4fa6-807a-c23006df2f8f'
         assert dataset.harvest.remote_id == '0437a976-cff1-4fa6-807a-c23006df2f8f'
@@ -383,3 +384,27 @@ class DcatBackendTest:
         error = job.errors[0]
         expected = 'Unable to detect format from extension or mime type'
         assert error.message == expected
+
+    def test_use_replaced_uris(self, rmock, mocker):
+        mocker.patch.dict(
+            URIS_TO_REPLACE,
+            {'http://example.org/this-url-does-not-exist': 'https://json-ld.org/contexts/person.jsonld'}
+        )
+        url = DCAT_URL_PATTERN.format(path='', domain=TEST_DOMAIN)
+        rmock.get(url, json={
+            '@context': 'http://example.org/this-url-does-not-exist',
+            '@type': 'dcat:Catalog',
+            'dataset': []
+        })
+        rmock.head(url, headers={'Content-Type': 'application/json'})
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='dcat',
+                                      url=url,
+                                      organization=org)
+        actions.run(source.slug)
+
+        source.reload()
+
+        job = source.get_last_job()
+        assert len(job.items) == 0
+        assert job.status == 'done'

udata/models/slug_fields.py

@@ -163,6 +163,10 @@ def populate_slug(instance, field):
             return qs(**{field.db_field: s}).clear_cls_query().limit(1).count(True) > 0
 
         while exists(slug):
+            # keep space for index suffix, trim slug if needed
+            slug_overflow = len('{0}-{1}'.format(base_slug, index)) - field.max_length
+            if slug_overflow >= 1:
+                base_slug = base_slug[:-slug_overflow]
             slug = '{0}-{1}'.format(base_slug, index)
             index += 1
 

udata/search/commands.py

@@ -52,7 +52,10 @@ def index_model(adapter, start, reindex=False, from_datetime=None):
     log.info('Indexing %s objects', model.__name__)
     qs = model.objects
     if from_datetime:
-        qs = qs.filter(last_modified_internal__gte=from_datetime)
+        date_property = ('last_modified_internal'
+                         if model.__name__.lower() in ['dataset']
+                         else 'last_modified')
+        qs = qs.filter(**{f'{date_property}__gte': from_datetime})
     index_name = adapter.model.__name__.lower()
     if reindex:
         index_name += '-' + default_index_suffix_name(start)
@@ -101,7 +104,10 @@ def finalize_reindex(models, start):
     modified_since_reindex = 0
     for adapter in iter_adapters():
         if not models or adapter.model.__name__.lower() in models:
-            modified_since_reindex += adapter.model.objects(last_modified__gte=start).count()
+            date_property = ('last_modified_internal'
+                             if adapter.model.__name__.lower() in ['dataset']
+                             else 'last_modified')
+            modified_since_reindex += adapter.model.objects(**{f'{date_property}__gte': start}).count()
 
     log.warning(
         f'{modified_since_reindex} documents have been modified since reindexation start. '

udata/settings.py

@@ -187,6 +187,8 @@ class Defaults(object):
         'tbody',
         'thead',
         'tfooter',
+        'details',
+        'summary'
         # 'title',
     ]
 
@@ -362,6 +364,11 @@ class Defaults(object):
     # Default pagination size on listing
     POST_DEFAULT_PAGINATION = 20
 
+    # Organization settings
+    ###########################################################################
+    # The business identification format to use for validation
+    ORG_BID_FORMAT = 'siret'
+
     # Dataset settings
     ###########################################################################
     # Max number of resources to display uncollapsed in dataset view