udata 7.0.8.dev28841__py2.py3-none-any.whl → 8.0.1__py2.py3-none-any.whl

udata/core/organization/api_fields.py

@@ -1,5 +1,8 @@
+from flask import request
+
 from udata.api import api, fields, base_reference
 from udata.core.badges.fields import badge_fields
+from udata.core.organization.permissions import OrganizationPrivatePermission
 
 from .constants import ORG_ROLES, DEFAULT_ROLE, MEMBERSHIP_STATUS, BIGGEST_LOGO_SIZE
 
@@ -27,9 +30,29 @@ org_ref_fields = api.inherit('OrganizationReference', base_reference, {
 
 from udata.core.user.api_fields import user_ref_fields  # noqa: required
 
+def check_can_access_email():
+    # This endpoint is secure, only organization member has access.
+    if request.endpoint == 'api.request_membership':
+        return True
+
+    if request.endpoint != 'api.organization':
+        return False
+
+    org = request.view_args.get('org')
+    if org is None:
+        return False
+    
+    return OrganizationPrivatePermission(org).can()
+
+member_user_with_email_fields = api.inherit('MemberUserWithEmail', user_ref_fields, {
+    'email': fields.Raw(
+        attribute=lambda o: o.email if check_can_access_email() else None,
+        description='The user email (only present on show organization endpoint if the current user has edit permission on the org)', readonly=True),
+})
+
 request_fields = api.model('MembershipRequest', {
     'id': fields.String(readonly=True),
-    'user': fields.Nested(user_ref_fields),
+    'user': fields.Nested(member_user_with_email_fields),
     'created': fields.ISODateTime(
         description='The request creation date', readonly=True),
     'status': fields.String(
@@ -40,10 +63,12 @@ request_fields = api.model('MembershipRequest', {
 })
 
 member_fields = api.model('Member', {
-    'user': fields.Nested(user_ref_fields),
+    'user': fields.Nested(member_user_with_email_fields),
     'role': fields.String(
         description='The member role in the organization', required=True,
-        enum=list(ORG_ROLES), default=DEFAULT_ROLE)
+        enum=list(ORG_ROLES), default=DEFAULT_ROLE),
+    'since': fields.ISODateTime(
+        description='The date the user joined the organization', readonly=True),
 })
 
 org_fields = api.model('Organization', {

udata/core/organization/models.py

@@ -5,9 +5,11 @@ from blinker import Signal
 from mongoengine.signals import pre_save, post_save
 from werkzeug.utils import cached_property
 
+from udata.core.badges.models import BadgeMixin
+from udata.core.metrics.models import WithMetrics
 from udata.core.storages import avatars, default_image_basename
 from udata.frontend.markdown import mdstrip
-from udata.models import db, BadgeMixin, WithMetrics
+from udata.mongo import db
 from udata.i18n import lazy_gettext as _
 from udata.uris import endpoint_for
 from .constants import ASSOCIATION, CERTIFIED, COMPANY, LOCAL_AUTHORITY, LOGO_SIZES, ORG_BID_SIZE_LIMIT, ORG_ROLES, DEFAULT_ROLE, MEMBERSHIP_STATUS, LOGO_MAX_SIZE, PUBLIC_SERVICE

udata/core/owned.py

@@ -4,7 +4,15 @@ from blinker import signal
 from mongoengine import NULLIFY, Q, post_save
 from mongoengine.fields import ReferenceField
 
+from udata.api_fields import field
+from udata.core.organization.models import Organization
+from udata.core.user.models import User
 from udata.mongo.queryset import UDataQuerySet
+from udata.core.user.api_fields import user_ref_fields
+from udata.core.organization.api_fields import org_ref_fields
+from udata.core.organization.permissions import OrganizationPrivatePermission
+from udata.mongo.errors import FieldValidationError
+from udata.i18n import lazy_gettext as _
 
 log = logging.getLogger(__name__)
 
@@ -15,14 +23,42 @@ class OwnedQuerySet(UDataQuerySet):
         for owner in owners:
             qs |= Q(owner=owner) | Q(organization=owner)
         return self(qs)
+    
+def check_owner_is_current_user(owner):
+    from udata.auth import current_user, admin_permission
+    if current_user.is_authenticated and owner and not admin_permission and current_user.id != owner:
+        raise FieldValidationError(_('You can only set yourself as owner'), field="owner")
+
+def check_organization_is_valid_for_current_user(organization):
+    from udata.auth import current_user
+    from udata.models import Organization
+
+    org = Organization.objects(id=organization).first()
+    if org is None:
+        raise FieldValidationError(_("Unknown organization"), field="organization")
+
+    if current_user.is_authenticated and org and not OrganizationPrivatePermission(org).can():
+        raise FieldValidationError(_("Permission denied for this organization"), field="organization")
 
 
 class Owned(object):
     '''
     A mixin to factorize owning behvaior between users and organizations.
     '''
-    owner = ReferenceField('User', reverse_delete_rule=NULLIFY)
-    organization = ReferenceField('Organization', reverse_delete_rule=NULLIFY)
+    owner = field(
+        ReferenceField(User, reverse_delete_rule=NULLIFY),
+        nested_fields=user_ref_fields,
+        description="Only present if organization is not set. Can only be set to the current authenticated user.",
+        check=check_owner_is_current_user,
+        allow_null=True,
+    )
+    organization = field(
+        ReferenceField(Organization, reverse_delete_rule=NULLIFY),
+        nested_fields=org_ref_fields,
+        description="Only present if owner is not set. Can only be set to an organization of the current authenticated user.",
+        check=check_organization_is_valid_for_current_user,
+        allow_null=True,
+    )
 
     on_owner_change = signal('Owned.on_owner_change')
 
@@ -38,6 +74,7 @@ class Owned(object):
         '''
         Verify owner consistency and fetch original owner before the new one erase it.
         '''
+
         changed_fields = self._get_changed_fields()
         if 'organization' in changed_fields and 'owner' in changed_fields:
             # Ownership changes (org to owner or the other way around) have already been made

udata/core/spatial/api.py

@@ -85,7 +85,7 @@ dataset_parser.add_argument(
     location='args', default=25)
 
 
-@ns.route('/zones/<pathlist:ids>/', endpoint='zones')
+@ns.route('/zones/<list:ids>/', endpoint='zones')
 class ZonesAPI(API):
     @api.doc('spatial_zones',
              params={'ids': 'A zone identifiers list (comma separated)'})
@@ -101,7 +101,7 @@ class ZonesAPI(API):
         }
 
 
-@ns.route('/zone/<path:id>/datasets/', endpoint='zone_datasets')
+@ns.route('/zone/<id>/datasets/', endpoint='zone_datasets')
 class ZoneDatasetsAPI(API):
     @api.doc('spatial_zone_datasets', params={'id': 'A zone identifier'})
     @api.expect(dataset_parser)
@@ -118,7 +118,7 @@ class ZoneDatasetsAPI(API):
         return datasets
 
 
-@ns.route('/zone/<path:id>/', endpoint='zone')
+@ns.route('/zone/<id>/', endpoint='zone')
 class ZoneAPI(API):
     @api.doc('spatial_zone', params={'id': 'A zone identifier'})
     def get(self, id):
@@ -152,7 +152,7 @@ class SpatialGranularitiesAPI(API):
         } for id, name in spatial_granularities]
 
 
-@ns.route('/coverage/<path:level>/', endpoint='spatial_coverage')
+@ns.route('/coverage/<level>/', endpoint='spatial_coverage')
 class SpatialCoverageAPI(API):
     @api.doc('spatial_coverage')
     @api.marshal_list_with(feature_collection_fields)
@@ -162,11 +162,6 @@ class SpatialCoverageAPI(API):
         features = []
 
         for zone in GeoZone.objects(level=level.id):
-            # fetch nested levels IDs
-            ids = []
-            ids.append(zone.id)
-            # Count datasets in zone
-            nb_datasets = Dataset.objects(spatial__zones__in=ids).count()
             features.append({
                 'id': zone.id,
                 'type': 'Feature',
@@ -174,7 +169,7 @@ class SpatialCoverageAPI(API):
                     'name': _(zone.name),
                     'code': zone.code,
                     'uri': zone.uri,
-                    'datasets': nb_datasets
+                    'datasets': zone.metrics.get('datasets', 0)
                 }
             })
 

udata/core/spatial/models.py

@@ -3,6 +3,7 @@ from werkzeug.local import LocalProxy
 from werkzeug.utils import cached_property
 
 from udata.app import cache
+from udata.core.metrics.models import WithMetrics
 from udata.uris import endpoint_for
 from udata.i18n import _, get_locale, language
 from udata.mongo import db
@@ -21,7 +22,6 @@ class GeoLevel(db.Document):
                               max_value=ADMIN_LEVEL_MAX,
                               default=100)
 
-
 class GeoZoneQuerySet(db.BaseQuerySet):
 
     def resolve(self, geoid, id_only=False):
@@ -40,7 +40,7 @@ class GeoZoneQuerySet(db.BaseQuerySet):
         return result.id if id_only and result else result
 
 
-class GeoZone(db.Document):
+class GeoZone(WithMetrics, db.Document):
     SEPARATOR = ':'
 
     id = db.StringField(primary_key=True)
@@ -101,6 +101,11 @@ class GeoZone(db.Document):
     def external_url(self):
         return endpoint_for('territories.territory', territory=self, _external=True)
 
+    def count_datasets(self):
+        from udata.models import Dataset
+        self.metrics['datasets'] = Dataset.objects(spatial__zones=self.id).visible().count()
+        self.save()
+
     def toGeoJSON(self):
         return {
             'id': self.id,

udata/core/spatial/tasks.py

@@ -0,0 +1,7 @@
+from udata.core.spatial.models import GeoZone
+from udata.tasks import job
+
+@job('compute-geozones-metrics')
+def compute_geozones_metrics(self):
+    for geozone in GeoZone.objects.timeout(False):
+        geozone.count_datasets()

udata/core/spatial/tests/test_api.py

@@ -10,6 +10,7 @@ from udata.core.dataset.factories import DatasetFactory
 from udata.core.spatial.factories import (
     SpatialCoverageFactory, GeoZoneFactory, GeoLevelFactory
 )
+from udata.core.spatial.tasks import compute_geozones_metrics
 
 
 class SpatialApiTest(APITestCase):
@@ -229,6 +230,31 @@ class SpatialApiTest(APITestCase):
             'features': [],
         })
 
+    def test_coverage_datasets_count(self):
+        GeoLevelFactory(id='fr:commune')
+        paris = GeoZoneFactory(
+            id='fr:commune:75056', level='fr:commune',
+            name='Paris', code='75056')
+        arles = GeoZoneFactory(
+            id='fr:commune:13004', level='fr:commune',
+            name='Arles', code='13004')
+
+        for _ in range(3):
+            DatasetFactory(
+                spatial=SpatialCoverageFactory(zones=[paris.id]))
+        for _ in range(2):
+            DatasetFactory(
+                spatial=SpatialCoverageFactory(zones=[arles.id]))
+                    
+        compute_geozones_metrics()
+
+        response = self.get(url_for('api.spatial_coverage', level='fr:commune'))
+        self.assert200(response)
+        self.assertEqual(response.json['features'][0]['id'], 'fr:commune:13004')
+        self.assertEqual(response.json['features'][0]['properties']['datasets'], 2)
+        self.assertEqual(response.json['features'][1]['id'], 'fr:commune:75056')
+        self.assertEqual(response.json['features'][1]['properties']['datasets'], 3)
+
 
 class SpatialTerritoriesApiTest(APITestCase):
     modules = []

udata/core/user/api.py

@@ -226,6 +226,7 @@ class UserListAPI(API):
     fields = user_fields
     form = UserProfileForm
 
+    @api.secure(admin_permission)
     @api.doc('list_users')
     @api.expect(user_parser.parser)
     @api.marshal_with(user_page_fields)
@@ -269,6 +270,12 @@ class UserAvatarAPI(API):
         return {'image': user.avatar}
 
 
+
+delete_parser = api.parser()
+delete_parser.add_argument(
+    'no_mail', type=bool, help='Do not send a mail to notify the user of the deletion',
+    location='args', default=False)
+
 @ns.route('/<user:user>/', endpoint='user')
 @api.response(404, 'User not found')
 @api.response(410, 'User is not active or has been deleted')
@@ -297,22 +304,19 @@ class UserAPI(API):
 
     @api.secure(admin_permission)
     @api.doc('delete_user')
+    @api.expect(delete_parser)
     @api.response(204, 'Object deleted')
     @api.response(403, 'When trying to delete yourself')
     def delete(self, user):
         '''Delete a user given its identifier'''
+        args = delete_parser.parse_args()
         if user.deleted:
             api.abort(410, 'User has already been deleted')
         if user == current_user._get_current_object():
             api.abort(403, 'You cannot delete yourself with this API. ' +
                       'Use the "me" API instead.')
-        if user.avatar.filename is not None:
-            storage = storages.avatars
-            storage.delete(user.avatar.filename)
-            storage.delete(user.avatar.original)
-            for key, value in user.avatar.thumbnails.items():
-                storage.delete(value)
-        user.mark_as_deleted()
+        
+        user.mark_as_deleted(notify=not args['no_mail'])
         return '', 204
 
 

udata/core/user/models.py

@@ -13,6 +13,7 @@ from mongoengine.signals import pre_save, post_save
 from werkzeug.utils import cached_property
 
 from udata import mail
+from udata.core import storages
 from udata.uris import endpoint_for
 from udata.frontend.markdown import mdstrip
 from udata.i18n import lazy_gettext as _
@@ -233,7 +234,15 @@ class User(WithMetrics, UserMixin, db.Document):
         raise NotImplementedError('''This method should not be using directly.
         Use `mark_as_deleted` (or `_delete` if you know what you're doing)''')
 
-    def mark_as_deleted(self):
+    def mark_as_deleted(self, notify: bool = True):
+        if self.avatar.filename is not None:
+            storage = storages.avatars
+            storage.delete(self.avatar.filename)
+            storage.delete(self.avatar.original)
+            for key, value in self.avatar.thumbnails.items():
+                storage.delete(value)
+
+
         copied_user = copy(self)
         self.email = '{}@deleted'.format(self.id)
         self.slug = 'deleted'
@@ -270,7 +279,9 @@ class User(WithMetrics, UserMixin, db.Document):
         from udata.models import ContactPoint
         ContactPoint.objects(owner=self).delete()
 
-        mail.send(_('Account deletion'), copied_user, 'account_deleted')
+
+        if notify:
+            mail.send(_('Account deletion'), copied_user, 'account_deleted')
 
     def count_datasets(self):
         from udata.models import Dataset

udata/harvest/backends/dcat.py

@@ -268,7 +268,6 @@ class CswDcatBackend(DcatBackend):
                           headers=headers).content)
 
         return graphs
-    
 
 
 class CswIso19139DcatBackend(DcatBackend):
@@ -295,6 +294,7 @@ class CswIso19139DcatBackend(DcatBackend):
         transform = ET.XSLT(xsl)
 
         # Start querying and parsing graph
+        # Filter on dataset or serie records
         body = '''<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
                                   xmlns:gmd="http://www.isotc211.org/2005/gmd"
                                   service="CSW" version="2.0.2" resultType="results"
@@ -304,10 +304,16 @@ class CswIso19139DcatBackend(DcatBackend):
                         <csw:ElementSetName>full</csw:ElementSetName>
                         <csw:Constraint version="1.1.0">
                             <ogc:Filter xmlns:ogc="http://www.opengis.net/ogc">
-                                <ogc:PropertyIsEqualTo>
-                                    <ogc:PropertyName>dc:type</ogc:PropertyName>
-                                    <ogc:Literal>dataset</ogc:Literal>
-                                </ogc:PropertyIsEqualTo>
+                                <ogc:Or xmlns:ogc="http://www.opengis.net/ogc">
+                                    <ogc:PropertyIsEqualTo>
+                                        <ogc:PropertyName>dc:type</ogc:PropertyName>
+                                        <ogc:Literal>dataset</ogc:Literal>
+                                    </ogc:PropertyIsEqualTo>
+                                    <ogc:PropertyIsEqualTo>
+                                        <ogc:PropertyName>dc:type</ogc:PropertyName>
+                                        <ogc:Literal>series</ogc:Literal>
+                                    </ogc:PropertyIsEqualTo>
+                                </ogc:Or>
                             </ogc:Filter>
                         </csw:Constraint>
                     </csw:Query>
@@ -319,7 +325,7 @@ class CswIso19139DcatBackend(DcatBackend):
         start = 1
 
         response = self.post(url, data=body.format(start=start, schema=self.ISO_SCHEMA),
-                            headers=headers)
+                             headers=headers)
         response.raise_for_status()
 
         tree_before_transform = ET.fromstring(response.content)
@@ -351,12 +357,12 @@ class CswIso19139DcatBackend(DcatBackend):
             next_record = self.next_record_if_should_continue(start, search_results)
             if not next_record:
                 break
-            
+
             start = next_record
             page += 1
 
             response = self.post(url, data=body.format(start=start, schema=self.ISO_SCHEMA),
-                          headers=headers)
+                                 headers=headers)
             response.raise_for_status()
 
             tree_before_transform = ET.fromstring(response.content)

udata/harvest/tests/dcat/catalog.xml

@@ -23,6 +23,7 @@
         <dcterms:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T18:59:02.737480</dcterms:issued>
         <dcterms:description>Dataset 3 description</dcterms:description>
         <dcat:keyword>Tag 1</dcat:keyword>
+        <dcat:theme rdf:resource="http://data.europa.eu/bna/c_dd313021"/>
         <dcat:distribution rdf:resource="datasets/3/resources/1"/>
         <dct:license>Licence Ouverte Version 2.0</dct:license>
         <dct:accessRights rdf:resource="http://inspire.ec.europa.eu/metadata-codelist/LimitationsOnPublicAccess/INSPIRE_Directive_Article13_1e"/>

udata/harvest/tests/test_dcat_backend.py

@@ -440,6 +440,9 @@ class DcatBackendTest:
         assert dataset.extras["harvest"]["dct:accessRights"] == "http://inspire.ec.europa.eu/metadata-codelist/LimitationsOnPublicAccess/INSPIRE_Directive_Article13_1e"
         assert dataset.extras["harvest"]["dct:provenance"] == ["Description de la provenance des données"]
 
+        assert 'observation-de-la-terre-et-environnement' in dataset.tags
+        assert 'hvd' in dataset.tags
+
         dataset = Dataset.objects.get(harvest__dct_identifier='1')
         # test html abstract description support
         assert dataset.description == '# h1 title\n\n## h2 title\n\n **and bold text**'

udata/routing.py

@@ -10,6 +10,7 @@ from werkzeug.urls import url_quote
 from udata import models
 from udata.mongo import db
 from udata.core.spatial.models import GeoZone
+from udata.core.dataservices.models import Dataservice
 from udata.i18n import ISO_639_1_CODES
 
 
@@ -121,6 +122,10 @@ class DatasetConverter(ModelConverter):
     model = models.Dataset
 
 
+class DataserviceConverter(ModelConverter):
+    model = Dataservice
+
+
 class CommunityResourceConverter(ModelConverter):
     model = models.CommunityResource
 
@@ -222,6 +227,7 @@ def init_app(app):
     app.url_map.converters['pathlist'] = PathListConverter
     app.url_map.converters['uuid'] = UUIDConverter
     app.url_map.converters['dataset'] = DatasetConverter
+    app.url_map.converters['dataservice'] = DataserviceConverter
     app.url_map.converters['crid'] = CommunityResourceConverter
     app.url_map.converters['org'] = OrganizationConverter
     app.url_map.converters['reuse'] = ReuseConverter

udata/settings.py

@@ -269,7 +269,10 @@ class Defaults(object):
     # S3 connection details
     S3_URL = None
     S3_ACCESS_KEY_ID = None
-    S3_SECRET_ACCESS_KEY = None 
+    S3_SECRET_ACCESS_KEY = None
+
+    # Specific support for hvd (map HVD categories URIs to keywords)
+    HVD_SUPPORT = True
 
     ACTIVATE_TERRITORIES = False
     # The order is important to compute parents/children, smaller first.