udata 8.0.1__py2.py3-none-any.whl → 8.0.1.dev28882__py2.py3-none-any.whl

udata/__init__.py

@@ -4,5 +4,5 @@
 udata
 '''
 
-__version__ = '8.0.1'
+__version__ = '8.0.1.dev'
 __description__ = 'Open data portal'

udata/api/__init__.py

@@ -22,7 +22,7 @@ from udata.auth import (
 from udata.utils import safe_unicode
 from udata.mongo.errors import FieldValidationError
 
-from . import fields
+from . import fields, oauth2
 from .signals import on_api_call
 
 
@@ -129,7 +129,6 @@ class UDataApi(Api):
         @wraps(func)
         def wrapper(*args, **kwargs):
             from udata.core.user.models import User
-            from udata.api.oauth2 import check_credentials
 
             if current_user.is_authenticated:
                 return func(*args, **kwargs)
@@ -144,7 +143,7 @@ class UDataApi(Api):
                 if not login_user(user, False):
                     self.abort(401, 'Inactive user')
             else:
-                check_credentials()
+                oauth2.check_credentials()
             return func(*args, **kwargs)
         return wrapper
 
@@ -325,7 +324,6 @@ def init_app(app):
     import udata.core.user.api  # noqa
     import udata.core.dataset.api  # noqa
     import udata.core.dataset.apiv2  # noqa
-    import udata.core.dataservices.api  # noqa
     import udata.core.discussions.api  # noqa
     import udata.core.reuse.api  # noqa
     import udata.core.reuse.apiv2  # noqa
@@ -352,6 +350,5 @@ def init_app(app):
     app.register_blueprint(apiv1_blueprint)
     app.register_blueprint(apiv2_blueprint)
 
-    from udata.api.oauth2 import init_app as oauth2_init_app
-    oauth2_init_app(app)
+    oauth2.init_app(app)
     cors.init_app(app)

udata/api/oauth2.py

@@ -34,7 +34,6 @@ from werkzeug.security import gen_salt
 
 from udata.app import csrf
 from udata.auth import current_user, login_required, login_user
-from udata.core.organization.models import Organization
 from udata.i18n import I18nBlueprint, lazy_gettext as _
 from udata.mongo import db
 from udata.core.storages import images, default_image_basename
@@ -67,7 +66,7 @@ class OAuth2Client(ClientMixin, db.Datetimed, db.Document):
     description = db.StringField()
 
     owner = db.ReferenceField('User')
-    organization = db.ReferenceField(Organization, reverse_delete_rule=db.NULLIFY)
+    organization = db.ReferenceField('Organization', reverse_delete_rule=db.NULLIFY)
     image = db.ImageField(fs=images, basename=default_image_basename,
                           thumbnails=[150, 25])
 

udata/core/badges/models.py

@@ -1,15 +1,14 @@
 import logging
+import weakref
 
 from datetime import datetime
 
 from mongoengine.signals import post_save
 
-from udata.api_fields import field
 from udata.auth import current_user
 from udata.mongo import db
 
 from .signals import on_badge_added, on_badge_removed
-from .fields import badge_fields
 
 log = logging.getLogger(__name__)
 

udata/core/dataset/apiv2.py

@@ -8,7 +8,6 @@ from udata import search
 from udata.api import apiv2, API, fields
 from udata.utils import multi_to_dict, get_by
 
-from udata.core.organization.api_fields import member_user_with_email_fields
 from .api_fields import (
     badge_fields,
     org_ref_fields,
@@ -166,7 +165,6 @@ specific_resource_fields = apiv2.model('SpecificResource', {
 apiv2.inherit('Badge', badge_fields)
 apiv2.inherit('OrganizationReference', org_ref_fields)
 apiv2.inherit('UserReference', user_ref_fields)
-apiv2.inherit('MemberUserWithEmail', member_user_with_email_fields)
 apiv2.inherit('Resource', resource_fields)
 apiv2.inherit('SpatialCoverage', spatial_coverage_fields)
 apiv2.inherit('TemporalCoverage', temporal_coverage_fields)

udata/core/dataset/models.py

@@ -505,7 +505,6 @@ class Dataset(WithMetrics, BadgeMixin, Owned, db.Document):
         'reuses',
         'followers',
         'views',
-        'resources_downloads',
     ]
 
     meta = {

udata/core/dataset/rdf.py

@@ -76,16 +76,6 @@ EU_RDF_REQUENCIES = {
     EUFREQ.NEVER: 'punctual',
 }
 
-# Map High Value Datasets URIs to keyword categories
-EU_HVD_CATEGORIES = {
-    "http://data.europa.eu/bna/c_164e0bf5": "Météorologiques",
-    "http://data.europa.eu/bna/c_a9135398": "Entreprises et propriété d'entreprises",
-    "http://data.europa.eu/bna/c_ac64a52d": "Géospatiales",
-    "http://data.europa.eu/bna/c_b79e35eb": "Mobilité",
-    "http://data.europa.eu/bna/c_dd313021": "Observation de la terre et environnement",
-    "http://data.europa.eu/bna/c_e1da4e07": "Statistiques"
-}
-
 
 class HTMLDetector(HTMLParser):
     def __init__(self, *args, **kwargs):
@@ -479,19 +469,9 @@ def remote_url_from_rdf(rdf):
 
 
 def theme_labels_from_rdf(rdf):
-    '''
-    Get theme labels to use as keywords.
-    Map HVD keywords from known URIs resources if HVD support is activated.
-    '''
     for theme in rdf.objects(DCAT.theme):
         if isinstance(theme, RdfResource):
-            uri = theme.identifier.toPython()
-            if current_app.config['HVD_SUPPORT'] and uri in EU_HVD_CATEGORIES:
-                label = EU_HVD_CATEGORIES[uri]
-                # Additionnally yield hvd keyword
-                yield 'hvd'
-            else:
-                label = rdf_value(theme, SKOS.prefLabel)
+            label = rdf_value(theme, SKOS.prefLabel)
         else:
             label = theme.toPython()
         if label:

udata/core/metrics/models.py

@@ -1,4 +1,5 @@
-from udata.api_fields import field
+from datetime import date, timedelta
+
 from udata.mongo import db
 
 
@@ -6,7 +7,7 @@ __all__ = ('WithMetrics',)
 
 
 class WithMetrics(object):
-    metrics = field(db.DictField())
+    metrics = db.DictField()
 
     __metrics_keys__ = []
 

udata/core/organization/api_fields.py

@@ -1,8 +1,5 @@
-from flask import request
-
 from udata.api import api, fields, base_reference
 from udata.core.badges.fields import badge_fields
-from udata.core.organization.permissions import OrganizationPrivatePermission
 
 from .constants import ORG_ROLES, DEFAULT_ROLE, MEMBERSHIP_STATUS, BIGGEST_LOGO_SIZE
 
@@ -30,29 +27,9 @@ org_ref_fields = api.inherit('OrganizationReference', base_reference, {
 
 from udata.core.user.api_fields import user_ref_fields  # noqa: required
 
-def check_can_access_email():
-    # This endpoint is secure, only organization member has access.
-    if request.endpoint == 'api.request_membership':
-        return True
-
-    if request.endpoint != 'api.organization':
-        return False
-
-    org = request.view_args.get('org')
-    if org is None:
-        return False
-    
-    return OrganizationPrivatePermission(org).can()
-
-member_user_with_email_fields = api.inherit('MemberUserWithEmail', user_ref_fields, {
-    'email': fields.Raw(
-        attribute=lambda o: o.email if check_can_access_email() else None,
-        description='The user email (only present on show organization endpoint if the current user has edit permission on the org)', readonly=True),
-})
-
 request_fields = api.model('MembershipRequest', {
     'id': fields.String(readonly=True),
-    'user': fields.Nested(member_user_with_email_fields),
+    'user': fields.Nested(user_ref_fields),
     'created': fields.ISODateTime(
         description='The request creation date', readonly=True),
     'status': fields.String(
@@ -63,12 +40,10 @@ request_fields = api.model('MembershipRequest', {
 })
 
 member_fields = api.model('Member', {
-    'user': fields.Nested(member_user_with_email_fields),
+    'user': fields.Nested(user_ref_fields),
     'role': fields.String(
         description='The member role in the organization', required=True,
-        enum=list(ORG_ROLES), default=DEFAULT_ROLE),
-    'since': fields.ISODateTime(
-        description='The date the user joined the organization', readonly=True),
+        enum=list(ORG_ROLES), default=DEFAULT_ROLE)
 })
 
 org_fields = api.model('Organization', {

udata/core/organization/models.py

@@ -5,11 +5,9 @@ from blinker import Signal
 from mongoengine.signals import pre_save, post_save
 from werkzeug.utils import cached_property
 
-from udata.core.badges.models import BadgeMixin
-from udata.core.metrics.models import WithMetrics
 from udata.core.storages import avatars, default_image_basename
 from udata.frontend.markdown import mdstrip
-from udata.mongo import db
+from udata.models import db, BadgeMixin, WithMetrics
 from udata.i18n import lazy_gettext as _
 from udata.uris import endpoint_for
 from .constants import ASSOCIATION, CERTIFIED, COMPANY, LOCAL_AUTHORITY, LOGO_SIZES, ORG_BID_SIZE_LIMIT, ORG_ROLES, DEFAULT_ROLE, MEMBERSHIP_STATUS, LOGO_MAX_SIZE, PUBLIC_SERVICE

udata/core/owned.py

@@ -4,15 +4,7 @@ from blinker import signal
 from mongoengine import NULLIFY, Q, post_save
 from mongoengine.fields import ReferenceField
 
-from udata.api_fields import field
-from udata.core.organization.models import Organization
-from udata.core.user.models import User
 from udata.mongo.queryset import UDataQuerySet
-from udata.core.user.api_fields import user_ref_fields
-from udata.core.organization.api_fields import org_ref_fields
-from udata.core.organization.permissions import OrganizationPrivatePermission
-from udata.mongo.errors import FieldValidationError
-from udata.i18n import lazy_gettext as _
 
 log = logging.getLogger(__name__)
 
@@ -23,42 +15,14 @@ class OwnedQuerySet(UDataQuerySet):
         for owner in owners:
             qs |= Q(owner=owner) | Q(organization=owner)
         return self(qs)
-    
-def check_owner_is_current_user(owner):
-    from udata.auth import current_user, admin_permission
-    if current_user.is_authenticated and owner and not admin_permission and current_user.id != owner:
-        raise FieldValidationError(_('You can only set yourself as owner'), field="owner")
-
-def check_organization_is_valid_for_current_user(organization):
-    from udata.auth import current_user
-    from udata.models import Organization
-
-    org = Organization.objects(id=organization).first()
-    if org is None:
-        raise FieldValidationError(_("Unknown organization"), field="organization")
-
-    if current_user.is_authenticated and org and not OrganizationPrivatePermission(org).can():
-        raise FieldValidationError(_("Permission denied for this organization"), field="organization")
 
 
 class Owned(object):
     '''
     A mixin to factorize owning behvaior between users and organizations.
     '''
-    owner = field(
-        ReferenceField(User, reverse_delete_rule=NULLIFY),
-        nested_fields=user_ref_fields,
-        description="Only present if organization is not set. Can only be set to the current authenticated user.",
-        check=check_owner_is_current_user,
-        allow_null=True,
-    )
-    organization = field(
-        ReferenceField(Organization, reverse_delete_rule=NULLIFY),
-        nested_fields=org_ref_fields,
-        description="Only present if owner is not set. Can only be set to an organization of the current authenticated user.",
-        check=check_organization_is_valid_for_current_user,
-        allow_null=True,
-    )
+    owner = ReferenceField('User', reverse_delete_rule=NULLIFY)
+    organization = ReferenceField('Organization', reverse_delete_rule=NULLIFY)
 
     on_owner_change = signal('Owned.on_owner_change')
 
@@ -74,7 +38,6 @@ class Owned(object):
         '''
         Verify owner consistency and fetch original owner before the new one erase it.
         '''
-
         changed_fields = self._get_changed_fields()
         if 'organization' in changed_fields and 'owner' in changed_fields:
             # Ownership changes (org to owner or the other way around) have already been made

udata/core/spatial/api.py

@@ -85,7 +85,7 @@ dataset_parser.add_argument(
     location='args', default=25)
 
 
-@ns.route('/zones/<list:ids>/', endpoint='zones')
+@ns.route('/zones/<pathlist:ids>/', endpoint='zones')
 class ZonesAPI(API):
     @api.doc('spatial_zones',
              params={'ids': 'A zone identifiers list (comma separated)'})
@@ -101,7 +101,7 @@ class ZonesAPI(API):
         }
 
 
-@ns.route('/zone/<id>/datasets/', endpoint='zone_datasets')
+@ns.route('/zone/<path:id>/datasets/', endpoint='zone_datasets')
 class ZoneDatasetsAPI(API):
     @api.doc('spatial_zone_datasets', params={'id': 'A zone identifier'})
     @api.expect(dataset_parser)
@@ -118,7 +118,7 @@ class ZoneDatasetsAPI(API):
         return datasets
 
 
-@ns.route('/zone/<id>/', endpoint='zone')
+@ns.route('/zone/<path:id>/', endpoint='zone')
 class ZoneAPI(API):
     @api.doc('spatial_zone', params={'id': 'A zone identifier'})
     def get(self, id):
@@ -152,7 +152,7 @@ class SpatialGranularitiesAPI(API):
         } for id, name in spatial_granularities]
 
 
-@ns.route('/coverage/<level>/', endpoint='spatial_coverage')
+@ns.route('/coverage/<path:level>/', endpoint='spatial_coverage')
 class SpatialCoverageAPI(API):
     @api.doc('spatial_coverage')
     @api.marshal_list_with(feature_collection_fields)

udata/core/user/api.py

@@ -226,7 +226,6 @@ class UserListAPI(API):
     fields = user_fields
     form = UserProfileForm
 
-    @api.secure(admin_permission)
     @api.doc('list_users')
     @api.expect(user_parser.parser)
     @api.marshal_with(user_page_fields)
@@ -270,12 +269,6 @@ class UserAvatarAPI(API):
         return {'image': user.avatar}
 
 
-
-delete_parser = api.parser()
-delete_parser.add_argument(
-    'no_mail', type=bool, help='Do not send a mail to notify the user of the deletion',
-    location='args', default=False)
-
 @ns.route('/<user:user>/', endpoint='user')
 @api.response(404, 'User not found')
 @api.response(410, 'User is not active or has been deleted')
@@ -304,19 +297,22 @@ class UserAPI(API):
 
     @api.secure(admin_permission)
     @api.doc('delete_user')
-    @api.expect(delete_parser)
     @api.response(204, 'Object deleted')
     @api.response(403, 'When trying to delete yourself')
     def delete(self, user):
         '''Delete a user given its identifier'''
-        args = delete_parser.parse_args()
         if user.deleted:
             api.abort(410, 'User has already been deleted')
         if user == current_user._get_current_object():
             api.abort(403, 'You cannot delete yourself with this API. ' +
                       'Use the "me" API instead.')
-        
-        user.mark_as_deleted(notify=not args['no_mail'])
+        if user.avatar.filename is not None:
+            storage = storages.avatars
+            storage.delete(user.avatar.filename)
+            storage.delete(user.avatar.original)
+            for key, value in user.avatar.thumbnails.items():
+                storage.delete(value)
+        user.mark_as_deleted()
         return '', 204
 
 

udata/core/user/models.py

@@ -13,7 +13,6 @@ from mongoengine.signals import pre_save, post_save
 from werkzeug.utils import cached_property
 
 from udata import mail
-from udata.core import storages
 from udata.uris import endpoint_for
 from udata.frontend.markdown import mdstrip
 from udata.i18n import lazy_gettext as _
@@ -234,15 +233,7 @@ class User(WithMetrics, UserMixin, db.Document):
         raise NotImplementedError('''This method should not be using directly.
         Use `mark_as_deleted` (or `_delete` if you know what you're doing)''')
 
-    def mark_as_deleted(self, notify: bool = True):
-        if self.avatar.filename is not None:
-            storage = storages.avatars
-            storage.delete(self.avatar.filename)
-            storage.delete(self.avatar.original)
-            for key, value in self.avatar.thumbnails.items():
-                storage.delete(value)
-
-
+    def mark_as_deleted(self):
         copied_user = copy(self)
         self.email = '{}@deleted'.format(self.id)
         self.slug = 'deleted'
@@ -279,9 +270,7 @@ class User(WithMetrics, UserMixin, db.Document):
         from udata.models import ContactPoint
         ContactPoint.objects(owner=self).delete()
 
-
-        if notify:
-            mail.send(_('Account deletion'), copied_user, 'account_deleted')
+        mail.send(_('Account deletion'), copied_user, 'account_deleted')
 
     def count_datasets(self):
         from udata.models import Dataset

udata/harvest/backends/dcat.py

@@ -268,6 +268,7 @@ class CswDcatBackend(DcatBackend):
                           headers=headers).content)
 
         return graphs
+    
 
 
 class CswIso19139DcatBackend(DcatBackend):
@@ -294,7 +295,6 @@ class CswIso19139DcatBackend(DcatBackend):
         transform = ET.XSLT(xsl)
 
         # Start querying and parsing graph
-        # Filter on dataset or serie records
         body = '''<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
                                   xmlns:gmd="http://www.isotc211.org/2005/gmd"
                                   service="CSW" version="2.0.2" resultType="results"
@@ -304,16 +304,10 @@ class CswIso19139DcatBackend(DcatBackend):
                         <csw:ElementSetName>full</csw:ElementSetName>
                         <csw:Constraint version="1.1.0">
                             <ogc:Filter xmlns:ogc="http://www.opengis.net/ogc">
-                                <ogc:Or xmlns:ogc="http://www.opengis.net/ogc">
-                                    <ogc:PropertyIsEqualTo>
-                                        <ogc:PropertyName>dc:type</ogc:PropertyName>
-                                        <ogc:Literal>dataset</ogc:Literal>
-                                    </ogc:PropertyIsEqualTo>
-                                    <ogc:PropertyIsEqualTo>
-                                        <ogc:PropertyName>dc:type</ogc:PropertyName>
-                                        <ogc:Literal>series</ogc:Literal>
-                                    </ogc:PropertyIsEqualTo>
-                                </ogc:Or>
+                                <ogc:PropertyIsEqualTo>
+                                    <ogc:PropertyName>dc:type</ogc:PropertyName>
+                                    <ogc:Literal>dataset</ogc:Literal>
+                                </ogc:PropertyIsEqualTo>
                             </ogc:Filter>
                         </csw:Constraint>
                     </csw:Query>
@@ -325,7 +319,7 @@ class CswIso19139DcatBackend(DcatBackend):
         start = 1
 
         response = self.post(url, data=body.format(start=start, schema=self.ISO_SCHEMA),
-                             headers=headers)
+                            headers=headers)
         response.raise_for_status()
 
         tree_before_transform = ET.fromstring(response.content)
@@ -357,12 +351,12 @@ class CswIso19139DcatBackend(DcatBackend):
             next_record = self.next_record_if_should_continue(start, search_results)
             if not next_record:
                 break
-
+            
             start = next_record
             page += 1
 
             response = self.post(url, data=body.format(start=start, schema=self.ISO_SCHEMA),
-                                 headers=headers)
+                          headers=headers)
             response.raise_for_status()
 
             tree_before_transform = ET.fromstring(response.content)

udata/harvest/tests/dcat/catalog.xml

@@ -23,7 +23,6 @@
         <dcterms:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T18:59:02.737480</dcterms:issued>
         <dcterms:description>Dataset 3 description</dcterms:description>
         <dcat:keyword>Tag 1</dcat:keyword>
-        <dcat:theme rdf:resource="http://data.europa.eu/bna/c_dd313021"/>
         <dcat:distribution rdf:resource="datasets/3/resources/1"/>
         <dct:license>Licence Ouverte Version 2.0</dct:license>
         <dct:accessRights rdf:resource="http://inspire.ec.europa.eu/metadata-codelist/LimitationsOnPublicAccess/INSPIRE_Directive_Article13_1e"/>

udata/harvest/tests/test_dcat_backend.py

@@ -440,9 +440,6 @@ class DcatBackendTest:
         assert dataset.extras["harvest"]["dct:accessRights"] == "http://inspire.ec.europa.eu/metadata-codelist/LimitationsOnPublicAccess/INSPIRE_Directive_Article13_1e"
         assert dataset.extras["harvest"]["dct:provenance"] == ["Description de la provenance des données"]
 
-        assert 'observation-de-la-terre-et-environnement' in dataset.tags
-        assert 'hvd' in dataset.tags
-
         dataset = Dataset.objects.get(harvest__dct_identifier='1')
         # test html abstract description support
         assert dataset.description == '# h1 title\n\n## h2 title\n\n **and bold text**'

udata/routing.py

@@ -10,7 +10,6 @@ from werkzeug.urls import url_quote
 from udata import models
 from udata.mongo import db
 from udata.core.spatial.models import GeoZone
-from udata.core.dataservices.models import Dataservice
 from udata.i18n import ISO_639_1_CODES
 
 
@@ -122,10 +121,6 @@ class DatasetConverter(ModelConverter):
     model = models.Dataset
 
 
-class DataserviceConverter(ModelConverter):
-    model = Dataservice
-
-
 class CommunityResourceConverter(ModelConverter):
     model = models.CommunityResource
 
@@ -227,7 +222,6 @@ def init_app(app):
     app.url_map.converters['pathlist'] = PathListConverter
     app.url_map.converters['uuid'] = UUIDConverter
     app.url_map.converters['dataset'] = DatasetConverter
-    app.url_map.converters['dataservice'] = DataserviceConverter
     app.url_map.converters['crid'] = CommunityResourceConverter
     app.url_map.converters['org'] = OrganizationConverter
     app.url_map.converters['reuse'] = ReuseConverter

udata/settings.py

@@ -269,10 +269,7 @@ class Defaults(object):
     # S3 connection details
     S3_URL = None
     S3_ACCESS_KEY_ID = None
-    S3_SECRET_ACCESS_KEY = None
-
-    # Specific support for hvd (map HVD categories URIs to keywords)
-    HVD_SUPPORT = True
+    S3_SECRET_ACCESS_KEY = None 
 
     ACTIVATE_TERRITORIES = False
     # The order is important to compute parents/children, smaller first.