udata 7.0.8.dev28841__py2.py3-none-any.whl → 9.0.1.dev29390__py2.py3-none-any.whl

udata/__init__.py

@@ -4,5 +4,5 @@
 udata
 '''
 
-__version__ = '7.0.8.dev'
+__version__ = '9.0.1.dev'
 __description__ = 'Open data portal'

udata/api/__init__.py

@@ -22,7 +22,7 @@ from udata.auth import (
 from udata.utils import safe_unicode
 from udata.mongo.errors import FieldValidationError
 
-from . import fields, oauth2
+from . import fields
 from .signals import on_api_call
 
 
@@ -129,6 +129,7 @@ class UDataApi(Api):
         @wraps(func)
         def wrapper(*args, **kwargs):
             from udata.core.user.models import User
+            from udata.api.oauth2 import check_credentials
 
             if current_user.is_authenticated:
                 return func(*args, **kwargs)
@@ -143,7 +144,7 @@ class UDataApi(Api):
                 if not login_user(user, False):
                     self.abort(401, 'Inactive user')
             else:
-                oauth2.check_credentials()
+                check_credentials()
             return func(*args, **kwargs)
         return wrapper
 
@@ -321,10 +322,10 @@ def init_app(app):
     # Load all core APIs
     import udata.core.activity.api  # noqa
     import udata.core.spatial.api  # noqa
-    import udata.core.metrics.api  # noqa
     import udata.core.user.api  # noqa
     import udata.core.dataset.api  # noqa
     import udata.core.dataset.apiv2  # noqa
+    import udata.core.dataservices.api  # noqa
     import udata.core.discussions.api  # noqa
     import udata.core.reuse.api  # noqa
     import udata.core.reuse.apiv2  # noqa
@@ -351,5 +352,6 @@ def init_app(app):
     app.register_blueprint(apiv1_blueprint)
     app.register_blueprint(apiv2_blueprint)
 
-    oauth2.init_app(app)
+    from udata.api.oauth2 import init_app as oauth2_init_app
+    oauth2_init_app(app)
     cors.init_app(app)

udata/api/oauth2.py

@@ -34,6 +34,7 @@ from werkzeug.security import gen_salt
 
 from udata.app import csrf
 from udata.auth import current_user, login_required, login_user
+from udata.core.organization.models import Organization
 from udata.i18n import I18nBlueprint, lazy_gettext as _
 from udata.mongo import db
 from udata.core.storages import images, default_image_basename
@@ -66,7 +67,7 @@ class OAuth2Client(ClientMixin, db.Datetimed, db.Document):
     description = db.StringField()
 
     owner = db.ReferenceField('User')
-    organization = db.ReferenceField('Organization', reverse_delete_rule=db.NULLIFY)
+    organization = db.ReferenceField(Organization, reverse_delete_rule=db.NULLIFY)
     image = db.ImageField(fs=images, basename=default_image_basename,
                           thumbnails=[150, 25])
 

udata/api_fields.py

@@ -0,0 +1,254 @@
+from udata.api import api
+import flask_restx.fields as restx_fields
+import udata.api.fields as custom_restx_fields
+from bson import ObjectId
+import mongoengine
+import mongoengine.fields as mongo_fields
+
+from udata.mongo.errors import FieldValidationError
+
+def convert_db_to_field(key, field, info = {}):
+    '''
+    This function maps a Mongo field to a Flask RestX field.
+    Most of the types are a simple 1-to-1 mapping except lists and references that requires
+    more work.
+    We currently only map the params that we use from Mongo to RestX (for example min_length / max_length…).
+
+    In the first part of the function we save the RestX constructor as a lambda because we need to call it with the
+    params. Since merging the params involve a litte bit of work (merging default params with read/write params and then with
+    user-supplied overrides, setting the readonly flag…), it's easier to have do this one time at the end of the function.
+    '''
+    info = { **getattr(field, '__additional_field_info__', {}), **info }
+
+    params = {}
+    params['required'] = field.required
+
+    read_params = {}
+    write_params = {}
+
+    constructor = None
+    constructor_read = None
+    constructor_write = None
+
+    if info.get('convert_to'):
+        # TODO: this is currently never used. We may remove it if the auto-conversion
+        # is always good enough.
+        return info.get('convert_to'), info.get('convert_to')
+    elif isinstance(field, mongo_fields.StringField):
+        constructor = restx_fields.String
+        params['min_length'] = field.min_length
+        params['max_length'] = field.max_length
+    elif isinstance(field, mongo_fields.FloatField):
+        constructor = restx_fields.Float
+        params['min'] = field.min # TODO min_value?
+        params['max'] = field.max
+    elif isinstance(field, mongo_fields.BooleanField):
+        constructor = restx_fields.Boolean
+    elif isinstance(field, mongo_fields.DateTimeField):
+        constructor = custom_restx_fields.ISODateTime
+    elif isinstance(field, mongo_fields.DictField):
+        constructor = restx_fields.Raw
+    elif isinstance(field, mongo_fields.ListField):
+        # For lists, we convert the inner value from Mongo to RestX then we create
+        # the `List` RestX type with this converted inner value.
+        field_read, field_write = convert_db_to_field(f"{key}.inner", field.field, info.get('inner_field_info', {}))
+        constructor_read = lambda **kwargs: restx_fields.List(field_read, **kwargs)
+        constructor_write = lambda **kwargs: restx_fields.List(field_write, **kwargs)
+    elif isinstance(field, mongo_fields.ReferenceField):
+        # For reference we accept while writing a String representing the ID of the referenced model.
+        # For reading, if the user supplied a `nested_fields` (RestX model), we use it to convert
+        # the referenced model, if not we return a String (and RestX will call the `str()` of the model
+        # when returning from an endpoint)
+        nested_fields = info.get('nested_fields')
+        if nested_fields is None:
+            # If there is no `nested_fields` convert the object to the string representation.
+            constructor_read = restx_fields.String
+        else:
+            constructor_read = lambda **kwargs: restx_fields.Nested(nested_fields, **kwargs)
+
+        write_params['description'] = "ID of the reference"
+        constructor_write = restx_fields.String
+    elif isinstance(field, mongo_fields.EmbeddedDocumentField):
+        nested_fields = info.get('nested_fields')
+        if nested_fields is None:
+            raise ValueError(f"EmbeddedDocumentField `{key}` requires a `nested_fields` param to serialize/deserialize.")
+
+        constructor = lambda **kwargs: restx_fields.Nested(nested_fields, **kwargs)
+    else:
+        raise ValueError(f"Unsupported MongoEngine field type {field.__class__.__name__}")
+    
+    read_params = {**params, **read_params, **info}
+    write_params = {**params, **write_params, **info}
+
+    read = constructor_read(**read_params) if constructor_read else constructor(**read_params)
+    if write_params.get('readonly', False):
+        write = None
+    else:
+        write = constructor_write(**write_params) if constructor_write else constructor(**write_params)
+    return read, write
+
+def generate_fields(**kwargs):
+    '''
+    This decorator will create two auto-generated attributes on the class `__read_fields__` and `__write_fields__`
+    that can be used in API endpoint inside `expect()` and `marshall_with()`.
+    '''
+    def wrapper(cls):
+        read_fields = {}
+        write_fields = {}
+        sortables = []
+
+        read_fields['id'] = restx_fields.String(required=True)
+
+        for key, field in cls._fields.items():
+            info = getattr(field, '__additional_field_info__', None)
+            if info is None: continue 
+
+            if info.get('sortable', False):
+                sortables.append(key)
+
+            read, write = convert_db_to_field(key, field)
+
+            if read:
+                read_fields[key] = read
+            if write:
+                write_fields[key] = write
+
+        # The goal of this loop is to fetch all functions (getters) of the class
+        # If a function has an `__additional_field_info__` attribute it means 
+        # it has been decorated with `@function_field()` and should be included
+        # in the API response.
+        for method_name in dir(cls):
+            if method_name == 'objects': continue
+            if method_name.startswith('_'): continue
+            if method_name in read_fields: continue # Do not override if the attribute is also callable like for Extras
+
+            method = getattr(cls, method_name)
+            if not callable(method): continue
+
+            info = getattr(method, '__additional_field_info__', None)
+            if info is None: continue
+
+            def make_lambda(method):
+                '''
+                Factory function to create a lambda with the correct scope.
+                If we don't have this factory function, the `method` will be the 
+                last method assigned in this loop?
+                '''
+                return lambda o: method(o)
+
+            read_fields[method_name] = restx_fields.String(attribute=make_lambda(method), **{ 'readonly':True, **info })
+
+
+        cls.__read_fields__ = api.model(f"{cls.__name__} (read)", read_fields, **kwargs)
+        cls.__write_fields__ = api.model(f"{cls.__name__} (write)", write_fields, **kwargs)
+
+        mask = kwargs.pop('mask', None)
+        if mask is not None:
+            mask = 'data{{{0}}},*'.format(mask)
+        cls.__page_fields__ = api.model(f"{cls.__name__}Page", custom_restx_fields.pager(cls.__read_fields__), mask=mask, **kwargs)
+
+        # Parser for index sort/filters
+        paginable = kwargs.get('paginable', True)
+        parser = api.parser()
+
+        if paginable:
+            parser.add_argument('page', type=int, location='args', default=1, help='The page to display')
+            parser.add_argument('page_size', type=int, location='args', default=20, help='The page size')
+        
+        if sortables:
+            choices = sortables + ['-' + k for k in sortables]
+            parser.add_argument('sort', type=str, location='args', choices=choices, help='The field (and direction) on which sorting apply')
+
+        cls.__index_parser__ = parser
+        def apply_sort_filters_and_pagination(base_query):
+            args = cls.__index_parser__.parse_args()
+
+            if sortables and args['sort']:
+                base_query = base_query.order_by(args['sort'])
+
+            if paginable:
+                base_query = base_query.paginate(args['page'], args['page_size'])
+
+            return base_query
+
+        cls.apply_sort_filters_and_pagination = apply_sort_filters_and_pagination
+        return cls
+    return wrapper
+
+
+def function_field(**info):
+    def inner(func):
+        func.__additional_field_info__ = info
+        return func
+
+    return inner
+
+def field(inner, **kwargs):
+    '''
+    Simple decorator to mark a field as visible for the API fields.
+    We can pass additional arguments that will be forward to the RestX field constructor.
+    '''
+    inner.__additional_field_info__ = kwargs
+    return inner
+
+
+def patch(obj, request): 
+    '''
+    Patch the object with the data from the request.
+    Only fields decorated with the `field()` decorator will be read (and not readonly).
+    '''
+    for key, value in request.json.items():
+        field = obj.__write_fields__.get(key)
+        if field is not None and not field.readonly:
+            model_attribute = getattr(obj.__class__, key)
+            if isinstance(model_attribute, mongoengine.fields.ListField) and isinstance(model_attribute.field, mongoengine.fields.ReferenceField):
+                # TODO `wrap_primary_key` do Mongo request, do a first pass to fetch all documents before calling it (to avoid multiple queries).
+                value = [wrap_primary_key(key, model_attribute.field, id) for id in value]
+            if isinstance(model_attribute, mongoengine.fields.ReferenceField):
+                value = wrap_primary_key(key, model_attribute, value)
+
+
+            info = getattr(model_attribute, '__additional_field_info__', {})
+
+            # `check` field attribute allows to do validation from the request before setting
+            # the attribute
+            check = info.get('check', None)
+            if check is not None:
+                check(**{key: value}) # TODO add other model attributes in function parameters
+
+            setattr(obj, key, value)
+
+    return obj
+
+def wrap_primary_key(field_name: str, foreign_field: mongoengine.fields.ReferenceField, value: str):
+    '''
+    We need to wrap the `String` inside an `ObjectId` most of the time. If the foreign ID is a `String` we need to get 
+    a `DBRef` from the database.
+
+    TODO: we only check the document reference if the ID is a `String` field (not in the case of a classic `ObjectId`).
+    '''
+    document_type = foreign_field.document_type()
+    id_field_name = document_type.__class__._meta["id_field"]
+
+    id_field = getattr(document_type.__class__, id_field_name)
+
+    # Get the foreign document from MongoDB because the othewise it fails during read
+    # Also useful to get a DBRef for non ObjectId references (see below)
+    foreign_document = document_type.__class__.objects(**{id_field_name: value}).first()
+    if foreign_document is None:
+        raise FieldValidationError(field=field_name, message=f"Unknown reference '{value}'")
+    
+    if isinstance(id_field, mongoengine.fields.ObjectIdField):
+        return ObjectId(value)
+    elif isinstance(id_field, mongoengine.fields.StringField):
+        # Right now I didn't find a simpler way to make mongoengine happy.
+        # For references, it expects `ObjectId`, `DBRef`, `LazyReference` or `document` but since
+        # the primary key a StringField (not an `ObjectId`) we cannot create an `ObjectId`, I didn't find
+        # a way to create a `DBRef` nor a `LazyReference` so I create a simple document with only the ID field.
+        # We could use a simple dict as follow instead:
+        # { 'id': value }
+        # … but it may be important to check before-hand that the reference point to a correct document.
+        return foreign_document.to_dbref()
+    else:
+        raise ValueError(f"Unknown ID field type {id_field.__class__} for {document_type.__class__} (ID field name is {id_field_name}, value was {value})")
+

udata/commands/purge.py

@@ -5,6 +5,7 @@ import click
 from udata.commands import cli, success
 
 from udata.core.dataset.tasks import purge_datasets
+from udata.core.dataservices.tasks import purge_dataservices
 from udata.core.organization.tasks import purge_organizations
 from udata.core.reuse.tasks import purge_reuses
 
@@ -15,13 +16,14 @@ log = logging.getLogger(__name__)
 @click.option('-d', '--datasets', is_flag=True)
 @click.option('-r', '--reuses', is_flag=True)
 @click.option('-o', '--organizations', is_flag=True)
-def purge(datasets, reuses, organizations):
+@click.option('--dataservices', is_flag=True)
+def purge(datasets, reuses, organizations, dataservices):
     '''
     Permanently remove data flagged as deleted.
 
     If no model flag is given, all models are purged.
     '''
-    purge_all = not any((datasets, reuses, organizations))
+    purge_all = not any((datasets, reuses, organizations, dataservices))
 
     if purge_all or datasets:
         log.info('Purging datasets')
@@ -35,4 +37,8 @@ def purge(datasets, reuses, organizations):
         log.info('Purging organizations')
         purge_organizations()
 
+    if purge_all or dataservices:
+        log.info('Purging dataservices')
+        purge_dataservices()
+
     success('Done')

udata/core/badges/models.py

@@ -1,14 +1,15 @@
 import logging
-import weakref
 
 from datetime import datetime
 
 from mongoengine.signals import post_save
 
+from udata.api_fields import field
 from udata.auth import current_user
 from udata.mongo import db
 
 from .signals import on_badge_added, on_badge_removed
+from .fields import badge_fields
 
 log = logging.getLogger(__name__)
 

udata/core/dataservices/api.py

@@ -0,0 +1,92 @@
+from datetime import datetime
+from flask import request
+from flask_login import current_user
+import mongoengine
+
+from udata.api import api, API
+from udata.api_fields import patch
+from udata.core.dataset.permissions import OwnablePermission
+from udata.core.followers.api import FollowAPI
+from .models import Dataservice
+
+ns = api.namespace('dataservices', 'Dataservices related operations (beta)')
+
+
+@ns.route('/', endpoint='dataservices')
+class DataservicesAPI(API):
+    '''Dataservices collection endpoint'''
+    @api.doc('list_dataservices')
+    @api.expect(Dataservice.__index_parser__)
+    @api.marshal_with(Dataservice.__page_fields__)
+    def get(self):
+        '''List or search all dataservices'''
+        query = Dataservice.objects.visible()
+
+        return Dataservice.apply_sort_filters_and_pagination(query)
+
+    @api.secure
+    @api.doc('create_dataservice', responses={400: 'Validation error'})
+    @api.expect(Dataservice.__write_fields__)
+    @api.marshal_with(Dataservice.__read_fields__, code=201)
+    def post(self):
+        dataservice = patch(Dataservice(), request)
+        if not dataservice.owner and not dataservice.organization:
+            dataservice.owner = current_user._get_current_object()
+
+        try:
+            dataservice.save()
+        except mongoengine.errors.ValidationError as e:
+            api.abort(400, e.message)
+
+        return dataservice, 201
+
+
+@ns.route('/<dataservice:dataservice>/', endpoint='dataservice')
+class DataserviceAPI(API):
+    @api.doc('get_dataservice')
+    @api.marshal_with(Dataservice.__read_fields__)
+    def get(self, dataservice):
+        if dataservice.deleted_at and not OwnablePermission(dataservice).can():
+            api.abort(410, 'Dataservice has been deleted')
+        return dataservice
+
+    @api.secure
+    @api.doc('update_dataservice', responses={400: 'Validation error'})
+    @api.expect(Dataservice.__write_fields__)
+    @api.marshal_with(Dataservice.__read_fields__)
+    def patch(self, dataservice):
+        if dataservice.deleted_at:
+            api.abort(410, 'dataservice has been deleted')
+
+        OwnablePermission(dataservice).test()
+
+        patch(dataservice, request)
+        dataservice.modified_at = datetime.utcnow()
+
+        try:
+            dataservice.save()
+            return dataservice
+        except mongoengine.errors.ValidationError as e:
+            api.abort(400, e.message)
+
+    @api.secure
+    @api.doc('delete_dataservice')
+    @api.response(204, 'dataservice deleted')
+    def delete(self, dataservice):
+        if dataservice.deleted_at:
+            api.abort(410, 'dataservice has been deleted')
+
+        OwnablePermission(dataservice).test()
+        dataservice.deleted_at = datetime.utcnow()
+        dataservice.modified_at = datetime.utcnow()
+        dataservice.save()
+
+        return '', 204
+
+
+@ns.route('/<id>/followers/', endpoint='dataservice_followers')
+@ns.doc(get={'id': 'list_dataservice_followers'},
+        post={'id': 'follow_dataservice'},
+        delete={'id': 'unfollow_dataservice'})
+class DataserviceFollowersAPI(FollowAPI):
+    model = Dataservice

udata/core/dataservices/models.py

@@ -0,0 +1,142 @@
+from datetime import datetime
+from udata.api_fields import field, function_field, generate_fields
+from udata.core.dataset.models import Dataset
+from udata.core.metrics.models import WithMetrics
+from udata.core.owned import Owned, OwnedQuerySet
+import udata.core.contact_point.api_fields as contact_api_fields
+import udata.core.dataset.api_fields as datasets_api_fields
+from udata.i18n import lazy_gettext as _
+
+from udata.models import db, Discussion, Follow
+from udata.uris import endpoint_for
+
+# "frequency"
+# "harvest"
+# "internal"
+# "page"
+# "quality" # Peut-être pas dans une v1 car la qualité sera probablement calculé différemment
+# "datasets" # objet : liste de datasets liés à une API
+# "spatial"
+# "temporal_coverage"
+
+DATASERVICE_FORMATS = ['REST', 'WMS', 'WSL']
+
+
+class DataserviceQuerySet(OwnedQuerySet):
+    def visible(self):
+        return self(archived_at=None, deleted_at=None, private=False)
+
+    def hidden(self):
+        return self(db.Q(private=True) |
+                    db.Q(deleted_at__ne=None) |
+                    db.Q(archived_at__ne=None))
+
+@generate_fields()
+class Dataservice(WithMetrics, Owned, db.Document):
+    meta = {
+        'indexes': [
+            '$title',
+        ] + Owned.meta['indexes'],
+        'queryset_class': DataserviceQuerySet,
+        'auto_create_index_on_save': True
+    }
+
+    title = field(
+        db.StringField(required=True),
+        example="My awesome API",
+        sortable=True,
+    )
+    acronym = field(
+        db.StringField(max_length=128),
+    )
+    # /!\ do not set directly the slug when creating or updating a dataset
+    # this will break the search indexation
+    slug = field(
+        db.SlugField(max_length=255, required=True, populate_from='title', update=True, follow=True),
+        readonly=True,
+    )
+    description = field(
+        db.StringField(default=''),
+        description="In markdown"
+    )
+    base_api_url = field(
+        db.URLField(required=True),
+        sortable=True,
+    )
+    endpoint_description_url = field(db.URLField())
+    authorization_request_url = field(db.URLField())
+    availability= field(
+        db.FloatField(min=0, max=100),
+        example='99.99'
+    )
+    rate_limiting = field(db.StringField())
+    is_restricted = field(db.BooleanField())
+    has_token = field(db.BooleanField())
+    format = field(db.StringField(choices=DATASERVICE_FORMATS))
+
+    license = field(
+        db.ReferenceField('License'),
+        allow_null=True,
+    )
+
+    tags = field(
+        db.TagListField(),
+    )
+
+    private = field(
+        db.BooleanField(default=False),
+        description='Is the dataservice private to the owner or the organization'
+    )
+    
+    extras = field(db.ExtrasField())
+
+    contact_point = field(
+        db.ReferenceField('ContactPoint', reverse_delete_rule=db.NULLIFY),
+        nested_fields=contact_api_fields.contact_point_fields,
+        allow_null=True,
+    )
+
+    created_at = field(
+        db.DateTimeField(verbose_name=_('Creation date'), default=datetime.utcnow, required=True),
+        readonly=True,
+    )
+    metadata_modified_at = field(
+        db.DateTimeField(verbose_name=_('Last modification date'), default=datetime.utcnow, required=True),
+        readonly=True,
+    )
+    deleted_at = field(db.DateTimeField(), readonly=True)
+    archived_at = field(db.DateTimeField(), readonly=True)
+
+    datasets = field(
+        db.ListField(
+            field(
+                db.ReferenceField(Dataset),
+                nested_fields=datasets_api_fields.dataset_fields,
+            )
+        )
+    )
+
+    @function_field(description="Link to the API endpoint for this dataservice")
+    def self_api_url(self):
+        return endpoint_for('api.dataservice', dataservice=self, _external=True)
+
+    def self_web_url():
+        pass
+
+    # TODO
+    # frequency = db.StringField(choices=list(UPDATE_FREQUENCIES.keys()))
+    # temporal_coverage = db.EmbeddedDocumentField(db.DateRange)
+    # spatial = db.EmbeddedDocumentField(SpatialCoverage)
+    # harvest = db.EmbeddedDocumentField(HarvestDatasetMetadata)
+
+    @property
+    def is_hidden(self):
+        return self.private or self.deleted_at or self.archived_at
+
+    def count_discussions(self):
+        self.metrics['discussions'] = Discussion.objects(subject=self, closed=None).count()
+        self.save()
+
+    def count_followers(self):
+        self.metrics['followers'] = Follow.objects(until=None).followers(self).count()
+        self.save()

udata/core/dataservices/permissions.py

@@ -0,0 +1,7 @@
+from udata.core.dataset.permissions import (
+    OwnablePermission
+)
+
+class DataserviceEditPermission(OwnablePermission):
+    '''Permissions to edit a Dataservice'''
+    pass

udata/core/dataservices/tasks.py

@@ -0,0 +1,25 @@
+from celery.utils.log import get_task_logger
+
+from udata.core.dataservices.models import Dataservice
+# from udata.harvest.models import HarvestJob
+from udata.models import (Follow, Discussion, Activity, Transfer)
+from udata.tasks import job
+
+log = get_task_logger(__name__)
+
+
+@job('purge-dataservices')
+def purge_dataservices(self):
+    for dataservice in Dataservice.objects(deleted_at__ne=None):
+        log.info(f'Purging dataservice {dataservice}')
+        # Remove followers
+        Follow.objects(following=dataservice).delete()
+        # Remove discussions
+        Discussion.objects(subject=dataservice).delete()
+        # Remove HarvestItem references
+        # TODO: uncomment when adding dataservice harvest
+        # HarvestJob.objects(items__dataservice=dataservice).update(set__items__S__dataservice=None)
+        # Remove associated Transfers
+        Transfer.objects(subject=dataservice).delete()
+        # Remove dataservice
+        dataservice.delete()

udata/core/dataset/apiv2.py

@@ -8,6 +8,7 @@ from udata import search
 from udata.api import apiv2, API, fields
 from udata.utils import multi_to_dict, get_by
 
+from udata.core.organization.api_fields import member_user_with_email_fields
 from .api_fields import (
     badge_fields,
     org_ref_fields,
@@ -165,6 +166,7 @@ specific_resource_fields = apiv2.model('SpecificResource', {
 apiv2.inherit('Badge', badge_fields)
 apiv2.inherit('OrganizationReference', org_ref_fields)
 apiv2.inherit('UserReference', user_ref_fields)
+apiv2.inherit('MemberUserWithEmail', member_user_with_email_fields)
 apiv2.inherit('Resource', resource_fields)
 apiv2.inherit('SpatialCoverage', spatial_coverage_fields)
 apiv2.inherit('TemporalCoverage', temporal_coverage_fields)