udata 8.0.2.dev29304__py2.py3-none-any.whl → 9.1.0__py2.py3-none-any.whl

udata/__init__.py

@@ -4,5 +4,5 @@
 udata
 '''
 
-__version__ = '8.0.2.dev'
+__version__ = '9.1.0'
 __description__ = 'Open data portal'

udata/api/__init__.py

@@ -323,6 +323,7 @@ def init_app(app):
     import udata.core.activity.api  # noqa
     import udata.core.spatial.api  # noqa
     import udata.core.user.api  # noqa
+    import udata.core.user.apiv2  # noqa
     import udata.core.dataset.api  # noqa
     import udata.core.dataset.apiv2  # noqa
     import udata.core.dataservices.api  # noqa
@@ -333,6 +334,7 @@ def init_app(app):
     import udata.core.organization.apiv2  # noqa
     import udata.core.followers.api  # noqa
     import udata.core.jobs.api  # noqa
+    import udata.core.reports.api  # noqa
     import udata.core.site.api  # noqa
     import udata.core.tags.api  # noqa
     import udata.core.topic.api  # noqa

udata/api/commands.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import time
 
 import click
 
@@ -19,7 +18,6 @@ log = logging.getLogger(__name__)
 @cli.group('api')
 def grp():
     '''API related operations'''
-    pass
 
 
 def json_to_file(data, filename, pretty=False):

udata/api_fields.py

@@ -38,6 +38,9 @@ def convert_db_to_field(key, field, info = {}):
         constructor = restx_fields.String
         params['min_length'] = field.min_length
         params['max_length'] = field.max_length
+        params['enum'] = field.choices
+    elif isinstance(field, mongo_fields.ObjectIdField):
+        constructor = restx_fields.String
     elif isinstance(field, mongo_fields.FloatField):
         constructor = restx_fields.Float
         params['min'] = field.min # TODO min_value?
@@ -70,10 +73,14 @@ def convert_db_to_field(key, field, info = {}):
         constructor_write = restx_fields.String
     elif isinstance(field, mongo_fields.EmbeddedDocumentField):
         nested_fields = info.get('nested_fields')
-        if nested_fields is None:
-            raise ValueError(f"EmbeddedDocumentField `{key}` requires a `nested_fields` param to serialize/deserialize.")
+        if nested_fields is not None:
+            constructor = lambda **kwargs: restx_fields.Nested(nested_fields, **kwargs)
+        elif hasattr(field.document_type_obj, '__read_fields__'):
+            constructor_read = lambda **kwargs: restx_fields.Nested(field.document_type_obj.__read_fields__, **kwargs)
+            constructor_write = lambda **kwargs: restx_fields.Nested(field.document_type_obj.__write_fields__, **kwargs)
+        else:
+            raise ValueError(f"EmbeddedDocumentField `{key}` requires a `nested_fields` param to serialize/deserialize or a `@generate_fields()` definition.")
 
-        constructor = lambda **kwargs: restx_fields.Nested(nested_fields, **kwargs)
     else:
         raise ValueError(f"Unsupported MongoEngine field type {field.__class__.__name__}")
     
@@ -96,6 +103,7 @@ def generate_fields(**kwargs):
         read_fields = {}
         write_fields = {}
         sortables = []
+        filterables = []
 
         read_fields['id'] = restx_fields.String(required=True)
 
@@ -106,6 +114,23 @@ def generate_fields(**kwargs):
             if info.get('sortable', False):
                 sortables.append(key)
 
+            filterable = info.get('filterable', None)
+            if filterable is not None:
+                if 'key' not in filterable:
+                    filterable['key'] = key
+                if 'column' not in filterable:
+                    filterable['column'] = key
+
+                if 'constraints' not in filterable:
+                    filterable['constraints'] = []
+                    if isinstance(field, mongo_fields.ReferenceField) or (isinstance(field, mongo_fields.ListField) and isinstance(field.field, mongo_fields.ReferenceField)):
+                        filterable['constraints'].append('objectid')
+
+                # We may add more information later here:
+                # - type of mongo query to execute (right now only simple =)
+
+                filterables.append(filterable)
+
             read, write = convert_db_to_field(key, field)
 
             if read:
@@ -159,6 +184,9 @@ def generate_fields(**kwargs):
             choices = sortables + ['-' + k for k in sortables]
             parser.add_argument('sort', type=str, location='args', choices=choices, help='The field (and direction) on which sorting apply')
 
+        for filterable in filterables:
+            parser.add_argument(filterable['key'], type=str, location='args')
+
         cls.__index_parser__ = parser
         def apply_sort_filters_and_pagination(base_query):
             args = cls.__index_parser__.parse_args()
@@ -166,6 +194,16 @@ def generate_fields(**kwargs):
             if sortables and args['sort']:
                 base_query = base_query.order_by(args['sort'])
 
+            for filterable in filterables:
+                if args.get(filterable['key']):
+                    for constraint in filterable['constraints']:
+                        if constraint == 'objectid' and not ObjectId.is_valid(args[filterable['key']]):
+                            api.abort(400, f'`{filterable["key"]}` must be an identifier')
+
+                    base_query = base_query.filter(**{
+                        filterable['column']: args[filterable['key']],
+                    })
+
             if paginable:
                 base_query = base_query.paginate(args['page'], args['page_size'])
 

udata/commands/db.py

@@ -1,7 +1,10 @@
 import collections
+from itertools import groupby
 import logging
 import os
+import traceback
 
+from bson import DBRef
 import click
 import mongoengine
 
@@ -135,8 +138,14 @@ def display_op(op):
     echo('{label:.<70} [{date}]'.format(label=label, date=timestamp))
     format_output(op['output'], success=op['success'], traceback=op.get('traceback'))
 
-
 def check_references(models_to_check):
+    # Cannot modify local scope from Python… :-(
+    class Log: errors = []
+
+    def print_and_save(text: str):
+        Log.errors.append(text.strip())
+        print(text) 
+
     errors = collections.defaultdict(int)
 
     _models = []
@@ -147,7 +156,7 @@ def check_references(models_to_check):
         ]
 
     references = []
-    for model in _models:
+    for model in set(_models):
         if model.__name__ == 'Activity':
             print(f'Skipping Activity model, scheduled for deprecation')
             continue
@@ -240,53 +249,84 @@ def check_references(models_to_check):
         print(f'- {reference["repr"]}({reference["destination"]}) — {reference["type"]}')
     print('')
 
-    for reference in references:
-        print(f'- {reference["repr"]}({reference["destination"]}) — {reference["type"]}...')
-        query = {f'{reference["name"]}__ne': None}
-        qs = reference['model'].objects(**query).no_cache().all()
-        try:
-            for obj in qs:
-                if reference['type'] == 'direct':
-                    try:
-                        _ = getattr(obj, reference['name'])
-                    except mongoengine.errors.DoesNotExist:
-                        errors[reference["repr"]] += 1
-                elif reference['type'] == 'list':
-                    for sub in getattr(obj, reference['name']):
-                        try:
-                            _ = sub.id
-                        except mongoengine.errors.DoesNotExist:
-                            errors[reference["repr"]] += 1
-                elif reference['type'] == 'embed_list':
-                    p1, p2 = reference['name'].split('__')
-                    for sub in getattr(obj, p1):
-                        try:
-                            getattr(sub, p2)
-                        except mongoengine.errors.DoesNotExist:
-                            errors[reference["repr"]] += 1
-                elif reference['type'] == 'embed':
-                    p1, p2 = reference['name'].split('__')
-                    sub = getattr(obj, p1)
-                    try:
-                        getattr(sub, p2)
-                    except mongoengine.errors.DoesNotExist:
-                        errors[reference["repr"]] += 1
-                elif reference['type'] == 'embed_list_ref':
-                    p1, p2 = reference['name'].split('__')
-                    sub = getattr(getattr(obj, p1), p2)
-                    for obj in sub:
-                        try:
-                            obj.id
-                        except mongoengine.errors.DoesNotExist:
-                            errors[reference["repr"]] += 1
-                else:
-                    print(f'Unknown ref type {reference["type"]}')
-            print('Errors:', errors[reference["repr"]])
-        except mongoengine.errors.FieldDoesNotExist as e:
-            print('[ERROR]', e)
-
-    print(f'\n Total errors: {sum(errors.values())}')
+    total = 0
+    for model, model_references in groupby(references, lambda i: i["model"]):
+        model_references = list(model_references)
+        count = model.objects.count()
+        print(f'- doing {count} {model.__name__}…')
+        errors[model] = {}
+
+        qs = model.objects().no_cache().all()
+        with click.progressbar(qs, length=count) as models:
+            for obj in models:
+                for reference in model_references:
+                    key = f'\t- {reference["repr"]}({reference["destination"]}) — {reference["type"]}…'
+                    if key not in errors[model]:
+                        errors[model][key] = 0
 
+                    try:
+                        if reference['type'] == 'direct':
+                            try:
+                                _ = getattr(obj, reference['name'])
+                            except mongoengine.errors.DoesNotExist:
+                                errors[model][key] += 1
+                                print_and_save(f'\t{model.__name__}#{obj.id} have a broken reference for `{reference["name"]}`')
+                        elif reference['type'] == 'list':
+                            attr_list = getattr(obj, reference['name'], [])
+                            for i, sub in enumerate(attr_list):
+                                # If it's still an instance of DBRef it means that it failed to
+                                # dereference the ID.
+                                if isinstance(sub, DBRef):
+                                    errors[model][key] += 1
+                                    print_and_save(f'\t{model.__name__}#{obj.id} have a broken reference for {reference["name"]}[{i}]')
+                        elif reference['type'] == 'embed_list':
+                            p1, p2 = reference['name'].split('__')
+                            attr_list = getattr(obj, p1, [])
+                            for i, sub in enumerate(attr_list):
+                                try:
+                                    getattr(sub, p2)
+                                except mongoengine.errors.DoesNotExist:
+                                    errors[model][key] += 1
+                                    print_and_save(f'\t{model.__name__}#{obj.id} have a broken reference for {p1}[{i}].{p2}')
+                        elif reference['type'] == 'embed':
+                            p1, p2 = reference['name'].split('__')
+                            sub = getattr(obj, p1)
+                            if sub is None: continue
+                            try:
+                                getattr(sub, p2)
+                            except mongoengine.errors.DoesNotExist:
+                                errors[model][key] += 1
+                                print_and_save(f'\t{model.__name__}#{obj.id} have a broken reference for {p1}.{p2}')
+                        elif reference['type'] == 'embed_list_ref':
+                            p1, p2 = reference['name'].split('__')
+                            a = getattr(obj, p1)
+                            if a is None: continue
+                            sub = getattr(a, p2, [])
+                            for i, child in enumerate(sub):
+                                # If it's still an instance of DBRef it means that it failed to
+                                # dereference the ID.
+                                if isinstance(child, DBRef):
+                                    errors[model][key] += 1
+                                    print_and_save(f'\t{model.__name__}#{obj.id} have a broken reference for {p1}.{p2}[{i}]')
+                        else:
+                            print_and_save(f'Unknown ref type {reference["type"]}')
+                    except mongoengine.errors.FieldDoesNotExist as e:
+                        print_and_save(f'[ERROR for {model.__name__} {obj.id}] {traceback.format_exc()}')
+
+        for key, nb_errors in errors[model].items():
+            print(f'{key}: {nb_errors}')
+            total += nb_errors
+
+    print(f'\n Total errors: {total}')
+
+    if total > 0:
+        try:
+            import sentry_sdk
+            with sentry_sdk.push_scope() as scope:
+                scope.set_extra("errors", Log.errors)
+                sentry_sdk.capture_message(f"{total} integrity errors", "fatal")
+        except ImportError:
+            print("`sentry_sdk` not installed. The errors weren't reported")
 
 @grp.command()
 @click.option('--models', multiple=True, default=[], help='Model(s) to check')

udata/core/dataservices/factories.py

@@ -0,0 +1,33 @@
+import factory
+
+from udata.core.dataservices.models import Dataservice, HarvestMetadata
+from udata.core.organization.factories import OrganizationFactory
+from udata.factories import ModelFactory
+
+class HarvestMetadataFactory(ModelFactory):
+    class Meta:
+        model = HarvestMetadata
+
+    backend = 'csw-dcat'
+    domain = 'data.gouv.fr'
+
+    source_id = factory.Faker('unique_string')
+    source_url = factory.Faker('url')
+
+    remote_id = factory.Faker('unique_string')
+    remote_url = factory.Faker('url')
+
+    uri = factory.Faker('url')
+
+class DataserviceFactory(ModelFactory):
+    class Meta:
+        model = Dataservice
+
+    title = factory.Faker('sentence')
+    description = factory.Faker('text')
+    base_api_url = factory.Faker('url')
+
+    class Params:
+        org = factory.Trait(
+            organization=factory.SubFactory(OrganizationFactory),
+        )

udata/core/dataservices/models.py

@@ -31,6 +31,35 @@ class DataserviceQuerySet(OwnedQuerySet):
                     db.Q(deleted_at__ne=None) |
                     db.Q(archived_at__ne=None))
 
+@generate_fields()
+class HarvestMetadata(db.EmbeddedDocument):
+    backend = field(db.StringField())
+    domain = field(db.StringField())
+
+    source_id = field(db.StringField())
+    source_url = field(db.URLField())
+
+    remote_id = field(db.StringField())
+    remote_url = field(db.URLField())
+
+    # If the node ID is a `URIRef` it means it links to something external, if it's not an `URIRef` it's often a
+    # auto-generated ID just to link multiple RDF node togethers. When exporting as RDF to other catalogs, we 
+    # want to re-use this node ID (only if it's not auto-generated) to improve compatibility.
+    uri = field(
+        db.URLField(),
+        description="RDF node ID if it's an `URIRef`. `None` if it's not present or if it's a random auto-generated ID inside the graph.",
+    )
+
+    created_at = field(
+        db.DateTimeField(),
+        description="Date of the creation as provided by the harvested catalog"
+    )
+    last_update = field(
+        db.DateTimeField(),
+        description="Date of the last harvesting"
+    )
+    archived_at = field(db.DateTimeField())
+
 @generate_fields()
 class Dataservice(WithMetrics, Owned, db.Document):
     meta = {
@@ -111,17 +140,26 @@ class Dataservice(WithMetrics, Owned, db.Document):
         db.ListField(
             field(
                 db.ReferenceField(Dataset),
-                nested_fields=datasets_api_fields.dataset_fields,
+                nested_fields=datasets_api_fields.dataset_ref_fields,
             )
-        )
+        ),
+        filterable={
+            'key': 'dataset',
+        },
+    )
+
+    harvest = field(
+        db.EmbeddedDocumentField(HarvestMetadata),
+        readonly=True,
     )
 
     @function_field(description="Link to the API endpoint for this dataservice")
     def self_api_url(self):
         return endpoint_for('api.dataservice', dataservice=self, _external=True)
 
-    def self_web_url():
-        pass
+    @function_field(description="Link to the udata web page for this dataservice")
+    def self_web_url(self):
+        return endpoint_for('dataservices.show', dataservice=self, _external=True)
 
     # TODO
     # frequency = db.StringField(choices=list(UPDATE_FREQUENCIES.keys()))

udata/core/dataservices/rdf.py

@@ -0,0 +1,106 @@
+
+from rdflib import RDF, BNode, Graph, Literal, URIRef
+
+from udata.core.dataservices.models import Dataservice, HarvestMetadata as HarvestDataserviceMetadata
+from udata.core.dataset.models import Dataset, License
+from udata.core.dataset.rdf import dataset_to_graph_id, sanitize_html
+from udata.rdf import namespace_manager, DCAT, DCT, contact_point_from_rdf, rdf_value, remote_url_from_rdf, themes_from_rdf, url_from_rdf
+from udata.uris import endpoint_for
+
+def dataservice_from_rdf(graph: Graph, dataservice: Dataservice, node, all_datasets: list[Dataset]) -> Dataservice :
+    '''
+    Create or update a dataset from a RDF/DCAT graph
+    '''
+    if node is None:  # Assume first match is the only match
+        node = graph.value(predicate=RDF.type, object=DCAT.DataService)
+
+    d = graph.resource(node)
+
+    dataservice.title = rdf_value(d, DCT.title)
+    dataservice.description = sanitize_html(d.value(DCT.description) or d.value(DCT.abstract))
+
+    dataservice.base_api_url = url_from_rdf(d, DCAT.endpointURL)
+    dataservice.endpoint_description_url = url_from_rdf(d, DCAT.endpointDescription)
+
+    dataservice.contact_point = contact_point_from_rdf(d, dataservice) or dataservice.contact_point
+
+    datasets = []
+    for dataset_node in d.objects(DCAT.servesDataset):
+        id = dataset_node.value(DCT.identifier)
+        dataset = next((d for d in all_datasets if d is not None and d.harvest.remote_id == id), None)
+
+        if dataset is None:
+            # We try with `endswith` because Europe XSLT have problems with IDs. Sometimes they are prefixed with the domain of the catalog, sometimes not.
+            dataset = next((d for d in all_datasets if d is not None and d.harvest.remote_id.endswith(id)), None)
+
+        if dataset is not None:
+            datasets.append(dataset.id)
+
+    if datasets:
+        dataservice.datasets = datasets
+
+    license = rdf_value(d, DCT.license)
+    if license is not None:
+        dataservice.license = License.guess(license)
+
+    if not dataservice.harvest:
+        dataservice.harvest = HarvestDataserviceMetadata()
+
+    dataservice.harvest.uri = d.identifier.toPython() if isinstance(d.identifier, URIRef) else None
+    dataservice.harvest.remote_url = remote_url_from_rdf(d)
+    dataservice.harvest.created_at = rdf_value(d, DCT.issued)
+    dataservice.metadata_modified_at = rdf_value(d, DCT.modified)
+
+    dataservice.tags = themes_from_rdf(d)
+
+    return dataservice
+
+
+def dataservice_to_rdf(dataservice: Dataservice, graph=None):
+    '''
+    Map a dataservice domain model to a DCAT/RDF graph
+    '''
+    # Use the unlocalized permalink to the dataset as URI when available
+    # unless there is already an upstream URI
+    if dataservice.harvest and dataservice.harvest.uri:
+        id = URIRef(dataservice.harvest.uri)
+    elif dataservice.id:
+        id = URIRef(endpoint_for('dataservices.show_redirect', 'api.dataservice',
+                    dataservice=dataservice.id, _external=True))
+    else:
+        # Should not happen in production. Some test only
+        # `build()` a dataset without saving it to the DB.
+        id = BNode()
+
+    # Expose upstream identifier if present
+    if dataservice.harvest:
+        identifier = dataservice.harvest.remote_id
+    else:
+        identifier = dataservice.id
+    graph = graph or Graph(namespace_manager=namespace_manager)
+
+    d = graph.resource(id)
+    d.set(RDF.type, DCAT.DataService)
+    d.set(DCT.identifier, Literal(identifier))
+    d.set(DCT.title, Literal(dataservice.title))
+    d.set(DCT.description, Literal(dataservice.description))
+    d.set(DCT.issued, Literal(dataservice.created_at))
+    
+    if dataservice.base_api_url:
+        d.set(DCAT.endpointURL, Literal(dataservice.base_api_url))
+    
+    if dataservice.endpoint_description_url:
+        d.set(DCAT.endpointDescription, Literal(dataservice.endpoint_description_url))
+
+    for tag in dataservice.tags:
+        d.add(DCAT.keyword, Literal(tag))
+
+    # `dataset_to_graph_id(dataset)` URIRef may not exist in the current page
+    # but should exists in the catalog somewhere. Maybe we should create a Node 
+    # with some basic information about this dataset (but this will return a page 
+    # with more datasets than the page size… and could be problematic when processing the
+    # correct Node with all the information in a future page)
+    for dataset in dataservice.datasets:
+        d.add(DCAT.servesDataset, dataset_to_graph_id(dataset))
+
+    return d

udata/core/dataset/csv.py

@@ -19,6 +19,9 @@ class DatasetCsvAdapter(csv.Adapter):
         ('url', 'external_url'),
         ('organization', 'organization.name'),
         ('organization_id', 'organization.id'),
+        ('owner', 'owner.slug'),  # in case it's owned by a user, or introduce 'owner_type'?
+        ('owner_id', 'owner.id'),
+        # 'contact_point', #  ?
         'description',
         'frequency',
         'license',
@@ -26,19 +29,20 @@ class DatasetCsvAdapter(csv.Adapter):
         'temporal_coverage.end',
         'spatial.granularity',
         ('spatial.zones', serialize_spatial_zones),
-        'private',
         ('featured', lambda o: o.featured or False),
         'created_at',
         'last_modified',
         ('tags', lambda o: ','.join(o.tags)),
         ('archived', lambda o: o.archived or False),
         ('resources_count', lambda o: len(o.resources)),
+        ('main_resources_count', lambda o: len([r for r in o.resources if r.type == 'main'])),
         'downloads',
         ('harvest.backend', lambda r: r.harvest and r.harvest.backend),
         ('harvest.domain', lambda r: r.harvest and r.harvest.domain),
         ('harvest.created_at', lambda r: r.harvest and r.harvest.created_at),
         ('harvest.modified_at', lambda r: r.harvest and r.harvest.modified_at),
         ('quality_score', lambda o: format(o.quality['score'], '.2f')),
+        # schema? what is the schema of a dataset?
     )
 
     def dynamic_fields(self):
@@ -85,6 +89,9 @@ class ResourcesCsvAdapter(csv.NestedAdapter):
         ('downloads', lambda o: int(o.metrics.get('views', 0))),
         ('harvest.created_at', lambda o: o.harvest and o.harvest.created_at),
         ('harvest.modified_at', lambda o: o.harvest and o.harvest.modified_at),
+        ('schema_name', 'schema.name'),
+        ('schema_version', 'schema.version'),
+        ('preview_url', lambda o: o.preview_url or False),
     )
     attribute = 'resources'
 

udata/core/dataset/models.py

@@ -13,7 +13,6 @@ from pydoc import locate
 from stringdist import rdlevenshtein
 from werkzeug.utils import cached_property
 import requests
-from typing import Optional, Tuple
 
 from udata.app import cache
 from udata.core import storages
@@ -964,7 +963,7 @@ class ResourceSchema(object):
     def assignable_schemas():
         return [s for s in ResourceSchema.all() if s.get('schema_type') not in NON_ASSIGNABLE_SCHEMA_TYPES]
 
-    def get_existing_schema_info_by_url(url: str) -> Optional[Tuple[str, Optional[str]]]:
+    def get_existing_schema_info_by_url(url: str) -> tuple[str, str | None] | None:
         '''
         Returns the name and the version if exists
         '''