udata 10.1.4.dev34377__py2.py3-none-any.whl → 10.1.4.dev34401__py2.py3-none-any.whl

udata/core/dataset/apiv2.py

@@ -2,6 +2,7 @@ import logging
 
 import mongoengine
 from flask import abort, request, url_for
+from flask_login import current_user
 from flask_restx import marshal
 
 from udata import search
@@ -11,7 +12,7 @@ from udata.core.organization.api_fields import member_user_with_email_fields
 from udata.core.spatial.api_fields import geojson
 from udata.utils import get_by
 
-from .api import ResourceMixin
+from .api import DEFAULT_SORTING, DatasetApiParser, ResourceMixin
 from .api_fields import (
     badge_fields,
     catalog_schema_fields,
@@ -128,7 +129,7 @@ dataset_fields = apiv2.model(
                     _external=True,
                 ),
                 "type": "GET",
-                "total": len(o.resources),
+                "total": o.resources_len,  # :ResourcesLengthProperty may call MongoDB to fetch the length if resources were not fetched
             },
             description="Link to the dataset resources",
         ),
@@ -155,7 +156,7 @@ dataset_fields = apiv2.model(
         ),
         "frequency_date": fields.ISODateTime(
             description=(
-                "Next expected update date, you will be notified " "once that date is reached."
+                "Next expected update date, you will be notified once that date is reached."
             )
         ),
         "harvest": fields.Nested(
@@ -276,7 +277,28 @@ class DatasetSearchAPI(API):
             abort(500, "Internal search service error")
 
 
-@ns.route("/<dataset:dataset>/", endpoint="dataset", doc=common_doc)
+dataset_parser = DatasetApiParser()
+
+
+@ns.route("/", endpoint="datasets")
+class DatasetListAPI(API):
+    """Datasets collection endpoint"""
+
+    @apiv2.doc("list_datasets")
+    @apiv2.expect(dataset_parser.parser)
+    @apiv2.marshal_with(dataset_page_fields)
+    def get(self):
+        """List or search all datasets"""
+        args = dataset_parser.parse()
+        datasets = Dataset.objects.exclude("resources").visible_by_user(
+            current_user, mongoengine.Q(private__ne=True, archived=None, deleted=None)
+        )
+        datasets = dataset_parser.parse_filters(datasets, args)
+        sort = args["sort"] or ("$text_score" if args["q"] else None) or DEFAULT_SORTING
+        return datasets.order_by(sort).paginate(args["page"], args["page_size"])
+
+
+@ns.route("/<dataset_without_resources:dataset>/", endpoint="dataset", doc=common_doc)
 @apiv2.response(404, "Dataset not found")
 @apiv2.response(410, "Dataset has been deleted")
 class DatasetAPI(API):

udata/core/dataset/models.py

@@ -2,6 +2,7 @@ import logging
 import re
 from datetime import datetime, timedelta
 from pydoc import locate
+from typing import Self
 from urllib.parse import urlparse
 
 import requests
@@ -11,7 +12,7 @@ from flask import current_app
 from mongoengine import DynamicEmbeddedDocument
 from mongoengine import ValidationError as MongoEngineValidationError
 from mongoengine.fields import DateTimeField
-from mongoengine.signals import post_save, pre_save
+from mongoengine.signals import post_save, pre_init, pre_save
 from stringdist import rdlevenshtein
 from werkzeug.utils import cached_property
 
@@ -617,6 +618,31 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
 
     verbose_name = _("dataset")
 
+    missing_resources = False
+
+    @cached_property
+    def resources_len(self):
+        # :ResourcesLengthProperty
+        # If we've excluded the resources from the Mongo request we need
+        # to do a new Mongo request to fetch the resources length manually.
+        # If the resources are already present, just return the `len()` of the array.
+        if not self.missing_resources:
+            return len(self.resources)
+
+        pipeline = [{"$project": {"_id": 1, "resources_len": {"$size": "$resources"}}}]
+        data = Dataset.objects(id=self.id).aggregate(pipeline)
+
+        return next(data)["resources_len"]
+
+    @classmethod
+    def pre_init(cls, sender, document: Self, values, **kwargs):
+        # MongoEngine loses the information about raw values during the __init__ function
+        # Here we catch the raw values from the database (or from the creation of the object)
+        # and we check if resources were returned (sometimes we exclude `resources` from the
+        # Mongo request to improve perfs)
+        # This is used in :ResourcesLengthProperty
+        document.missing_resources = "resources" not in values
+
     @classmethod
     def pre_save(cls, sender, document, **kwargs):
         cls.before_save.send(document)
@@ -941,9 +967,17 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
             "url": endpoint_for("datasets.show", "api.dataset", dataset=self, _external=True),
             "name": self.title,
             "keywords": ",".join(self.tags),
-            "distribution": [resource.json_ld for resource in self.resources],
+            "distribution": [
+                resource.json_ld
+                for resource in self.resources[: current_app.config["MAX_RESOURCES_IN_JSON_LD"]]
+            ],
             # Theses values are not standard
-            "contributedDistribution": [resource.json_ld for resource in self.community_resources],
+            "contributedDistribution": [
+                resource.json_ld
+                for resource in self.community_resources[
+                    : current_app.config["MAX_RESOURCES_IN_JSON_LD"]
+                ]
+            ],
             "extras": [get_json_ld_extra(*item) for item in self.extras.items()],
         }
 
@@ -995,6 +1029,7 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
         self.save()
 
 
+pre_init.connect(Dataset.pre_init, sender=Dataset)
 pre_save.connect(Dataset.pre_save, sender=Dataset)
 post_save.connect(Dataset.post_save, sender=Dataset)
 

udata/routing.py

@@ -77,6 +77,9 @@ class ModelConverter(BaseConverter):
     def has_redirected_slug(self):
         return self.has_slug and self.model.slug.follow
 
+    def get_excludes(self):
+        return []
+
     def quote(self, value):
         if self.has_slug:
             return self.model.slug.slugify(value)
@@ -85,13 +88,13 @@ class ModelConverter(BaseConverter):
 
     def to_python(self, value):
         try:
-            return self.model.objects.get_or_404(id=value)
+            return self.model.objects.exclude(*self.get_excludes()).get_or_404(id=value)
         except (NotFound, ValidationError):
             pass
         try:
             quoted = self.quote(value)
             query = db.Q(slug=value) | db.Q(slug=quoted)
-            obj = self.model.objects(query).get()
+            obj = self.model.objects(query).exclude(*self.get_excludes()).get()
         except (InvalidQueryError, self.model.DoesNotExist):
             # If the model doesn't have a slug or matching slug doesn't exist.
             if self.has_redirected_slug:
@@ -121,6 +124,13 @@ class DatasetConverter(ModelConverter):
     model = models.Dataset
 
 
+class DatasetWithoutResourcesConverter(ModelConverter):
+    model = models.Dataset
+
+    def get_excludes(self):
+        return ["resources"]
+
+
 class DataserviceConverter(ModelConverter):
     model = Dataservice
 
@@ -226,6 +236,7 @@ def init_app(app):
     app.url_map.converters["pathlist"] = PathListConverter
     app.url_map.converters["uuid"] = UUIDConverter
     app.url_map.converters["dataset"] = DatasetConverter
+    app.url_map.converters["dataset_without_resources"] = DatasetWithoutResourcesConverter
     app.url_map.converters["dataservice"] = DataserviceConverter
     app.url_map.converters["crid"] = CommunityResourceConverter
     app.url_map.converters["org"] = OrganizationConverter

udata/settings.py

@@ -570,6 +570,10 @@ class Defaults(object):
     ###########################################################################
     TABULAR_API_DATASERVICE_ID = None
 
+    # JSON-LD settings
+    ###########################################################################
+    MAX_RESOURCES_IN_JSON_LD = 20
+
 
 class Testing(object):
     """Sane values for testing. Should be applied as override"""