udata 9.2.4.dev32010__py2.py3-none-any.whl → 9.2.4.dev32029__py2.py3-none-any.whl

udata/api_fields.py

@@ -76,7 +76,25 @@ def convert_db_to_field(key, field, info):
             constructor = custom_restx_fields.ImageField
 
     elif isinstance(field, mongo_fields.ListField):
-        # For lists, we convert the inner value from Mongo to RestX then we create
+        # For lists, we can expose them only by showing a link to the API
+        # with the results of the list to avoid listing a lot of sub-ressources
+        # (for example for a dataservices with thousands of datasets).
+        href = info.get("href", None)
+        if href:
+
+            def constructor_read(**kwargs):
+                return restx_fields.Raw(
+                    attribute=lambda o: {
+                        "rel": "subsection",
+                        "href": href(o),
+                        "type": "GET",
+                        "total": len(o[key]),
+                    },
+                    description="Visit this API link to see the list.",
+                    **kwargs,
+                )
+
+        # If it's a standard list, we convert the inner value from Mongo to RestX then we create
         # the `List` RestX type with this converted inner value.
         # There is three level of information, from most important to least
         #     1. `inner_field_info` inside `__additional_field_info__` on the parent
@@ -87,9 +105,13 @@ def convert_db_to_field(key, field, info):
             f"{key}.inner", field.field, {**info, **inner_info, **info.get("inner_field_info", {})}
         )
 
-        def constructor_read(**kwargs):
-            return restx_fields.List(field_read, **kwargs)
+        if constructor_read is None:
+            # We don't want to set the `constructor_read` if it's already set
+            # by the `href` code above.
+            def constructor_read(**kwargs):
+                return restx_fields.List(field_read, **kwargs)
 
+        # But we want to keep the `constructor_write` to allow changing the list.
         def constructor_write(**kwargs):
             return restx_fields.List(field_write, **kwargs)
     elif isinstance(
@@ -98,7 +120,7 @@ def convert_db_to_field(key, field, info):
 
         def constructor(**kwargs):
             return restx_fields.Nested(lazy_reference, **kwargs)
-    elif isinstance(field, mongo_fields.ReferenceField):
+    elif isinstance(field, (mongo_fields.ReferenceField, mongo_fields.LazyReferenceField)):
         # For reference we accept while writing a String representing the ID of the referenced model.
         # For reading, if the user supplied a `nested_fields` (RestX model), we use it to convert
         # the referenced model, if not we return a String (and RestX will call the `str()` of the model
@@ -199,9 +221,14 @@ def generate_fields(**kwargs):
 
                 if "constraints" not in filterable:
                     filterable["constraints"] = []
-                    if isinstance(field, mongo_fields.ReferenceField) or (
+                    if isinstance(
+                        field, (mongo_fields.ReferenceField, mongo_fields.LazyReferenceField)
+                    ) or (
                         isinstance(field, mongo_fields.ListField)
-                        and isinstance(field.field, mongo_fields.ReferenceField)
+                        and isinstance(
+                            field.field,
+                            (mongo_fields.ReferenceField, mongo_fields.LazyReferenceField),
+                        )
                     ):
                         filterable["constraints"].append("objectid")
 
@@ -389,11 +416,14 @@ def patch(obj, request):
             if hasattr(model_attribute, "from_input"):
                 value = model_attribute.from_input(value)
             elif isinstance(model_attribute, mongoengine.fields.ListField) and isinstance(
-                model_attribute.field, mongoengine.fields.ReferenceField
+                model_attribute.field,
+                (mongo_fields.ReferenceField, mongo_fields.LazyReferenceField),
             ):
                 # TODO `wrap_primary_key` do Mongo request, do a first pass to fetch all documents before calling it (to avoid multiple queries).
                 value = [wrap_primary_key(key, model_attribute.field, id) for id in value]
-            elif isinstance(model_attribute, mongoengine.fields.ReferenceField):
+            elif isinstance(
+                model_attribute, (mongo_fields.ReferenceField, mongo_fields.LazyReferenceField)
+            ):
                 value = wrap_primary_key(key, model_attribute, value)
             elif isinstance(
                 model_attribute,

udata/core/dataservices/api.py

@@ -1,11 +1,13 @@
 from datetime import datetime
 
 import mongoengine
+from bson import ObjectId
 from flask import make_response, redirect, request, url_for
 from flask_login import current_user
 
-from udata.api import API, api
+from udata.api import API, api, fields
 from udata.api_fields import patch
+from udata.core.dataset.models import Dataset
 from udata.core.dataset.permissions import OwnablePermission
 from udata.core.followers.api import FollowAPI
 from udata.rdf import RDF_EXTENSIONS, graph_response, negociate_content
@@ -95,6 +97,76 @@ class DataserviceAPI(API):
         return "", 204
 
 
+dataservice_add_datasets_fields = api.model(
+    "DataserviceDatasetsAdd",
+    {
+        "id": fields.String(description="Id of the dataset to add", required=True),
+    },
+    location="json",
+)
+
+
+@ns.route("/<dataservice:dataservice>/datasets/", endpoint="dataservice_datasets", doc=common_doc)
+class DataserviceDatasetsAPI(API):
+    @api.secure
+    @api.doc("dataservice_datasets_create")
+    @api.expect([dataservice_add_datasets_fields])
+    @api.marshal_with(Dataservice.__read_fields__)
+    @api.response(400, "Malformed object id(s) in request")
+    @api.response(400, "Expecting a list")
+    @api.response(400, "Expecting a list of dicts with id attribute")
+    @api.response(404, "Dataservice not found")
+    @api.response(403, "Forbidden")
+    @api.response(410, "Dataservice has been deleted")
+    def post(self, dataservice):
+        if dataservice.deleted_at:
+            api.abort(410, "Dataservice has been deleted")
+
+        OwnablePermission(dataservice).test()
+
+        data = request.json
+
+        if not isinstance(data, list):
+            api.abort(400, "Expecting a list")
+        if not all(isinstance(d, dict) and d.get("id") for d in data):
+            api.abort(400, "Expecting a list of dicts with id attribute")
+
+        try:
+            datasets = Dataset.objects.filter(id__in=[d["id"] for d in data]).only("id")
+            diff = set(d.id for d in datasets) - set(d.id for d in dataservice.datasets)
+        except mongoengine.errors.ValidationError:
+            api.abort(400, "Malformed object id(s) in request")
+
+        if diff:
+            dataservice.datasets += [ObjectId(did) for did in diff]
+            dataservice.save()
+
+        return dataservice, 201
+
+
+@ns.route(
+    "/<dataservice:dataservice>/datasets/<dataset:dataset>/",
+    endpoint="dataservice_dataset",
+    doc=common_doc,
+)
+class DataserviceDatasetAPI(API):
+    @api.secure
+    @api.response(404, "Dataservice not found")
+    @api.response(404, "Dataset not found in dataservice")
+    def delete(self, dataservice, dataset):
+        if dataservice.deleted_at:
+            api.abort(410, "Dataservice has been deleted")
+
+        OwnablePermission(dataservice).test()
+
+        if dataset not in dataservice.datasets:
+            api.abort(404, "Dataset not found in dataservice")
+        dataservice.datasets = [d for d in dataservice.datasets if d.id != dataset.id]
+        dataservice.save()
+
+        return None, 204
+
+
 @ns.route("/<dataservice:dataservice>/rdf", endpoint="dataservice_rdf", doc=common_doc)
 @api.response(404, "Dataservice not found")
 @api.response(410, "Dataservice has been deleted")

udata/core/dataservices/models.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 
+from flask import url_for
 from mongoengine import Q
 
 import udata.core.contact_point.api_fields as contact_api_fields
@@ -171,13 +172,16 @@ class Dataservice(WithMetrics, Owned, db.Document):
     datasets = field(
         db.ListField(
             field(
-                db.ReferenceField(Dataset),
+                db.LazyReferenceField(Dataset, passthrough=True),
                 nested_fields=datasets_api_fields.dataset_ref_fields,
             )
         ),
         filterable={
             "key": "dataset",
         },
+        href=lambda dataservice: url_for(
+            "api.datasets", dataservice=dataservice.id, _external=True
+        ),
     )
 
     harvest = field(

udata/core/dataset/api.py

@@ -33,6 +33,7 @@ from udata.auth import admin_permission
 from udata.core import storages
 from udata.core.badges import api as badges_api
 from udata.core.badges.fields import badge_fields
+from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.models import CHECKSUM_TYPES
 from udata.core.followers.api import FollowAPI
 from udata.core.storages.api import handle_upload, upload_parser
@@ -100,6 +101,7 @@ class DatasetApiParser(ModelApiParser):
         self.parser.add_argument("schema", type=str, location="args")
         self.parser.add_argument("schema_version", type=str, location="args")
         self.parser.add_argument("topic", type=str, location="args")
+        self.parser.add_argument("dataservice", type=str, location="args")
 
     @staticmethod
     def parse_filters(datasets, args):
@@ -148,6 +150,15 @@ class DatasetApiParser(ModelApiParser):
                 pass
             else:
                 datasets = datasets.filter(id__in=[d.id for d in topic.datasets])
+        if args.get("dataservice"):
+            if not ObjectId.is_valid(args["dataservice"]):
+                api.abort(400, "Dataservice arg must be an identifier")
+            try:
+                dataservice = Dataservice.objects.get(id=args["dataservice"])
+            except Dataservice.DoesNotExist:
+                pass
+            else:
+                datasets = datasets.filter(id__in=[d.id for d in dataservice.datasets])
         return datasets
 
 

udata/core/dataset/tasks.py

@@ -5,10 +5,12 @@ from tempfile import NamedTemporaryFile
 
 from celery.utils.log import get_task_logger
 from flask import current_app
+from mongoengine import ValidationError
 
 from udata import mail
 from udata import models as udata_models
 from udata.core import storages
+from udata.core.dataservices.models import Dataservice
 from udata.frontend import csv
 from udata.harvest.models import HarvestJob
 from udata.i18n import lazy_gettext as _
@@ -231,3 +233,36 @@ def export_csv(self, model=None):
     models = (model,) if model else ALLOWED_MODELS
     for model in models:
         export_csv_for_model(model, dataset)
+
+
+@job("bind-tabular-dataservice")
+def bind_tabular_dataservice(self):
+    """
+    Bind the datasets served by TabularAPI to its dataservice objects
+    """
+    TABULAR_API_DATASERVICE_ID = current_app.config.get("TABULAR_API_DATASERVICE_ID", [])
+
+    if not TABULAR_API_DATASERVICE_ID:
+        log.error("TABULAR_API_DATASERVICE_ID setting value not set")
+        return
+    try:
+        dataservice = Dataservice.objects.get(id=TABULAR_API_DATASERVICE_ID)
+    except Dataservice.DoesNotExist:
+        log.error("TABULAR_API_DATASERVICE_ID points to a non existent dataservice")
+        return
+
+    datasets = Dataset.objects(
+        **{
+            "resources__extras__analysis:parsing:finished_at__exists": True,
+            "resources__extras__analysis:parsing:error": None,
+        }
+    ).visible()
+
+    dataservice.datasets = datasets
+
+    try:
+        dataservice.save()
+    except ValidationError as e:
+        log.error(exc_info=e)
+
+    log.info(f"Bound {datasets.count()} datasets to TabularAPI dataservice")

udata/settings.py

@@ -538,6 +538,10 @@ class Defaults(object):
     ###########################################################################
     MATTERMOST_WEBHOOK = None
 
+    # Tabular API Dataservice ID
+    ###########################################################################
+    TABULAR_API_DATASERVICE_ID = None
+
 
 class Testing(object):
     """Sane values for testing. Should be applied as override"""