udata 10.1.3.dev34359__py2.py3-none-any.whl → 10.1.4__py2.py3-none-any.whl

udata/__init__.py

@@ -4,5 +4,5 @@
 udata
 """
 
-__version__ = "10.1.3.dev"
+__version__ = "10.1.4"
 __description__ = "Open data portal"

udata/core/dataset/api.py

@@ -402,7 +402,21 @@ class ResourcesAPI(API):
     def put(self, dataset):
         """Reorder resources"""
         ResourceEditPermission(dataset).test()
-        data = {"resources": request.json}
+        resources = request.json
+        if len(dataset.resources) != len(resources):
+            api.abort(
+                400,
+                f"All resources must be reordered, you provided {len(resources)} "
+                f"out of {len(dataset.resources)}",
+            )
+        if set(r["id"] if isinstance(r, dict) else r for r in resources) != set(
+            str(r.id) for r in dataset.resources
+        ):
+            api.abort(
+                400,
+                f"Resource ids must match existing ones in dataset, ie: {set(str(r.id) for r in dataset.resources)}",
+            )
+        data = {"resources": resources}
         form = ResourcesListForm.from_json(
             data, obj=dataset, instance=dataset, meta={"csrf": False}
         )

udata/core/dataset/apiv2.py

@@ -2,6 +2,7 @@ import logging
 
 import mongoengine
 from flask import abort, request, url_for
+from flask_login import current_user
 from flask_restx import marshal
 
 from udata import search
@@ -11,7 +12,7 @@ from udata.core.organization.api_fields import member_user_with_email_fields
 from udata.core.spatial.api_fields import geojson
 from udata.utils import get_by
 
-from .api import ResourceMixin
+from .api import DEFAULT_SORTING, DatasetApiParser, ResourceMixin
 from .api_fields import (
     badge_fields,
     catalog_schema_fields,
@@ -128,7 +129,7 @@ dataset_fields = apiv2.model(
                     _external=True,
                 ),
                 "type": "GET",
-                "total": len(o.resources),
+                "total": o.resources_len,  # :ResourcesLengthProperty may call MongoDB to fetch the length if resources were not fetched
             },
             description="Link to the dataset resources",
         ),
@@ -155,7 +156,7 @@ dataset_fields = apiv2.model(
         ),
         "frequency_date": fields.ISODateTime(
             description=(
-                "Next expected update date, you will be notified " "once that date is reached."
+                "Next expected update date, you will be notified once that date is reached."
             )
         ),
         "harvest": fields.Nested(
@@ -276,7 +277,28 @@ class DatasetSearchAPI(API):
             abort(500, "Internal search service error")
 
 
-@ns.route("/<dataset:dataset>/", endpoint="dataset", doc=common_doc)
+dataset_parser = DatasetApiParser()
+
+
+@ns.route("/", endpoint="datasets")
+class DatasetListAPI(API):
+    """Datasets collection endpoint"""
+
+    @apiv2.doc("list_datasets")
+    @apiv2.expect(dataset_parser.parser)
+    @apiv2.marshal_with(dataset_page_fields)
+    def get(self):
+        """List or search all datasets"""
+        args = dataset_parser.parse()
+        datasets = Dataset.objects.exclude("resources").visible_by_user(
+            current_user, mongoengine.Q(private__ne=True, archived=None, deleted=None)
+        )
+        datasets = dataset_parser.parse_filters(datasets, args)
+        sort = args["sort"] or ("$text_score" if args["q"] else None) or DEFAULT_SORTING
+        return datasets.order_by(sort).paginate(args["page"], args["page_size"])
+
+
+@ns.route("/<dataset_without_resources:dataset>/", endpoint="dataset", doc=common_doc)
 @apiv2.response(404, "Dataset not found")
 @apiv2.response(410, "Dataset has been deleted")
 class DatasetAPI(API):

udata/core/dataset/models.py

@@ -2,6 +2,7 @@ import logging
 import re
 from datetime import datetime, timedelta
 from pydoc import locate
+from typing import Self
 from urllib.parse import urlparse
 
 import requests
@@ -11,7 +12,7 @@ from flask import current_app
 from mongoengine import DynamicEmbeddedDocument
 from mongoengine import ValidationError as MongoEngineValidationError
 from mongoengine.fields import DateTimeField
-from mongoengine.signals import post_save, pre_save
+from mongoengine.signals import post_save, pre_init, pre_save
 from stringdist import rdlevenshtein
 from werkzeug.utils import cached_property
 
@@ -617,6 +618,33 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
 
     verbose_name = _("dataset")
 
+    missing_resources = False
+
+    @cached_property
+    def resources_len(self):
+        # :ResourcesLengthProperty
+        # If we've excluded the resources from the Mongo request we need
+        # to do a new Mongo request to fetch the resources length manually.
+        # If the resources are already present, just return the `len()` of the array.
+        if not self.missing_resources:
+            return len(self.resources)
+
+        pipeline = [
+            {"$project": {"_id": 1, "resources_len": {"$size": {"$ifNull": ["$resources", []]}}}}
+        ]
+        data = Dataset.objects(id=self.id).aggregate(pipeline)
+
+        return next(data)["resources_len"]
+
+    @classmethod
+    def pre_init(cls, sender, document: Self, values, **kwargs):
+        # MongoEngine loses the information about raw values during the __init__ function
+        # Here we catch the raw values from the database (or from the creation of the object)
+        # and we check if resources were returned (sometimes we exclude `resources` from the
+        # Mongo request to improve perfs)
+        # This is used in :ResourcesLengthProperty
+        document.missing_resources = "resources" not in values
+
     @classmethod
     def pre_save(cls, sender, document, **kwargs):
         cls.before_save.send(document)
@@ -941,9 +969,17 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
             "url": endpoint_for("datasets.show", "api.dataset", dataset=self, _external=True),
             "name": self.title,
             "keywords": ",".join(self.tags),
-            "distribution": [resource.json_ld for resource in self.resources],
+            "distribution": [
+                resource.json_ld
+                for resource in self.resources[: current_app.config["MAX_RESOURCES_IN_JSON_LD"]]
+            ],
             # Theses values are not standard
-            "contributedDistribution": [resource.json_ld for resource in self.community_resources],
+            "contributedDistribution": [
+                resource.json_ld
+                for resource in self.community_resources[
+                    : current_app.config["MAX_RESOURCES_IN_JSON_LD"]
+                ]
+            ],
             "extras": [get_json_ld_extra(*item) for item in self.extras.items()],
         }
 
@@ -995,6 +1031,7 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
         self.save()
 
 
+pre_init.connect(Dataset.pre_init, sender=Dataset)
 pre_save.connect(Dataset.pre_save, sender=Dataset)
 post_save.connect(Dataset.post_save, sender=Dataset)
 

udata/core/user/models.py

@@ -83,7 +83,7 @@ class User(WithMetrics, UserMixin, db.Document):
     extras = db.ExtrasField()
 
     # Used to track notification for automatic inactive users deletion
-    # when YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION is set
+    # when YEARS_OF_INACTIVITY_BEFORE_DELETION is set
     inactive_deletion_notified_at = db.DateTimeField()
 
     before_save = Signal()

udata/core/user/tasks.py

@@ -21,14 +21,14 @@ def send_test_mail(email):
 
 @job("notify-inactive-users")
 def notify_inactive_users(self):
-    if not current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"]:
+    if not current_app.config["YEARS_OF_INACTIVITY_BEFORE_DELETION"]:
         logging.warning(
-            "YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION setting is not set, no deletion planned"
+            "YEARS_OF_INACTIVITY_BEFORE_DELETION setting is not set, no deletion planned"
         )
         return
     notification_comparison_date = (
         datetime.utcnow()
-        - timedelta(days=current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"] * 365)
+        - timedelta(days=current_app.config["YEARS_OF_INACTIVITY_BEFORE_DELETION"] * 365)
         + timedelta(days=current_app.config["DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY"])
     )
 
@@ -56,9 +56,9 @@ def notify_inactive_users(self):
 
 @job("delete-inactive-users")
 def delete_inactive_users(self):
-    if not current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"]:
+    if not current_app.config["YEARS_OF_INACTIVITY_BEFORE_DELETION"]:
         logging.warning(
-            "YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION setting is not set, no deletion planned"
+            "YEARS_OF_INACTIVITY_BEFORE_DELETION setting is not set, no deletion planned"
         )
         return
 
@@ -70,7 +70,7 @@ def delete_inactive_users(self):
 
     # Delete inactive users upon notification delay if user still hasn't logged in
     deletion_comparison_date = datetime.utcnow() - timedelta(
-        days=current_app.config["YEARS_OF_INACTIVITY_BEFORE_DEACTIVATION"] * 365
+        days=current_app.config["YEARS_OF_INACTIVITY_BEFORE_DELETION"] * 365
     )
     notified_at = datetime.utcnow() - timedelta(
         days=current_app.config["DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY"]

udata/harvest/tests/dcat/udata.xml

@@ -0,0 +1,180 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rdf:RDF
+   xmlns:adms="http://www.w3.org/ns/adms#"
+   xmlns:dcat="http://www.w3.org/ns/dcat#"
+   xmlns:dct="http://purl.org/dc/terms/"
+   xmlns:foaf="http://xmlns.com/foaf/0.1/"
+   xmlns:hydra="http://www.w3.org/ns/hydra/core#"
+   xmlns:ns1="http://data.europa.eu/930/"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
+   xmlns:skos="http://www.w3.org/2004/02/skos/core#"
+   xmlns:vcard="http://www.w3.org/2006/vcard/ns#"
+>
+  <rdf:Description rdf:about="https://vanves-seineouest.opendatasoft.com/explore/dataset/bureau-de-vote-vanves/">
+    <dct:identifier>https://vanves-seineouest.opendatasoft.com/explore/dataset/bureau-de-vote-vanves/</dct:identifier>
+    <adms:identifier rdf:nodeID="N7cbef9d8b574438f86f859e81e04e4c0"/>
+    <rdf:type rdf:resource="http://www.w3.org/ns/dcat#Dataset"/>
+    <dct:title>Bureaux de vote - Vanves</dct:title>
+    <dct:description>La liste des 23 bureaux de vote à Vanves</dct:description>
+    <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:issued>
+    <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:modified>
+    <dcat:landingPage rdf:resource="https://vanves-seineouest.opendatasoft.com/explore/dataset/bureau-de-vote-vanves/"/>
+    <dcat:keyword>administration-finances-publiques</dcat:keyword>
+    <dcat:keyword>bureaux-de-vote</dcat:keyword>
+    <dcat:keyword>elections</dcat:keyword>
+    <dcat:distribution rdf:resource="https://www.data.gouv.fr/datasets/61892c9d076a4d62434a1318/#resource-5dd4e0b2-4d96-4f36-b73e-b78ec993703c"/>
+    <dcat:distribution rdf:resource="https://www.data.gouv.fr/datasets/61892c9d076a4d62434a1318/#resource-0f80d285-72f8-49f8-a691-1dad6bd2f6db"/>
+    <dcat:distribution rdf:resource="https://www.data.gouv.fr/datasets/61892c9d076a4d62434a1318/#resource-d78d1245-6e8e-44d8-88cd-87b1dd66034f"/>
+    <dcat:distribution rdf:resource="https://www.data.gouv.fr/datasets/61892c9d076a4d62434a1318/#resource-b18fb2bd-6c8b-47a8-84fb-cdc8e29f4f1a"/>
+    <ns1:distributor rdf:resource="https://www.data.gouv.fr/organizations/54884a24c751df7226a3fc16/"/>
+    <dct:publisher rdf:resource="https://www.data.gouv.fr/api/1/contacts/67b7c3b0e2714e0e9de260a3/"/>
+    <dct:creator rdf:resource="https://www.data.gouv.fr/api/1/contacts/67b7c3b0e2714e0e9de260a4/"/>
+  </rdf:Description>
+  <rdf:Description rdf:about="https://www.data.gouv.fr/datasets/61892c9d076a4d62434a1318/#resource-5dd4e0b2-4d96-4f36-b73e-b78ec993703c">
+    <rdf:type rdf:resource="http://www.w3.org/ns/dcat#Distribution"/>
+    <dct:identifier>5dd4e0b2-4d96-4f36-b73e-b78ec993703c</dct:identifier>
+    <dct:title>bureau-de-vote-vanves.csv</dct:title>
+    <dct:description>Bureaux de vote - Vanves (csv)</dct:description>
+    <dcat:downloadURL rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/catalog/datasets/bureau-de-vote-vanves/exports/csv?use_labels=true"/>
+    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/fr/datasets/r/5dd4e0b2-4d96-4f36-b73e-b78ec993703c"/>
+    <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:issued>
+    <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:modified>
+    <dct:rights>License Not Specified</dct:rights>
+    <dcat:mediaType>text/csv</dcat:mediaType>
+    <dct:format>csv</dct:format>
+  </rdf:Description>
+  <rdf:Description rdf:about="https://www.data.gouv.fr/api/1/organizations/54884a24c751df7226a3fc16/catalog">
+    <rdf:type rdf:resource="http://www.w3.org/ns/dcat#Catalog"/>
+    <rdf:type rdf:resource="http://www.w3.org/ns/hydra/core#Collection"/>
+    <dct:publisher rdf:resource="https://www.data.gouv.fr/organizations/54884a24c751df7226a3fc16/"/>
+    <dct:title>Ville de Vanves</dct:title>
+    <dct:description>Ville de Vanves</dct:description>
+    <dcat:dataset rdf:resource="https://vanves-seineouest.opendatasoft.com/explore/dataset/vfe_public_219200755_20240506/"/>
+    <dcat:dataset rdf:resource="https://vanves-seineouest.opendatasoft.com/explore/dataset/bureau-de-vote-vanves/"/>
+    <dcat:service rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/"/>
+    <hydra:totalItems rdf:datatype="http://www.w3.org/2001/XMLSchema#integer">2</hydra:totalItems>
+    <hydra:view rdf:resource="https://www.data.gouv.fr/api/1/organizations/54884a24c751df7226a3fc16/catalog.rdf?page=1&amp;page_size=100"/>
+  </rdf:Description>
+  <rdf:Description rdf:about="https://www.data.gouv.fr/datasets/61892c9d076a4d62434a1318/#resource-d78d1245-6e8e-44d8-88cd-87b1dd66034f">
+    <rdf:type rdf:resource="http://www.w3.org/ns/dcat#Distribution"/>
+    <dct:identifier>d78d1245-6e8e-44d8-88cd-87b1dd66034f</dct:identifier>
+    <dct:title>bureau-de-vote-vanves.geojson</dct:title>
+    <dct:description>Bureaux de vote - Vanves (geojson)</dct:description>
+    <dcat:downloadURL rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/catalog/datasets/bureau-de-vote-vanves/exports/geojson"/>
+    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/fr/datasets/r/d78d1245-6e8e-44d8-88cd-87b1dd66034f"/>
+    <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:issued>
+    <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:modified>
+    <dct:rights>License Not Specified</dct:rights>
+    <dcat:mediaType>application/json</dcat:mediaType>
+    <dct:format>json</dct:format>
+  </rdf:Description>
+  <rdf:Description rdf:about="https://www.data.gouv.fr/api/1/organizations/54884a24c751df7226a3fc16/catalog.rdf?page=1&amp;page_size=100">
+    <rdf:type rdf:resource="http://www.w3.org/ns/hydra/core#PartialCollectionView"/>
+    <hydra:first rdf:resource="https://www.data.gouv.fr/api/1/organizations/54884a24c751df7226a3fc16/catalog.rdf?page=1&amp;page_size=100"/>
+    <hydra:last rdf:resource="https://www.data.gouv.fr/api/1/organizations/54884a24c751df7226a3fc16/catalog.rdf?page=1&amp;page_size=100"/>
+  </rdf:Description>
+  <rdf:Description rdf:about="https://www.data.gouv.fr/organizations/54884a24c751df7226a3fc16/">
+    <rdf:type rdf:resource="http://xmlns.com/foaf/0.1/Organization"/>
+    <foaf:name>Ville de Vanves</foaf:name>
+    <rdfs:label>Ville de Vanves</rdfs:label>
+    <foaf:homepage rdf:resource="http://www.vanves.fr"/>
+  </rdf:Description>
+  <rdf:Description rdf:about="https://www.data.gouv.fr/datasets/66397176ff2a31840e29304c/#resource-482677b8-379c-45a5-83ef-0361fecb4cc3">
+    <rdf:type rdf:resource="http://www.w3.org/ns/dcat#Distribution"/>
+    <dct:identifier>482677b8-379c-45a5-83ef-0361fecb4cc3</dct:identifier>
+    <dct:title>vfe_public_219200755_20240506.json</dct:title>
+    <dct:description>Ville de Vanves - Part des véhicules à faibles émissions dans le renouvellement du parc (json)</dct:description>
+    <dcat:downloadURL rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/catalog/datasets/vfe_public_219200755_20240506/exports/json"/>
+    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/fr/datasets/r/482677b8-379c-45a5-83ef-0361fecb4cc3"/>
+    <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2024-05-06T15:15:18</dct:issued>
+    <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2024-05-06T15:15:18</dct:modified>
+    <dct:rights>License Not Specified</dct:rights>
+    <dcat:mediaType>application/json</dcat:mediaType>
+    <dct:format>json</dct:format>
+  </rdf:Description>
+  <rdf:Description rdf:about="https://www.data.gouv.fr/datasets/61892c9d076a4d62434a1318/#resource-b18fb2bd-6c8b-47a8-84fb-cdc8e29f4f1a">
+    <rdf:type rdf:resource="http://www.w3.org/ns/dcat#Distribution"/>
+    <dct:identifier>b18fb2bd-6c8b-47a8-84fb-cdc8e29f4f1a</dct:identifier>
+    <dct:title>bureau-de-vote-vanves.zip</dct:title>
+    <dct:description>Bureaux de vote - Vanves (shp)</dct:description>
+    <dcat:downloadURL rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/catalog/datasets/bureau-de-vote-vanves/exports/shp"/>
+    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/fr/datasets/r/b18fb2bd-6c8b-47a8-84fb-cdc8e29f4f1a"/>
+    <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:issued>
+    <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:modified>
+    <dct:rights>License Not Specified</dct:rights>
+    <dcat:mediaType>application/zip</dcat:mediaType>
+    <dct:format>zip</dct:format>
+  </rdf:Description>
+  <rdf:Description rdf:about="https://www.data.gouv.fr/datasets/66397176ff2a31840e29304c/#resource-aab4d337-a617-4c18-a045-291d68787a7d">
+    <rdf:type rdf:resource="http://www.w3.org/ns/dcat#Distribution"/>
+    <dct:identifier>aab4d337-a617-4c18-a045-291d68787a7d</dct:identifier>
+    <dct:title>vfe_public_219200755_20240506.csv</dct:title>
+    <dct:description>Ville de Vanves - Part des véhicules à faibles émissions dans le renouvellement du parc (csv)</dct:description>
+    <dcat:downloadURL rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/catalog/datasets/vfe_public_219200755_20240506/exports/csv?use_labels=true"/>
+    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/fr/datasets/r/aab4d337-a617-4c18-a045-291d68787a7d"/>
+    <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2024-05-06T15:15:18</dct:issued>
+    <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2024-05-06T15:15:18</dct:modified>
+    <dct:rights>License Not Specified</dct:rights>
+    <dcat:mediaType>text/csv</dcat:mediaType>
+    <dct:format>csv</dct:format>
+  </rdf:Description>
+  <rdf:Description rdf:about="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/">
+    <rdf:type rdf:resource="http://www.w3.org/ns/dcat#DataService"/>
+    <dct:identifier>https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/</dct:identifier>
+    <dct:title>Explore API v2 https://vanves-seineouest.opendatasoft.com</dct:title>
+    <dct:description></dct:description>
+    <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2024-07-12T00:03:38.764000</dct:issued>
+    <dcat:endpointURL rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/"/>
+    <dcat:landingPage rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/console"/>
+    <dcat:endpointDescription rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/swagger.json"/>
+    <dcat:servesDataset rdf:resource="https://vanves-seineouest.opendatasoft.com/explore/dataset/vfe_public_219200755_20240506/"/>
+    <dcat:servesDataset rdf:resource="https://vanves-seineouest.opendatasoft.com/explore/dataset/bureau-de-vote-vanves/"/>
+  </rdf:Description>
+  <rdf:Description rdf:nodeID="N25911b8ad0604e9f8de4d7e1965eb81a">
+    <rdf:type rdf:resource="http://www.w3.org/ns/adms#Identifier"/>
+    <dct:creator>data.gouv.fr</dct:creator>
+    <skos:notation>https://www.data.gouv.fr/datasets/66397176ff2a31840e29304c/</skos:notation>
+  </rdf:Description>
+  <rdf:Description rdf:about="https://vanves-seineouest.opendatasoft.com/explore/dataset/vfe_public_219200755_20240506/">
+    <dct:identifier>https://vanves-seineouest.opendatasoft.com/explore/dataset/vfe_public_219200755_20240506/</dct:identifier>
+    <adms:identifier rdf:nodeID="N25911b8ad0604e9f8de4d7e1965eb81a"/>
+    <rdf:type rdf:resource="http://www.w3.org/ns/dcat#Dataset"/>
+    <dct:title>Ville de Vanves - Part des véhicules à faibles émissions dans le renouvellement du parc</dct:title>
+    <dct:description>Ville de Vanves - Part des véhicules à faibles émissions dans le renouvellement du parc</dct:description>
+    <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2024-05-06T15:15:18</dct:issued>
+    <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2024-05-06T15:15:18</dct:modified>
+    <dcat:landingPage rdf:resource="https://vanves-seineouest.opendatasoft.com/explore/dataset/vfe_public_219200755_20240506/"/>
+    <dcat:distribution rdf:resource="https://www.data.gouv.fr/datasets/66397176ff2a31840e29304c/#resource-aab4d337-a617-4c18-a045-291d68787a7d"/>
+    <dcat:distribution rdf:resource="https://www.data.gouv.fr/datasets/66397176ff2a31840e29304c/#resource-482677b8-379c-45a5-83ef-0361fecb4cc3"/>
+    <ns1:distributor rdf:resource="https://www.data.gouv.fr/organizations/54884a24c751df7226a3fc16/"/>
+    <dct:publisher rdf:resource="https://www.data.gouv.fr/api/1/contacts/67b7c3b0e2714e0e9de260a3/"/>
+    <dct:creator rdf:resource="https://www.data.gouv.fr/api/1/contacts/67b7c3b0e2714e0e9de260a4/"/>
+  </rdf:Description>
+  <rdf:Description rdf:about="https://www.data.gouv.fr/api/1/contacts/67b7c3b0e2714e0e9de260a4/">
+    <rdf:type rdf:resource="http://www.w3.org/2006/vcard/ns#Kind"/>
+    <vcard:fn>Vanves</vcard:fn>
+  </rdf:Description>
+  <rdf:Description rdf:about="https://www.data.gouv.fr/datasets/61892c9d076a4d62434a1318/#resource-0f80d285-72f8-49f8-a691-1dad6bd2f6db">
+    <rdf:type rdf:resource="http://www.w3.org/ns/dcat#Distribution"/>
+    <dct:identifier>0f80d285-72f8-49f8-a691-1dad6bd2f6db</dct:identifier>
+    <dct:title>bureau-de-vote-vanves.json</dct:title>
+    <dct:description>Bureaux de vote - Vanves (json)</dct:description>
+    <dcat:downloadURL rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/catalog/datasets/bureau-de-vote-vanves/exports/json"/>
+    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/fr/datasets/r/0f80d285-72f8-49f8-a691-1dad6bd2f6db"/>
+    <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:issued>
+    <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:modified>
+    <dct:rights>License Not Specified</dct:rights>
+    <dcat:mediaType>application/json</dcat:mediaType>
+    <dct:format>json</dct:format>
+  </rdf:Description>
+  <rdf:Description rdf:about="https://www.data.gouv.fr/api/1/contacts/67b7c3b0e2714e0e9de260a3/">
+    <rdf:type rdf:resource="http://www.w3.org/2006/vcard/ns#Kind"/>
+    <vcard:fn>Vanves</vcard:fn>
+  </rdf:Description>
+  <rdf:Description rdf:nodeID="N7cbef9d8b574438f86f859e81e04e4c0">
+    <rdf:type rdf:resource="http://www.w3.org/ns/adms#Identifier"/>
+    <dct:creator>data.gouv.fr</dct:creator>
+    <skos:notation>https://www.data.gouv.fr/datasets/61892c9d076a4d62434a1318/</skos:notation>
+  </rdf:Description>
+</rdf:RDF>

udata/harvest/tests/test_dcat_backend.py

@@ -624,6 +624,71 @@ class DcatBackendTest:
         )  # noqa
         assert dataset.harvest.last_update.date() == date.today()
 
+    def test_udata_xml_catalog(self, rmock):
+        LicenseFactory(id="fr-lo", title="Licence ouverte / Open Licence")
+        url = mock_dcat(rmock, "udata.xml")
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
+        actions.run(source.slug)
+
+        source.reload()
+        job = source.get_last_job()
+        assert len(job.items) == 3
+
+        assert Dataset.objects.filter(organization=org).count() == 2
+        dataset = Dataset.objects.filter(organization=org, title="Bureaux de vote - Vanves").first()
+
+        assert dataset is not None
+        assert "bureaux-de-vote" in dataset.tags  # support dcat:keyword
+        assert len(dataset.resources) == 4
+        assert dataset.description == "La liste des 23 bureaux de vote à Vanves"
+        assert dataset.harvest is not None
+        assert (
+            dataset.harvest.dct_identifier
+            == "https://vanves-seineouest.opendatasoft.com/explore/dataset/bureau-de-vote-vanves/"
+        )
+        assert (
+            dataset.harvest.remote_id
+            == "https://vanves-seineouest.opendatasoft.com/explore/dataset/bureau-de-vote-vanves/"
+        )
+        assert dataset.harvest.created_at.isoformat() == "2019-04-19T12:21:56"
+        assert dataset.harvest.modified_at.isoformat() == "2019-04-19T12:21:56"
+        assert (
+            dataset.harvest.uri
+            == "https://vanves-seineouest.opendatasoft.com/explore/dataset/bureau-de-vote-vanves/"
+        )
+        assert (
+            dataset.harvest.remote_url
+            == "https://vanves-seineouest.opendatasoft.com/explore/dataset/bureau-de-vote-vanves/"
+        )
+        assert dataset.harvest.last_update.date() == date.today()
+
+        assert Dataservice.objects(organization=org).count() == 1
+        service = Dataservice.objects(organization=org).first()
+
+        assert service is not None
+        assert len(service.datasets) == 2
+        assert service.title == "Explore API v2 https://vanves-seineouest.opendatasoft.com"
+        assert service.description == ""
+        assert (
+            service.machine_documentation_url
+            == "https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/swagger.json"
+        )
+        assert (
+            service.base_api_url == "https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/"
+        )
+        assert service.harvest is not None
+        assert (
+            service.harvest.remote_id
+            == "https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/"
+        )
+        assert service.harvest.created_at.isoformat() == "2024-07-12T00:03:38.764000"
+        assert (
+            service.harvest.remote_url
+            == "https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/console"
+        )
+        assert service.harvest.last_update.date() == date.today()
+
     def test_user_agent_get(self, rmock):
         url = mock_dcat(rmock, "catalog.xml", path="without/extension")
         rmock.head(url, headers={"Content-Type": "application/xml; charset=utf-8"})

udata/rdf.py

@@ -312,7 +312,12 @@ def themes_from_rdf(rdf):
 def contact_points_from_rdf(rdf, prop, role, dataset):
     for contact_point in rdf.objects(prop):
         # Read contact point information
-        if prop == DCAT.contactPoint:  # Could be split on the type of contact_point instead
+        if isinstance(contact_point, Literal):
+            log.warning(f"Found a `Literal` inside {prop}, `foaf:Agent` or `vcard:Kind` expected.")
+            name = contact_point.toPython()
+            email = None
+            contact_form = None
+        elif prop == DCAT.contactPoint:  # Could be split on the type of contact_point instead
             name = rdf_value(contact_point, VCARD.fn) or ""
             email = (
                 rdf_value(contact_point, VCARD.hasEmail)

udata/routing.py

@@ -77,6 +77,9 @@ class ModelConverter(BaseConverter):
     def has_redirected_slug(self):
         return self.has_slug and self.model.slug.follow
 
+    def get_excludes(self):
+        return []
+
     def quote(self, value):
         if self.has_slug:
             return self.model.slug.slugify(value)
@@ -85,13 +88,13 @@ class ModelConverter(BaseConverter):
 
     def to_python(self, value):
         try:
-            return self.model.objects.get_or_404(id=value)
+            return self.model.objects.exclude(*self.get_excludes()).get_or_404(id=value)
         except (NotFound, ValidationError):
             pass
         try:
             quoted = self.quote(value)
             query = db.Q(slug=value) | db.Q(slug=quoted)
-            obj = self.model.objects(query).get()
+            obj = self.model.objects(query).exclude(*self.get_excludes()).get()
         except (InvalidQueryError, self.model.DoesNotExist):
             # If the model doesn't have a slug or matching slug doesn't exist.
             if self.has_redirected_slug:
@@ -121,6 +124,13 @@ class DatasetConverter(ModelConverter):
     model = models.Dataset
 
 
+class DatasetWithoutResourcesConverter(ModelConverter):
+    model = models.Dataset
+
+    def get_excludes(self):
+        return ["resources"]
+
+
 class DataserviceConverter(ModelConverter):
     model = Dataservice
 
@@ -226,6 +236,7 @@ def init_app(app):
     app.url_map.converters["pathlist"] = PathListConverter
     app.url_map.converters["uuid"] = UUIDConverter
     app.url_map.converters["dataset"] = DatasetConverter
+    app.url_map.converters["dataset_without_resources"] = DatasetWithoutResourcesConverter
     app.url_map.converters["dataservice"] = DataserviceConverter
     app.url_map.converters["crid"] = CommunityResourceConverter
     app.url_map.converters["org"] = OrganizationConverter

udata/settings.py

@@ -570,6 +570,10 @@ class Defaults(object):
     ###########################################################################
     TABULAR_API_DATASERVICE_ID = None
 
+    # JSON-LD settings
+    ###########################################################################
+    MAX_RESOURCES_IN_JSON_LD = 20
+
 
 class Testing(object):
     """Sane values for testing. Should be applied as override"""