udata 10.2.1.dev34728__py2.py3-none-any.whl → 10.2.1.dev34761__py2.py3-none-any.whl

udata/core/dataset/models.py

@@ -384,7 +384,7 @@ class ResourceMixin(object):
             return to_naive_datetime(self.harvest.modified_at)
         if self.filetype == "remote" and self.extras.get("analysis:last-modified-at"):
             return to_naive_datetime(self.extras.get("analysis:last-modified-at"))
-        return self.last_modified_internal
+        return to_naive_datetime(self.last_modified_internal)
 
     def clean(self):
         super(ResourceMixin, self).clean()
@@ -565,6 +565,8 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
     extras = db.ExtrasField()
     harvest = db.EmbeddedDocumentField(HarvestDatasetMetadata)
 
+    quality_cached = db.DictField()
+
     featured = db.BooleanField(required=True, default=False)
 
     contact_points = db.ListField(db.ReferenceField("ContactPoint", reverse_delete_rule=db.PULL))
@@ -672,6 +674,8 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
         if len(set(res.id for res in self.resources)) != len(self.resources):
             raise MongoEngineValidationError(f"Duplicate resource ID in dataset #{self.id}.")
 
+        self.quality_cached = self.compute_quality()
+
         for key, value in self.extras.items():
             if not key.startswith("custom:"):
                 continue
@@ -763,13 +767,9 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
 
     @property
     def last_modified(self):
-        if (
-            self.harvest
-            and self.harvest.modified_at
-            and to_naive_datetime(self.harvest.modified_at) < datetime.utcnow()
-        ):
+        if self.harvest and self.harvest.modified_at:
             return to_naive_datetime(self.harvest.modified_at)
-        return self.last_modified_internal
+        return to_naive_datetime(self.last_modified_internal)
 
     @property
     def last_update(self):
@@ -824,8 +824,34 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
         else:
             return self.last_update + delta
 
-    @cached_property
+    @property
     def quality(self):
+        # `quality_cached` should always be set, except during the migration
+        # creating this property. We could remove `or self.compute_quality()`
+        # after the migration but since we need to keep the computed property for
+        # `update_fulfilled_in_time`, maybe we leave it here? Just in case?
+        quality = self.quality_cached or self.compute_quality()
+
+        # :UpdateFulfilledInTime
+        # `next_update_for_update_fulfilled_in_time` is only useful to compute the
+        # real `update_fulfilled_in_time` check, so we pop it to not polute the `quality`
+        # object for users.
+        next_update = quality.pop("next_update_for_update_fulfilled_in_time", None)
+        if next_update:
+            # Allow for being one day late on update.
+            # We may have up to one day delay due to harvesting for example
+            quality["update_fulfilled_in_time"] = (next_update - datetime.utcnow()).days >= -1
+        elif self.frequency in ["continuous", "irregular", "punctual"]:
+            # For these frequencies, we don't expect regular updates or can't quantify them.
+            # Thus we consider the update_fulfilled_in_time quality criterion to be true.
+            quality["update_fulfilled_in_time"] = True
+
+        # Since `update_fulfilled_in_time` cannot be precomputed, `score` cannot either.
+        quality["score"] = self.compute_quality_score(quality)
+
+        return quality
+
+    def compute_quality(self):
         """Return a dict filled with metrics related to the inner
 
         quality of the dataset:
@@ -835,25 +861,18 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
             * and so on
         """
         result = {}
-        if not self.id:
-            # Quality is only relevant on saved Datasets
-            return result
 
         result["license"] = True if self.license else False
         result["temporal_coverage"] = True if self.temporal_coverage else False
         result["spatial"] = True if self.spatial else False
 
         result["update_frequency"] = self.frequency and self.frequency != "unknown"
-        if self.next_update:
-            # Allow for being one day late on update.
-            # We may have up to one day delay due to harvesting for example
-            result["update_fulfilled_in_time"] = (
-                True if (self.next_update - datetime.utcnow()).days >= -1 else False
-            )
-        elif self.frequency in ["continuous", "irregular", "punctual"]:
-            # For these frequencies, we don't expect regular updates or can't quantify them.
-            # Thus we consider the update_fulfilled_in_time quality criterion to be true.
-            result["update_fulfilled_in_time"] = True
+
+        # We only save the next_update here because it is based on resources
+        # We cannot save the `update_fulfilled_in_time` because it is time
+        # sensitive (so setting it on save is not really useful…)
+        # See :UpdateFulfilledInTime
+        result["next_update_for_update_fulfilled_in_time"] = self.next_update
 
         result["dataset_description_quality"] = (
             True
@@ -876,7 +895,6 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
                     resource_desc = True
             result["resources_documentation"] = resource_doc or resource_desc
 
-        result["score"] = self.compute_quality_score(result)
         return result
 
     @property
@@ -934,8 +952,16 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
         if resource.id in [r.id for r in self.resources]:
             raise MongoEngineValidationError("Cannot add resource with already existing ID")
 
+        self.resources.insert(0, resource)
         self.update(
-            __raw__={"$push": {"resources": {"$each": [resource.to_mongo()], "$position": 0}}}
+            __raw__={
+                "$set": {
+                    "quality_cached": self.compute_quality(),
+                },
+                "$push": {
+                    "resources": {"$each": [resource.to_mongo()], "$position": 0},
+                },
+            }
         )
         self.reload()
         self.on_resource_added.send(self.__class__, document=self, resource_id=resource.id)

udata/core/dataset/rdf.py

@@ -5,7 +5,7 @@ This module centralize dataset helpers for RDF/DCAT serialization and parsing
 import calendar
 import json
 import logging
-from datetime import date
+from datetime import date, datetime
 from typing import Optional
 
 from dateutil.parser import parse as parse_dt
@@ -50,7 +50,7 @@ from udata.rdf import (
     url_from_rdf,
 )
 from udata.uris import endpoint_for
-from udata.utils import get_by, safe_unicode
+from udata.utils import get_by, safe_unicode, to_naive_datetime
 
 from .constants import OGC_SERVICE_FORMATS, UPDATE_FREQUENCIES
 from .models import Checksum, Dataset, License, Resource
@@ -735,7 +735,14 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
     if not resource.harvest:
         resource.harvest = HarvestResourceMetadata()
     resource.harvest.created_at = created_at
-    resource.harvest.modified_at = modified_at
+
+    # In the past, we've encountered future `modified_at` during harvesting
+    # do not save it. :FutureHarvestModifiedAt
+    if modified_at and to_naive_datetime(modified_at) > datetime.utcnow():
+        log.warning(f"Future `DCT.modified` date '{modified_at}' in resource")
+    else:
+        resource.harvest.modified_at = modified_at
+
     resource.harvest.dct_identifier = identifier
     resource.harvest.uri = uri
 
@@ -836,7 +843,13 @@ def dataset_from_rdf(graph: Graph, dataset=None, node=None, remote_url_prefix: s
     dataset.harvest.uri = uri
     dataset.harvest.remote_url = remote_url
     dataset.harvest.created_at = created_at
-    dataset.harvest.modified_at = modified_at
+
+    # In the past, we've encountered future `modified_at` during harvesting
+    # do not save it. :FutureHarvestModifiedAt
+    if modified_at and to_naive_datetime(modified_at) > datetime.utcnow():
+        log.warning(f"Future `DCT.modified` date '{modified_at}' in dataset")
+    else:
+        dataset.harvest.modified_at = modified_at
 
     return dataset
 

udata/migrations/2025-03-20-save-quality-for-datasets.py

@@ -0,0 +1,25 @@
+"""
+This migration keeps only the "Local authority" badge if the organization also has the "Public service" badge.
+"""
+
+import logging
+
+import click
+
+from udata.core.dataset.models import Dataset
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Saving all datasets")
+
+    count = Dataset.objects().count()
+    with click.progressbar(Dataset.objects(), length=count) as datasets:
+        for dataset in datasets:
+            try:
+                dataset.save()
+            except Exception as err:
+                log.error(f"Cannot save dataset {dataset.id} {err}")
+
+    log.info("Done")

udata/tests/apiv2/test_datasets.py

@@ -21,7 +21,7 @@ class DatasetAPIV2Test(APITestCase):
         resources_a = [ResourceFactory() for _ in range(2)]
         dataset_a = DatasetFactory(title="Dataset A", resources=resources_a)
 
-        resources_b = [ResourceFactory() for _ in range(4)]
+        resources_b = [ResourceFactory(format="csv") for _ in range(4)]
         dataset_b = DatasetFactory(title="Dataset B", resources=resources_b)
 
         response = self.get(url_for("apiv2.datasets"))
@@ -32,6 +32,11 @@ class DatasetAPIV2Test(APITestCase):
         assert data["data"][1]["title"] == dataset_a.title
         assert data["data"][0]["title"] == dataset_b.title
 
+        assert data["data"][1]["quality"]["has_resources"]
+        assert not data["data"][1]["quality"]["has_open_format"]
+        assert data["data"][0]["quality"]["has_resources"]
+        assert data["data"][0]["quality"]["has_open_format"]
+
         assert data["data"][1]["resources"]["total"] == len(resources_a)
         assert data["data"][0]["resources"]["total"] == len(resources_b)
 
@@ -45,6 +50,7 @@ class DatasetAPIV2Test(APITestCase):
         response = self.get(url_for("apiv2.dataset", dataset=dataset))
         self.assert200(response)
         data = response.json
+        assert data["quality"]["has_resources"]
         assert data["resources"]["rel"] == "subsection"
         assert data["resources"]["href"] == url_for(
             "apiv2.resources",

udata/tests/dataset/test_dataset_model.py

@@ -748,16 +748,6 @@ class HarvestMetadataTest:
         dataset.harvest = harvest_metadata
         dataset.save()
 
-    def test_harvest_dataset_metadata_future_modifed_at(self):
-        dataset = DatasetFactory()
-
-        harvest_metadata = HarvestDatasetMetadata(
-            created_at=datetime.utcnow(), modified_at=datetime.utcnow() + timedelta(days=1)
-        )
-        dataset.harvest = harvest_metadata
-        dataset.save()
-        assert dataset.last_modified == dataset.last_modified_internal
-
     def test_harvest_dataset_metadata_past_modifed_at(self):
         dataset = DatasetFactory()
 

udata/tests/dataset/test_dataset_rdf.py

@@ -483,6 +483,24 @@ class RdfToDatasetTest:
         assert isinstance(dataset, Dataset)
         assert dataset.description == "a description"
 
+    def test_future_modified_at(self):
+        node = BNode()
+        g = Graph()
+
+        modified = faker.future_datetime()
+
+        g.add((node, RDF.type, DCAT.Dataset))
+        g.add((node, DCT.identifier, Literal(faker.uuid4())))
+        g.add((node, DCT.title, Literal(faker.sentence())))
+        g.add((node, DCT.description, Literal("<div>a description</div>")))
+        g.add((node, DCT.modified, Literal(modified)))
+
+        dataset = dataset_from_rdf(g)
+        dataset.validate()
+
+        assert isinstance(dataset, Dataset)
+        assert dataset.harvest.modified_at is None
+
     def test_theme_and_tags(self):
         node = BNode()
         g = Graph()

{udata-10.2.1.dev34728.dist-info → udata-10.2.1.dev34761.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata
-Version: 10.2.1.dev34728
+Version: 10.2.1.dev34761
 Summary: Open data portal
 Home-page: https://github.com/opendatateam/udata
 Author: Opendata Team
@@ -143,6 +143,7 @@ It is collectively taken care of by members of the
 
 - Skip empty datasets nodes referenced in dataservices before creating an harvest job item [#3285](https://github.com/opendatateam/udata/pull/3285)
 - Harvest Distribution with a DCAT.accessService property as type `api` [#3294](https://github.com/opendatateam/udata/pull/3294)
+- Compute quality score to show correct resources score in api v2 [#3290](https://github.com/opendatateam/udata/pull/3290/)
 
 ## 10.2.0 (2025-04-02)
 

{udata-10.2.1.dev34728.dist-info → udata-10.2.1.dev34761.dist-info}/RECORD

@@ -103,10 +103,10 @@ udata/core/dataset/events.py,sha256=bSM0nFEX14r4JHc-bAM-7OOuD3JAxUIpw9GgXbOsUyw,
 udata/core/dataset/exceptions.py,sha256=uKiayLSpSzsnLvClObS6hOO0qXEqvURKN7_w8eimQNU,498
 udata/core/dataset/factories.py,sha256=fRDWDlybR_ud4pDs1-ntWuYHKtV9LMHeBOBp2SmTT6M,9006
 udata/core/dataset/forms.py,sha256=nGHC92MH3NmyfASuYRgoBzyNTpsWL_1h3R7B9t7nnxU,6295
-udata/core/dataset/models.py,sha256=GurpAR_IyPqqp9l72IB0EQMafInYigE0iEU_h7CpCV0,39594
+udata/core/dataset/models.py,sha256=2c6vjr5KPq6Pqg6MK5pRFrXzpTIXtRC2zRxDlxiFTL8,40834
 udata/core/dataset/permissions.py,sha256=zXQ6kU-Ni3Pl5tDtat-ZPupug9InsNeCN7xRLc2Vcrc,1097
 udata/core/dataset/preview.py,sha256=IwCqiNTjjXbtA_SSKF52pwnzKKEz0GyYM95QNn2Dkog,2561
-udata/core/dataset/rdf.py,sha256=-XsXa7uN1Ovt1RjC_-LYewTAMMRxG270nEocMeBk_i0,31176
+udata/core/dataset/rdf.py,sha256=HkjzcWgq9AfPvUGMRI7-ufRrgnlfBmP8crbgRhg6Lz4,31789
 udata/core/dataset/search.py,sha256=E7LqHBnq3sMefvmLwTpiw-Ovem2a3NJswHesRjctboE,5627
 udata/core/dataset/signals.py,sha256=WN4sV-lJlNsRkhcnhoy0SYJvCoYmK_5QFYZd1u-h4gs,161
 udata/core/dataset/tasks.py,sha256=jG6U7cwGywhLK-MqUrvbZ1WLl_NLApiThtxmsckGzP8,10009
@@ -359,6 +359,7 @@ udata/migrations/2024-10-30-rename-organization-badges.py,sha256=Yu8PTayYY0m4YJJ
 udata/migrations/2024-11-19-keep-only-local_authority-if-also-public_service-organization-badges.py,sha256=xFn0nVqAh8Vee9-s8YFZHg5F-kukOr5B4fyz62G40eA,658
 udata/migrations/2024-12-05-contact-point-is-now-a-list.py,sha256=il2qSFhOTq-YhqhFaq_5OwysUlKXaK_En-qGP6v9rf0,1065
 udata/migrations/2025-01-05-dataservices-fields-changes.py,sha256=HlqHg3sG3rk3sYVrOwAlXMNhTmTKd1YT82P-gXOqmZM,4647
+udata/migrations/2025-03-20-save-quality-for-datasets.py,sha256=FPTfGVByXSHr18V4RFlktC7t-H-5rgEcZQMTRpMrGqo,607
 udata/migrations/__init__.py,sha256=RBCBDaTlLjuMs_Qzwji6Z6T4r7FCGXhESKoxQbT5qAA,11221
 udata/models/__init__.py,sha256=txbZwa-lRG3mq99eQ9E5YcFWiNUdjDVSyJJvlqUMFfs,1413
 udata/mongo/__init__.py,sha256=y4Rv-kq3o_kcEulcNpePLzocXPBNpx3Jd82G-VZPaMc,1421
@@ -634,7 +635,7 @@ udata/tests/api/test_topics_api.py,sha256=mpIjztDEQ2fx25E9dl4eIARGqtYuh1w8x68PhR
 udata/tests/api/test_transfer_api.py,sha256=-OLv-KjyLZL14J8UHl-ak_sYUj6wFiZWyoXC2SMXmEQ,7503
 udata/tests/api/test_user_api.py,sha256=hfDHOv5F81tehbc5u3dYH5rAyYgzgHGCWVcGR3kLIZw,16591
 udata/tests/apiv2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-udata/tests/apiv2/test_datasets.py,sha256=3M4pEvxQS1bCzP4Cje-Sj-1QpkR8q3EQnJDD4w15Dpk,21668
+udata/tests/apiv2/test_datasets.py,sha256=zjTtjY7lJKOOjiV698RqGvRMuuMxY2dMhEcnhOXWUQg,21973
 udata/tests/apiv2/test_me_api.py,sha256=RAgu5iKGB3Pp8cDN-427ovly-VOz5MTL4GpPZ1NcVXI,1418
 udata/tests/apiv2/test_organizations.py,sha256=os_43s-coSRqjgY-5fAjSiRlB3g2685u7d-Es0aOhks,6390
 udata/tests/apiv2/test_swagger.py,sha256=RKedaq-2UeyEuxlmUaAN7pmEe-lQYYmpDUVc8HF3CH4,785
@@ -652,8 +653,8 @@ udata/tests/dataset/test_csv_adapter.py,sha256=CSAHEur5-183fRsD_cBgsHnLuohn0DOb9
 udata/tests/dataset/test_dataset_actions.py,sha256=bgDjVYjOvu3sX_FCTCzf2snZYSprsqor2nAhIVuokSs,722
 udata/tests/dataset/test_dataset_commands.py,sha256=zMPJG2wYwKBee2zI65kmboxf59Zqa84DDjT8V5wj9uo,801
 udata/tests/dataset/test_dataset_events.py,sha256=hlrpoOiBbnX_COUI9Pzdqlp45GZZDqu5piwupbnPiTI,3601
-udata/tests/dataset/test_dataset_model.py,sha256=uz2oNuzL6tEqd34JrBP0J1BVlt8JQoAB_u3tYM9etPg,32407
-udata/tests/dataset/test_dataset_rdf.py,sha256=1eBbn7rl2bcHGaJoxXMnJpszteShXvVg3djEw6HxOUw,40721
+udata/tests/dataset/test_dataset_model.py,sha256=YX9ELWA6q2tW3OEI_rDS7BPfE2p5ocA5pB9_2Yyy8Ac,32017
+udata/tests/dataset/test_dataset_rdf.py,sha256=fWMfhJvxmRKPw7cdvksS-sJBJT4G9DepDIm5RTdvrJ4,41310
 udata/tests/dataset/test_dataset_tasks.py,sha256=n1W2Pg0ez02d66zQG3N93kh7dpR2yLMRDqUI6PnPaI0,3088
 udata/tests/dataset/test_resource_preview.py,sha256=fp9mSL7unhyM66GR0gwhgX3OGQ4TJt7G9xU-CjsL3HI,3908
 udata/tests/features/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -720,9 +721,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=xmSQ7RTl9XV24Vq3A7f_NIRDrOJwbs
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=PXrrhfIjZxheUotQan9VVh1fvNHSzeMAitaouIHaR7U,46793
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=1pAf_rXvbOoO_jjZmH77GbzvdT_YtPTJKFumMnMto2g,29169
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=AalMHaFLZobKmuAnZ4X1rQtk46NdW2rktMFQHD5DTcM,53768
-udata-10.2.1.dev34728.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-10.2.1.dev34728.dist-info/METADATA,sha256=X_iyv6wTV8PL7AqcqQYNU43ISlJ08S8OpCDrFbm1n98,143896
-udata-10.2.1.dev34728.dist-info/WHEEL,sha256=Kh9pAotZVRFj97E15yTA4iADqXdQfIVTHcNaZTjxeGM,110
-udata-10.2.1.dev34728.dist-info/entry_points.txt,sha256=3SKiqVy4HUqxf6iWspgMqH8d88Htk6KoLbG1BU-UddQ,451
-udata-10.2.1.dev34728.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
-udata-10.2.1.dev34728.dist-info/RECORD,,
+udata-10.2.1.dev34761.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-10.2.1.dev34761.dist-info/METADATA,sha256=T6dSfDKTVv_SfF8njP7sDL6RhZeoGgjVzqD1EMODnSM,144020
+udata-10.2.1.dev34761.dist-info/WHEEL,sha256=Kh9pAotZVRFj97E15yTA4iADqXdQfIVTHcNaZTjxeGM,110
+udata-10.2.1.dev34761.dist-info/entry_points.txt,sha256=3SKiqVy4HUqxf6iWspgMqH8d88Htk6KoLbG1BU-UddQ,451
+udata-10.2.1.dev34761.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
+udata-10.2.1.dev34761.dist-info/RECORD,,