udata 10.2.1.dev34693__py2.py3-none-any.whl → 10.2.1.dev34761__py2.py3-none-any.whl

udata/core/dataset/models.py

@@ -384,7 +384,7 @@ class ResourceMixin(object):
             return to_naive_datetime(self.harvest.modified_at)
         if self.filetype == "remote" and self.extras.get("analysis:last-modified-at"):
             return to_naive_datetime(self.extras.get("analysis:last-modified-at"))
-        return self.last_modified_internal
+        return to_naive_datetime(self.last_modified_internal)
 
     def clean(self):
         super(ResourceMixin, self).clean()
@@ -565,6 +565,8 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
     extras = db.ExtrasField()
     harvest = db.EmbeddedDocumentField(HarvestDatasetMetadata)
 
+    quality_cached = db.DictField()
+
     featured = db.BooleanField(required=True, default=False)
 
     contact_points = db.ListField(db.ReferenceField("ContactPoint", reverse_delete_rule=db.PULL))
@@ -672,6 +674,8 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
         if len(set(res.id for res in self.resources)) != len(self.resources):
             raise MongoEngineValidationError(f"Duplicate resource ID in dataset #{self.id}.")
 
+        self.quality_cached = self.compute_quality()
+
         for key, value in self.extras.items():
             if not key.startswith("custom:"):
                 continue
@@ -763,13 +767,9 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
 
     @property
     def last_modified(self):
-        if (
-            self.harvest
-            and self.harvest.modified_at
-            and to_naive_datetime(self.harvest.modified_at) < datetime.utcnow()
-        ):
+        if self.harvest and self.harvest.modified_at:
             return to_naive_datetime(self.harvest.modified_at)
-        return self.last_modified_internal
+        return to_naive_datetime(self.last_modified_internal)
 
     @property
     def last_update(self):
@@ -824,8 +824,34 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
         else:
             return self.last_update + delta
 
-    @cached_property
+    @property
     def quality(self):
+        # `quality_cached` should always be set, except during the migration
+        # creating this property. We could remove `or self.compute_quality()`
+        # after the migration but since we need to keep the computed property for
+        # `update_fulfilled_in_time`, maybe we leave it here? Just in case?
+        quality = self.quality_cached or self.compute_quality()
+
+        # :UpdateFulfilledInTime
+        # `next_update_for_update_fulfilled_in_time` is only useful to compute the
+        # real `update_fulfilled_in_time` check, so we pop it to not polute the `quality`
+        # object for users.
+        next_update = quality.pop("next_update_for_update_fulfilled_in_time", None)
+        if next_update:
+            # Allow for being one day late on update.
+            # We may have up to one day delay due to harvesting for example
+            quality["update_fulfilled_in_time"] = (next_update - datetime.utcnow()).days >= -1
+        elif self.frequency in ["continuous", "irregular", "punctual"]:
+            # For these frequencies, we don't expect regular updates or can't quantify them.
+            # Thus we consider the update_fulfilled_in_time quality criterion to be true.
+            quality["update_fulfilled_in_time"] = True
+
+        # Since `update_fulfilled_in_time` cannot be precomputed, `score` cannot either.
+        quality["score"] = self.compute_quality_score(quality)
+
+        return quality
+
+    def compute_quality(self):
         """Return a dict filled with metrics related to the inner
 
         quality of the dataset:
@@ -835,25 +861,18 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
             * and so on
         """
         result = {}
-        if not self.id:
-            # Quality is only relevant on saved Datasets
-            return result
 
         result["license"] = True if self.license else False
         result["temporal_coverage"] = True if self.temporal_coverage else False
         result["spatial"] = True if self.spatial else False
 
         result["update_frequency"] = self.frequency and self.frequency != "unknown"
-        if self.next_update:
-            # Allow for being one day late on update.
-            # We may have up to one day delay due to harvesting for example
-            result["update_fulfilled_in_time"] = (
-                True if (self.next_update - datetime.utcnow()).days >= -1 else False
-            )
-        elif self.frequency in ["continuous", "irregular", "punctual"]:
-            # For these frequencies, we don't expect regular updates or can't quantify them.
-            # Thus we consider the update_fulfilled_in_time quality criterion to be true.
-            result["update_fulfilled_in_time"] = True
+
+        # We only save the next_update here because it is based on resources
+        # We cannot save the `update_fulfilled_in_time` because it is time
+        # sensitive (so setting it on save is not really useful…)
+        # See :UpdateFulfilledInTime
+        result["next_update_for_update_fulfilled_in_time"] = self.next_update
 
         result["dataset_description_quality"] = (
             True
@@ -876,7 +895,6 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
                     resource_desc = True
             result["resources_documentation"] = resource_doc or resource_desc
 
-        result["score"] = self.compute_quality_score(result)
         return result
 
     @property
@@ -934,8 +952,16 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
         if resource.id in [r.id for r in self.resources]:
             raise MongoEngineValidationError("Cannot add resource with already existing ID")
 
+        self.resources.insert(0, resource)
         self.update(
-            __raw__={"$push": {"resources": {"$each": [resource.to_mongo()], "$position": 0}}}
+            __raw__={
+                "$set": {
+                    "quality_cached": self.compute_quality(),
+                },
+                "$push": {
+                    "resources": {"$each": [resource.to_mongo()], "$position": 0},
+                },
+            }
         )
         self.reload()
         self.on_resource_added.send(self.__class__, document=self, resource_id=resource.id)

udata/core/dataset/rdf.py

@@ -5,7 +5,7 @@ This module centralize dataset helpers for RDF/DCAT serialization and parsing
 import calendar
 import json
 import logging
-from datetime import date
+from datetime import date, datetime
 from typing import Optional
 
 from dateutil.parser import parse as parse_dt
@@ -50,7 +50,7 @@ from udata.rdf import (
     url_from_rdf,
 )
 from udata.uris import endpoint_for
-from udata.utils import get_by, safe_unicode
+from udata.utils import get_by, safe_unicode, to_naive_datetime
 
 from .constants import OGC_SERVICE_FORMATS, UPDATE_FREQUENCIES
 from .models import Checksum, Dataset, License, Resource
@@ -735,7 +735,14 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
     if not resource.harvest:
         resource.harvest = HarvestResourceMetadata()
     resource.harvest.created_at = created_at
-    resource.harvest.modified_at = modified_at
+
+    # In the past, we've encountered future `modified_at` during harvesting
+    # do not save it. :FutureHarvestModifiedAt
+    if modified_at and to_naive_datetime(modified_at) > datetime.utcnow():
+        log.warning(f"Future `DCT.modified` date '{modified_at}' in resource")
+    else:
+        resource.harvest.modified_at = modified_at
+
     resource.harvest.dct_identifier = identifier
     resource.harvest.uri = uri
 
@@ -755,6 +762,8 @@ def dataset_from_rdf(graph: Graph, dataset=None, node=None, remote_url_prefix: s
 
     dataset.title = rdf_value(d, DCT.title)
     if not dataset.title:
+        # If the dataset is externaly defined (so without title and just with a link to the dataset XML)
+        # we should have skipped it way before in :ExcludeExternalyDefinedDataset
         raise HarvestSkipException("missing title on dataset")
 
     # Support dct:abstract if dct:description is missing (sometimes used instead)
@@ -834,7 +843,13 @@ def dataset_from_rdf(graph: Graph, dataset=None, node=None, remote_url_prefix: s
     dataset.harvest.uri = uri
     dataset.harvest.remote_url = remote_url
     dataset.harvest.created_at = created_at
-    dataset.harvest.modified_at = modified_at
+
+    # In the past, we've encountered future `modified_at` during harvesting
+    # do not save it. :FutureHarvestModifiedAt
+    if modified_at and to_naive_datetime(modified_at) > datetime.utcnow():
+        log.warning(f"Future `DCT.modified` date '{modified_at}' in dataset")
+    else:
+        dataset.harvest.modified_at = modified_at
 
     return dataset
 

udata/harvest/backends/base.py

@@ -256,7 +256,7 @@ class BaseBackend(object):
             ]
             self.save_job()
 
-    def is_done(self) -> bool:
+    def has_reached_max_items(self) -> bool:
         """Should be called after process_dataset to know if we reach the max items"""
         return self.max_items and len(self.job.items) >= self.max_items
 

udata/harvest/backends/dcat.py

@@ -9,7 +9,7 @@ from rdflib.namespace import RDF
 
 from udata.core.dataservices.rdf import dataservice_from_rdf
 from udata.core.dataset.rdf import dataset_from_rdf
-from udata.harvest.models import HarvestItem
+from udata.harvest.models import HarvestError, HarvestItem
 from udata.i18n import gettext as _
 from udata.rdf import (
     DCAT,
@@ -18,6 +18,7 @@ from udata.rdf import (
     SPDX,
     guess_format,
     namespace_manager,
+    rdf_value,
     url_from_rdf,
 )
 from udata.storage.s3 import store_as_json
@@ -77,9 +78,19 @@ class DcatBackend(BaseBackend):
             self.process_one_datasets_page(page_number, page)
             serialized_graphs.append(page.serialize(format=fmt, indent=None))
 
+        # We do a second pass to have all datasets in memory and attach datasets
+        # to dataservices. It could be better to be one pass of graph walking and
+        # then one pass of attaching datasets to dataservices.
         for page_number, page in self.walk_graph(self.source.url, fmt):
             self.process_one_dataservices_page(page_number, page)
 
+        if not self.dryrun and self.has_reached_max_items():
+            # We have reached the max_items limit. Warn the user that all the datasets may not be present.
+            error = HarvestError(
+                message=f"{self.max_items} max items reached, not all datasets/dataservices were retrieved"
+            )
+            self.job.errors.append(error)
+
         # The official MongoDB document size in 16MB. The default value here is 15MB to account for other fields in the document (and for difference between * 1024 vs * 1000).
         max_harvest_graph_size_in_mongo = current_app.config.get(
             "HARVEST_MAX_CATALOG_SIZE_IN_MONGO"
@@ -146,7 +157,7 @@ class DcatBackend(BaseBackend):
                     break
 
             yield page_number, subgraph
-            if self.is_done():
+            if self.has_reached_max_items():
                 return
 
             page_number += 1
@@ -154,17 +165,48 @@ class DcatBackend(BaseBackend):
     def process_one_datasets_page(self, page_number: int, page: Graph):
         for node in page.subjects(RDF.type, DCAT.Dataset):
             remote_id = page.value(node, DCT.identifier)
+            if self.is_dataset_external_to_this_page(page, node):
+                continue
+
             self.process_dataset(remote_id, page_number=page_number, page=page, node=node)
 
-            if self.is_done():
+            if self.has_reached_max_items():
                 return
 
+    def is_dataset_external_to_this_page(self, page: Graph, node) -> bool:
+        # In dataservice nodes we have `servesDataset` or `hasPart` that can contains nodes
+        # with type=dataset. We don't want to process them because these nodes are empty (they
+        # only contains a link to the dataset definition).
+        # These datasets are either present in the catalog in previous or next pages or
+        # external from the catalog we are currently harvesting (so we don't want to harvest them).
+        # First we thought of skipping them inside `dataset_from_rdf` (see :ExcludeExternalyDefinedDataset)
+        # but it creates a lot of "fake" items in the job and raising problems (reaching the max harvest item for
+        # example and not getting to the "real" datasets/dataservices in subsequent pages)
+        # So to prevent creating a lot of useless items in the job we first thought about checking to see if there is no title and
+        # if `isPrimaryTopicOf` is present. But it may be better to check if the only link of the node with the current page is a
+        # `servesDataset` or `hasPart`. If it's the case, the node is only present in a dataservice. (maybe we could also check that
+        # the `_other_node` is a dataservice?)
+        # `isPrimaryTopicOf` is the tag present in the first harvester raising the problem, it may exists other
+        # values of the same sort we need to check here.
+
+        # This is not dangerous because we check for missing title in `dataset_from_rdf` later so we would have skipped
+        # this dataset anyway.
+        resource = page.resource(node)
+        title = rdf_value(resource, DCT.title)
+        if title:
+            return False
+
+        predicates = [link_type for (_other_node, link_type) in page.subject_predicates(node)]
+        return len(predicates) == 1 and (
+            predicates[0] == DCAT.servesDataset or predicates[0] == DCT.hasPart
+        )
+
     def process_one_dataservices_page(self, page_number: int, page: Graph):
         for node in page.subjects(RDF.type, DCAT.DataService):
             remote_id = page.value(node, DCT.identifier)
             self.process_dataservice(remote_id, page_number=page_number, page=page, node=node)
 
-            if self.is_done():
+            if self.has_reached_max_items():
                 return
 
     def inner_process_dataset(self, item: HarvestItem, page_number: int, page: Graph, node):
@@ -266,7 +308,7 @@ class CswDcatBackend(DcatBackend):
                 subgraph.parse(data=ET.tostring(child), format=fmt)
 
                 yield page_number, subgraph
-                if self.is_done():
+                if self.has_reached_max_items():
                     return
 
             next_record = self.next_record_if_should_continue(start, search_results)
@@ -375,7 +417,7 @@ class CswIso19139DcatBackend(DcatBackend):
                 raise ValueError("Failed to fetch CSW content")
 
             yield page_number, subgraph
-            if self.is_done():
+            if self.has_reached_max_items():
                 return
 
             next_record = self.next_record_if_should_continue(start, search_results)

udata/harvest/tests/factories.py

@@ -61,7 +61,7 @@ class FactoryBackend(backends.BaseBackend):
         mock_initialize.send(self)
         for i in range(self.config.get("count", DEFAULT_COUNT)):
             self.process_dataset(str(i))
-            if self.is_done():
+            if self.has_reached_max_items():
                 return
 
     def inner_process_dataset(self, item: HarvestItem):

udata/harvest/tests/test_base_backend.py

@@ -44,12 +44,12 @@ class FakeBackend(BaseBackend):
     def inner_harvest(self):
         for remote_id in self.source.config.get("dataset_remote_ids", []):
             self.process_dataset(remote_id)
-            if self.is_done():
+            if self.has_reached_max_items():
                 return
 
         for remote_id in self.source.config.get("dataservice_remote_ids", []):
             self.process_dataservice(remote_id)
-            if self.is_done():
+            if self.has_reached_max_items():
                 return
 
     def inner_process_dataset(self, item: HarvestItem):

udata/migrations/2025-03-20-save-quality-for-datasets.py

@@ -0,0 +1,25 @@
+"""
+This migration keeps only the "Local authority" badge if the organization also has the "Public service" badge.
+"""
+
+import logging
+
+import click
+
+from udata.core.dataset.models import Dataset
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Saving all datasets")
+
+    count = Dataset.objects().count()
+    with click.progressbar(Dataset.objects(), length=count) as datasets:
+        for dataset in datasets:
+            try:
+                dataset.save()
+            except Exception as err:
+                log.error(f"Cannot save dataset {dataset.id} {err}")
+
+    log.info("Done")