udata 11.1.2.dev9__py3-none-any.whl → 11.1.2.dev10__py3-none-any.whl

udata/harvest/backends/ckan/harvesters.py

@@ -4,19 +4,14 @@ from urllib.parse import urljoin
 from uuid import UUID
 
 from udata import uris
-from udata.harvest.models import HarvestItem
-from udata.i18n import lazy_gettext as _
-
-try:
-    from udata.core.dataset.constants import UPDATE_FREQUENCIES
-except ImportError:
-    # legacy import of constants in udata
-    from udata.models import UPDATE_FREQUENCIES
+from udata.core.dataset.constants import UpdateFrequency
 from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
 from udata.core.dataset.rdf import frequency_from_rdf
 from udata.frontend.markdown import parse_html
 from udata.harvest.backends.base import BaseBackend, HarvestFilter
 from udata.harvest.exceptions import HarvestException, HarvestSkipException
+from udata.harvest.models import HarvestItem
+from udata.i18n import lazy_gettext as _
 from udata.models import GeoZone, License, Resource, SpatialCoverage, db
 from udata.utils import daterange_end, daterange_start, get_by
 
@@ -193,14 +188,15 @@ class CkanBackend(BaseBackend):
                 log.debug("spatial-uri value not handled: %s", value)
             elif key == "frequency":
                 # Update frequency
-                freq = frequency_from_rdf(value)
-                if freq:
+                if freq := frequency_from_rdf(value):
                     dataset.frequency = freq
-                elif value in UPDATE_FREQUENCIES:
-                    dataset.frequency = value
                 else:
-                    dataset.extras["ckan:frequency"] = value
-                    log.debug("frequency value not handled: %s", value)
+                    # FIXME(python 3.12+): prefer `if value in UpdateFrequency`
+                    try:
+                        dataset.frequency = UpdateFrequency(value)
+                    except ValueError:
+                        dataset.extras["ckan:frequency"] = value
+                        log.debug("frequency value not handled: %s", value)
             # Temporal coverage start
             elif key == "temporal_start":
                 temporal_start = daterange_start(value)

udata/harvest/backends/maaf.py

@@ -7,6 +7,7 @@ from urllib.parse import urljoin
 from lxml import etree, html
 from voluptuous import All, Any, In, Length, Lower, Optional, Schema
 
+from udata.core.dataset.constants import UpdateFrequency
 from udata.harvest.backends import BaseBackend
 from udata.harvest.filters import (
     boolean,
@@ -38,19 +39,19 @@ ZONES = {
 
 
 FREQUENCIES = {
-    "ponctuelle": "punctual",
-    "temps réel": "continuous",
-    "quotidienne": "daily",
-    "hebdomadaire": "weekly",
-    "bimensuelle": "semimonthly",
-    "mensuelle": "monthly",
-    "bimestrielle": "bimonthly",
-    "trimestrielle": "quarterly",
-    "semestrielle": "semiannual",
-    "annuelle": "annual",
-    "triennale": "triennial",
-    "quinquennale": "quinquennial",
-    "aucune": "unknown",
+    "ponctuelle": UpdateFrequency.PUNCTUAL,
+    "temps réel": UpdateFrequency.CONTINUOUS,
+    "quotidienne": UpdateFrequency.DAILY,
+    "hebdomadaire": UpdateFrequency.WEEKLY,
+    "bimensuelle": UpdateFrequency.SEMIMONTHLY,
+    "mensuelle": UpdateFrequency.MONTHLY,
+    "bimestrielle": UpdateFrequency.BIMONTHLY,
+    "trimestrielle": UpdateFrequency.QUARTERLY,
+    "semestrielle": UpdateFrequency.SEMIANNUAL,
+    "annuelle": UpdateFrequency.ANNUAL,
+    "triennale": UpdateFrequency.TRIENNIAL,
+    "quinquennale": UpdateFrequency.QUINQUENNIAL,
+    "aucune": UpdateFrequency.UNKNOWN,
 }
 
 XSD_PATH = os.path.join(os.path.dirname(__file__), "maaf.xsd")
@@ -161,7 +162,7 @@ class MaafBackend(BaseBackend):
         dataset = self.get_dataset(item.remote_id)
 
         dataset.title = metadata["title"]
-        dataset.frequency = FREQUENCIES.get(metadata["frequency"], "unknown")
+        dataset.frequency = FREQUENCIES.get(metadata["frequency"], UpdateFrequency.UNKNOWN)
         dataset.description = metadata["notes"]
         dataset.private = metadata["private"]
         dataset.tags = sorted(set(metadata["tags"]))

udata/harvest/tests/ckan/test_ckan_backend.py

@@ -5,6 +5,7 @@ from datetime import date
 import pytest
 
 from udata.app import create_app
+from udata.core.dataset.constants import UpdateFrequency
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.spatial.factories import GeoZoneFactory
 from udata.harvest import actions
@@ -336,7 +337,7 @@ def frequency_as_rdf_uri(resource_data):
         "resources": [resource_data],
         "extras": [{"key": "frequency", "value": "http://purl.org/cld/freq/daily"}],
     }
-    return data, {"expected": "daily"}
+    return data, {"expected": UpdateFrequency.DAILY}
 
 
 @pytest.fixture
@@ -348,7 +349,7 @@ def frequency_as_exact_match(resource_data):
         "resources": [resource_data],
         "extras": [{"key": "frequency", "value": "daily"}],
     }
-    return data, {"expected": "daily"}
+    return data, {"expected": UpdateFrequency.DAILY}
 
 
 @pytest.fixture
@@ -486,7 +487,7 @@ def test_can_parse_frequency_as_exact_match(result, kwargs):
 
 
 @pytest.mark.ckan_data("frequency_as_unknown_value")
-def test_can_parse_frequency_as_unkown_value(result, kwargs):
+def test_can_parse_frequency_as_unknown_value(result, kwargs):
     dataset = dataset_for(result)
     assert dataset.extras["ckan:frequency"] == kwargs["expected"]
     assert dataset.frequency is None

udata/harvest/tests/test_dcat_backend.py

@@ -10,6 +10,7 @@ from rdflib import Graph
 
 from udata.core.dataservices.factories import DataserviceFactory
 from udata.core.dataservices.models import Dataservice
+from udata.core.dataset.constants import UpdateFrequency
 from udata.core.dataset.factories import DatasetFactory, LicenseFactory, ResourceSchemaMockData
 from udata.core.dataset.rdf import dataset_from_rdf
 from udata.core.organization.factories import OrganizationFactory
@@ -560,7 +561,7 @@ class DcatBackendTest:
         assert dataset.harvest.issued_at.date() == date(2016, 12, 14)
         assert dataset.harvest.created_at.date() == date(2016, 12, 12)
         assert dataset.harvest.modified_at.date() == date(2016, 12, 14)
-        assert dataset.frequency == "daily"
+        assert dataset.frequency == UpdateFrequency.DAILY
         assert dataset.description == "Dataset 3 description"
 
         assert dataset.temporal_coverage is not None
@@ -681,7 +682,7 @@ class DcatBackendTest:
         dataset = Dataset.objects.filter(organization=org).first()
 
         assert dataset is not None
-        assert dataset.frequency == "irregular"
+        assert dataset.frequency == UpdateFrequency.IRREGULAR
         assert "gravi" in dataset.tags  # support dcat:keyword
         assert "geodesy" in dataset.tags  # support dcat:theme
         assert dataset.license.id == "fr-lo"

udata/migrations/2025-09-04-update-legacy-frequencies.py

@@ -0,0 +1,36 @@
+"""
+Convert legacy frequencies to latest vocabulary
+"""
+
+import logging
+
+from udata.core.dataset.constants import UpdateFrequency
+from udata.models import Dataset
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Updating datasets legacy frequencies:")
+
+    for legacy_value, frequency in UpdateFrequency._LEGACY_FREQUENCIES.items():
+        count = 0
+        for dataset in Dataset.objects(frequency=legacy_value).no_cache().timeout(False):
+            # Explicitly call update() to force writing the new frequency string to mongo.
+            # We can't rely on save() here because:
+            # 1. save() only writes modified fields to mongo, basically comparing the Dataset's
+            #    object state initially returned by the query with its state when save() is called,
+            #    and only sending the diffset to mongo.
+            # 2. At the ODM layer, the Dataset.frequency field has already been instantiated as an
+            #    UpdateFrequency object, and so legacy frequency strings have already been
+            #    mapped to their new ids (via UpdateFrequency._missing_).
+            # => While the raw frequency string has changed, the save() function sees the same
+            #    UpdateFrequency and therefore ignores the field in is diffset.
+            dataset.update(frequency=frequency.value)
+            # Still call save() afterwards so computed fields like quality_cached are updated if
+            # necessary, e.g. if moving from a predictable timedelta to an unpredictable one.
+            dataset.save()
+            count += 1
+        log.info(f"- {legacy_value} -> {frequency.value}: {count} updated")
+
+    log.info("Completed.")