udata 10.8.1.dev36652__py2.py3-none-any.whl → 10.8.2__py2.py3-none-any.whl

udata/harvest/tests/test_api.py

@@ -395,7 +395,7 @@ class HarvestAPITest(MockBackendsMixin):
             "url": new_url,
             "backend": "factory",
         }
-        api_url = url_for("api.harvest_source", ident=str(source.id))
+        api_url = url_for("api.harvest_source", source=source)
         response = api.put(api_url, data)
         assert200(response)
         assert response.json["url"] == new_url
@@ -403,7 +403,7 @@ class HarvestAPITest(MockBackendsMixin):
         # Source is now owned by orga, with user as member
         source.organization = OrganizationFactory(members=[Member(user=user)])
         source.save()
-        api_url = url_for("api.harvest_source", ident=str(source.id))
+        api_url = url_for("api.harvest_source", source=source)
         response = api.put(api_url, data)
         assert200(response)
 
@@ -418,7 +418,7 @@ class HarvestAPITest(MockBackendsMixin):
             "url": new_url,
             "backend": "factory",
         }
-        api_url: str = url_for("api.harvest_source", ident=str(source.id))
+        api_url: str = url_for("api.harvest_source", source=source)
         response = api.put(api_url, data)
 
         assert403(response)
@@ -429,7 +429,7 @@ class HarvestAPITest(MockBackendsMixin):
         source = HarvestSourceFactory()
 
         data = {"state": VALIDATION_ACCEPTED}
-        url = url_for("api.validate_harvest_source", ident=str(source.id))
+        url = url_for("api.validate_harvest_source", source=source)
         response = api.post(url, data)
         assert200(response)
 
@@ -443,7 +443,7 @@ class HarvestAPITest(MockBackendsMixin):
         source = HarvestSourceFactory()
 
         data = {"state": VALIDATION_REFUSED, "comment": "Not valid"}
-        url = url_for("api.validate_harvest_source", ident=str(source.id))
+        url = url_for("api.validate_harvest_source", source=source)
         response = api.post(url, data)
         assert200(response)
 
@@ -458,22 +458,27 @@ class HarvestAPITest(MockBackendsMixin):
         source = HarvestSourceFactory()
 
         data = {"validate": True}
-        url = url_for("api.validate_harvest_source", ident=str(source.id))
+        url = url_for("api.validate_harvest_source", source=source)
         response = api.post(url, data)
         assert403(response)
 
     def test_get_source(self, api):
         source = HarvestSourceFactory()
 
-        url = url_for("api.harvest_source", ident=str(source.id))
+        url = url_for("api.harvest_source", source=source)
         response = api.get(url)
         assert200(response)
 
+    def test_get_missing_source(self, api):
+        url = url_for("api.harvest_source", source="685bb38b9cb9284b93fd9e72")
+        response = api.get(url)
+        assert404(response)
+
     def test_source_preview(self, api):
         api.login()
         source = HarvestSourceFactory(backend="factory")
 
-        url = url_for("api.preview_harvest_source", ident=str(source.id))
+        url = url_for("api.preview_harvest_source", source=source)
         response = api.get(url)
         assert200(response)
 
@@ -488,7 +493,7 @@ class HarvestAPITest(MockBackendsMixin):
             validation=HarvestSourceValidation(state=VALIDATION_ACCEPTED),
         )
 
-        url = url_for("api.run_harvest_source", ident=str(source.id))
+        url = url_for("api.run_harvest_source", source=source)
         response = api.post(url)
         assert200(response)
 
@@ -505,7 +510,7 @@ class HarvestAPITest(MockBackendsMixin):
             validation=HarvestSourceValidation(state=VALIDATION_ACCEPTED),
         )
 
-        url = url_for("api.run_harvest_source", ident=str(source.id))
+        url = url_for("api.run_harvest_source", source=source)
         response = api.post(url)
         assert400(response)
 
@@ -523,7 +528,7 @@ class HarvestAPITest(MockBackendsMixin):
             validation=HarvestSourceValidation(state=VALIDATION_ACCEPTED),
         )
 
-        url = url_for("api.run_harvest_source", ident=str(source.id))
+        url = url_for("api.run_harvest_source", source=source)
         response = api.post(url)
         assert403(response)
 
@@ -540,7 +545,7 @@ class HarvestAPITest(MockBackendsMixin):
             validation=HarvestSourceValidation(state=VALIDATION_PENDING),
         )
 
-        url = url_for("api.run_harvest_source", ident=str(source.id))
+        url = url_for("api.run_harvest_source", source=source)
         response = api.post(url)
         assert400(response)
 
@@ -556,7 +561,7 @@ class HarvestAPITest(MockBackendsMixin):
         user = api.login()
         source = HarvestSourceFactory(owner=user)
 
-        url = url_for("api.harvest_source", ident=str(source.id))
+        url = url_for("api.harvest_source", source=source)
         response = api.delete(url)
         assert204(response)
 
@@ -568,7 +573,7 @@ class HarvestAPITest(MockBackendsMixin):
         api.login()
         source = HarvestSourceFactory()
 
-        url = url_for("api.harvest_source", ident=str(source.id))
+        url = url_for("api.harvest_source", source=source)
         response = api.delete(url)
 
         assert403(response)
@@ -579,7 +584,7 @@ class HarvestAPITest(MockBackendsMixin):
         source = HarvestSourceFactory()
 
         data = "0 0 * * *"
-        url = url_for("api.schedule_harvest_source", ident=str(source.id))
+        url = url_for("api.schedule_harvest_source", source=source)
         response = api.post(url, data)
         assert200(response)
 
@@ -601,7 +606,7 @@ class HarvestAPITest(MockBackendsMixin):
         source = HarvestSourceFactory()
 
         data = "0 0 * * *"
-        url = url_for("api.schedule_harvest_source", ident=str(source.id))
+        url = url_for("api.schedule_harvest_source", source=source)
         response = api.post(url, data)
         assert403(response)
 
@@ -620,7 +625,7 @@ class HarvestAPITest(MockBackendsMixin):
         )
         source = HarvestSourceFactory(periodic_task=periodic_task)
 
-        url = url_for("api.schedule_harvest_source", ident=str(source.id))
+        url = url_for("api.schedule_harvest_source", source=source)
         response = api.delete(url)
         assert204(response)
 
@@ -639,7 +644,7 @@ class HarvestAPITest(MockBackendsMixin):
         )
         source = HarvestSourceFactory(periodic_task=periodic_task)
 
-        url = url_for("api.schedule_harvest_source", ident=str(source.id))
+        url = url_for("api.schedule_harvest_source", source=source)
         response = api.delete(url)
         assert403(response)
 

udata/harvest/tests/test_dcat_backend.py

@@ -74,7 +74,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         source.reload()
 
@@ -123,7 +123,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
 
@@ -154,7 +154,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
 
@@ -171,7 +171,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         dataservices = Dataservice.objects
 
@@ -192,7 +192,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
         assert len(datasets) == 8
@@ -244,7 +244,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
 
@@ -306,7 +306,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         assert Dataset.objects.count() == 2
         assert HarvestJob.objects.first().status == "done"
@@ -320,7 +320,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
 
@@ -345,7 +345,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
 
@@ -388,7 +388,7 @@ class DcatBackendTest:
         url = mock_dcat(rmock, filename)
         source = HarvestSourceFactory(backend="dcat", url=url, organization=OrganizationFactory())
 
-        actions.run(source.slug)
+        actions.run(source)
 
         source.reload()
 
@@ -416,8 +416,8 @@ class DcatBackendTest:
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
         # Run the same havester twice
-        actions.run(source.slug)
-        actions.run(source.slug)
+        actions.run(source)
+        actions.run(source)
 
         datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
 
@@ -431,7 +431,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         source.reload()
 
@@ -443,7 +443,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         source.reload()
 
@@ -456,7 +456,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         source.reload()
 
@@ -470,7 +470,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         source.reload()
 
@@ -493,7 +493,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         # test dct:license support
         dataset = Dataset.objects.get(harvest__dct_identifier="3")
@@ -587,7 +587,7 @@ class DcatBackendTest:
         url = mock_dcat(rmock, "geonetwork.xml", path="catalog.xml")
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
-        actions.run(source.slug)
+        actions.run(source)
         dataset = Dataset.objects.filter(organization=org).first()
         assert dataset is not None
         assert dataset.harvest is not None
@@ -616,7 +616,7 @@ class DcatBackendTest:
         url = mock_dcat(rmock, "sig.oreme.rdf")
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
-        actions.run(source.slug)
+        actions.run(source)
         dataset = Dataset.objects.filter(organization=org).first()
 
         assert dataset is not None
@@ -646,7 +646,7 @@ class DcatBackendTest:
         url = mock_dcat(rmock, "udata.xml")
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
-        actions.run(source.slug)
+        actions.run(source)
 
         source.reload()
         job = source.get_last_job()
@@ -712,7 +712,7 @@ class DcatBackendTest:
         get_mock = rmock.get(url)
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
-        actions.run(source.slug)
+        actions.run(source)
 
         assert "User-Agent" in get_mock.last_request.headers
         assert get_mock.last_request.headers["User-Agent"] == "uData/0.1 dcat"
@@ -723,7 +723,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         source.reload()
 
@@ -741,7 +741,7 @@ class DcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         source.reload()
 
@@ -773,7 +773,7 @@ class DcatBackendTest:
         rmock.head(url, headers={"Content-Type": "application/json"})
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
-        actions.run(source.slug)
+        actions.run(source)
 
         source.reload()
 
@@ -787,7 +787,7 @@ class DcatBackendTest:
         rmock.get(url, status_code=404)
 
         source = HarvestSourceFactory(backend="dcat", url=url, organization=OrganizationFactory())
-        actions.run(source.slug)
+        actions.run(source)
         source.reload()
 
         job = source.get_last_job()
@@ -800,7 +800,7 @@ class DcatBackendTest:
         rmock.head(url, status_code=404)
 
         source = HarvestSourceFactory(backend="dcat", url=url, organization=OrganizationFactory())
-        actions.run(source.slug)
+        actions.run(source)
         source.reload()
 
         job = source.get_last_job()
@@ -817,7 +817,7 @@ class CswDcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="csw-dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         source.reload()
 
@@ -863,7 +863,7 @@ class CswDcatBackendTest:
         org = OrganizationFactory()
         source = HarvestSourceFactory(backend="csw-dcat", url=url, organization=org)
 
-        actions.run(source.slug)
+        actions.run(source)
 
         assert "User-Agent" in get_mock.last_request.headers
         assert get_mock.last_request.headers["User-Agent"] == "uData/0.1 csw-dcat"
@@ -902,7 +902,7 @@ class CswIso19139DcatBackendTest:
             },
         )
 
-        actions.run(source.slug)
+        actions.run(source)
 
         source.reload()
 

udata/migrations/2025-07-30-purge-old-harvest-dynamic-fields.py

@@ -0,0 +1,29 @@
+"""
+This migration removes legacy harvest dynamic fields
+"""
+
+import logging
+
+from mongoengine.connection import get_db
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    # Remove legacy fields (`ods_has_records`, `ods_url`, ...) from old harvested datasets and resources
+    dataset_legacy_fields = ["ods_has_records", "ods_url", "ods_geo"]
+    for field in dataset_legacy_fields:
+        result = get_db().dataset.update_many({}, {"$unset": {f"harvest.{field}": 1}})
+        log.info(
+            f"Harvest Dataset dynamic legacy fields ({field}) removed from {result.modified_count} objects"
+        )
+
+    resource_legacy_fields = ["ods_type"]
+    for field in resource_legacy_fields:
+        result = get_db().dataset.update_many(
+            {"resources": {"$exists": True, "$type": "array"}},
+            {"$unset": {f"resources.$[].harvest.{field}": 1}},
+        )
+        log.info(
+            f"Harvest Resource dynamic legacy fields ({field}) removed from {result.modified_count} objects"
+        )

udata/mongo/slug_fields.py

@@ -172,21 +172,38 @@ def populate_slug(instance, field):
     # Ensure uniqueness
     if field.unique:
         base_slug = slug
-        index = 1
-        qs = instance.__class__.objects
+        qs = instance.__class__.objects.no_cache()
         if previous:
             qs = qs(id__ne=previous.id)
 
-        def exists(s):
-            return qs(**{field.db_field: s}).clear_cls_query().limit(1).count(True) > 0
+        def exists(slug):
+            return qs(**{field.db_field: slug}).clear_cls_query().limit(1).count(True) > 0
 
-        while exists(slug):
-            # keep space for index suffix, trim slug if needed
+        def get_existing_slug_suffixes(slug):
+            qs_suffix = qs(slug__regex=f"^{slug}-\d*$").clear_cls_query().only(field.db_field)
+            return [getattr(obj, field.db_field) for obj in qs_suffix]
+
+        def trim_base_slug(base_slug, index):
             slug_overflow = len("{0}-{1}".format(base_slug, index)) - field.max_length
             if slug_overflow >= 1:
                 base_slug = base_slug[:-slug_overflow]
-            slug = "{0}-{1}".format(base_slug, index)
-            index += 1
+            return base_slug
+
+        if exists(base_slug):
+            # We'll iterate to get the first free slug suffix
+            index = 1
+            existing_slugs = None
+            while True:
+                # Keep space for index suffix, trim slug if needed
+                trimmed_slug = trim_base_slug(base_slug, index)
+                # Find all existing slugs with suffixes
+                if existing_slugs is None or trimmed_slug != base_slug:
+                    base_slug = trimmed_slug
+                    existing_slugs = set(sorted(get_existing_slug_suffixes(base_slug)))
+                slug = "{0}-{1}".format(base_slug, index)
+                if slug not in existing_slugs:
+                    break
+                index += 1
 
         if is_uuid(slug):
             slug = "{0}-uuid".format(slug)

udata/routing.py

@@ -10,6 +10,7 @@ from werkzeug.urls import url_quote
 from udata import models
 from udata.core.dataservices.models import Dataservice
 from udata.core.spatial.models import GeoZone
+from udata.harvest.models import HarvestSource
 from udata.i18n import ISO_639_1_CODES
 from udata.mongo import db
 from udata.uris import cdata_url, homepage_url
@@ -136,6 +137,10 @@ class DataserviceConverter(ModelConverter):
     model = Dataservice
 
 
+class HarvestSourceConverter(ModelConverter):
+    model = HarvestSource
+
+
 class CommunityResourceConverter(ModelConverter):
     model = models.CommunityResource
 
@@ -239,6 +244,7 @@ def init_app(app):
     app.url_map.converters["dataset"] = DatasetConverter
     app.url_map.converters["dataset_without_resources"] = DatasetWithoutResourcesConverter
     app.url_map.converters["dataservice"] = DataserviceConverter
+    app.url_map.converters["harvest_source"] = HarvestSourceConverter
     app.url_map.converters["crid"] = CommunityResourceConverter
     app.url_map.converters["org"] = OrganizationConverter
     app.url_map.converters["reuse"] = ReuseConverter