udata 10.8.2.dev37001__py2.py3-none-any.whl → 10.8.2.dev37076__py2.py3-none-any.whl

udata/core/dataset/models.py

@@ -10,7 +10,6 @@ import requests
 from blinker import signal
 from dateutil.parser import parse as parse_dt
 from flask import current_app, url_for
-from mongoengine import DynamicEmbeddedDocument
 from mongoengine import ValidationError as MongoEngineValidationError
 from mongoengine.fields import DateTimeField
 from mongoengine.signals import post_save, pre_init, pre_save
@@ -78,7 +77,7 @@ def get_json_ld_extra(key, value):
     }
 
 
-class HarvestDatasetMetadata(DynamicEmbeddedDocument):
+class HarvestDatasetMetadata(db.EmbeddedDocument):
     backend = db.StringField()
     created_at = db.DateTimeField()
     modified_at = db.DateTimeField()
@@ -91,12 +90,15 @@ class HarvestDatasetMetadata(DynamicEmbeddedDocument):
     dct_identifier = db.StringField()
     archived_at = db.DateTimeField()
     archived = db.StringField()
+    ckan_name = db.StringField()
+    ckan_source = db.StringField()
 
 
-class HarvestResourceMetadata(DynamicEmbeddedDocument):
+class HarvestResourceMetadata(db.EmbeddedDocument):
     created_at = db.DateTimeField()
     modified_at = db.DateTimeField()
     uri = db.StringField()
+    dct_identifier = db.StringField()
 
 
 class Schema(db.EmbeddedDocument):

udata/harvest/tests/ckan/test_ckan_backend.py

@@ -460,7 +460,7 @@ def test_skip_no_resources(source, result):
 def test_ckan_url_is_url(data, result):
     dataset = dataset_for(result)
     assert dataset.harvest.remote_url == data["url"]
-    assert not hasattr(dataset.harvest, "ckan_source")
+    assert dataset.harvest.ckan_source is None
 
 
 @pytest.mark.ckan_data("ckan_url_is_a_string")

udata/migrations/2025-07-30-purge-old-harvest-dynamic-fields.py

@@ -0,0 +1,29 @@
+"""
+This migration removes legacy harvest dynamic fields
+"""
+
+import logging
+
+from mongoengine.connection import get_db
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    # Remove legacy fields (`ods_has_records`, `ods_url`, ...) from old harvested datasets and resources
+    dataset_legacy_fields = ["ods_has_records", "ods_url", "ods_geo"]
+    for field in dataset_legacy_fields:
+        result = get_db().dataset.update_many({}, {"$unset": {f"harvest.{field}": 1}})
+        log.info(
+            f"Harvest Dataset dynamic legacy fields ({field}) removed from {result.modified_count} objects"
+        )
+
+    resource_legacy_fields = ["ods_type"]
+    for field in resource_legacy_fields:
+        result = get_db().dataset.update_many(
+            {"resources": {"$exists": True, "$type": "array"}},
+            {"$unset": {f"resources.$[].harvest.{field}": 1}},
+        )
+        log.info(
+            f"Harvest Resource dynamic legacy fields ({field}) removed from {result.modified_count} objects"
+        )

udata/tests/api/test_datasets_api.py

@@ -15,10 +15,6 @@ from udata.api import fields
 from udata.app import cache
 from udata.core import storages
 from udata.core.badges.factories import badge_factory
-from udata.core.dataset.api_fields import (
-    dataset_harvest_fields,
-    resource_harvest_fields,
-)
 from udata.core.dataset.constants import (
     DEFAULT_FREQUENCY,
     DEFAULT_LICENSE,
@@ -2391,10 +2387,6 @@ class DatasetSchemasAPITest:
 class HarvestMetadataAPITest:
     modules = []
 
-    # api fields should be updated before app is created
-    dataset_harvest_fields["dynamic_field"] = fields.String(description="", allow_null=True)
-    resource_harvest_fields["dynamic_field"] = fields.String(description="", allow_null=True)
-
     def test_dataset_with_harvest_metadata(self, api):
         date = datetime(2022, 2, 22, tzinfo=pytz.UTC)
         harvest_metadata = HarvestDatasetMetadata(
@@ -2430,24 +2422,6 @@ class HarvestMetadataAPITest:
             "archived": "not-on-remote",
         }
 
-    def test_dataset_dynamic_harvest_metadata_without_api_field(self, api):
-        harvest_metadata = HarvestDatasetMetadata(dynamic_field_but_no_api_field_defined="DCAT")
-        dataset = DatasetFactory(harvest=harvest_metadata)
-
-        response = api.get(url_for("api.dataset", dataset=dataset))
-        assert200(response)
-        assert response.json["harvest"] == {}
-
-    def test_dataset_dynamic_harvest_metadata_with_api_field(self, api):
-        harvest_metadata = HarvestDatasetMetadata(dynamic_field="dynamic_value")
-        dataset = DatasetFactory(harvest=harvest_metadata)
-
-        response = api.get(url_for("api.dataset", dataset=dataset))
-        assert200(response)
-        assert response.json["harvest"] == {
-            "dynamic_field": "dynamic_value",
-        }
-
     def test_dataset_with_resource_harvest_metadata(self, api):
         date = datetime(2022, 2, 22, tzinfo=pytz.UTC)
 
@@ -2466,26 +2440,6 @@ class HarvestMetadataAPITest:
             "uri": "http://domain.gouv.fr/dataset/uri",
         }
 
-    def test_resource_dynamic_harvest_metadata_without_api_field(self, api):
-        harvest_metadata = HarvestResourceMetadata(
-            dynamic_field_but_no_api_field_defined="dynamic_value"
-        )
-        dataset = DatasetFactory(resources=[ResourceFactory(harvest=harvest_metadata)])
-
-        response = api.get(url_for("api.dataset", dataset=dataset))
-        assert200(response)
-        assert response.json["resources"][0]["harvest"] == {}
-
-    def test_resource_dynamic_harvest_metadata_with_api_field(self, api):
-        harvest_metadata = HarvestResourceMetadata(dynamic_field="dynamic_value")
-        dataset = DatasetFactory(resources=[ResourceFactory(harvest=harvest_metadata)])
-
-        response = api.get(url_for("api.dataset", dataset=dataset))
-        assert200(response)
-        assert response.json["resources"][0]["harvest"] == {
-            "dynamic_field": "dynamic_value",
-        }
-
     def test_dataset_with_harvest_computed_dates(self, api):
         creation_date = datetime(2022, 2, 22, tzinfo=pytz.UTC)
         modification_date = datetime(2022, 3, 19, tzinfo=pytz.UTC)

udata/tests/dataset/test_dataset_model.py

@@ -789,14 +789,6 @@ class HarvestMetadataTest:
         with pytest.raises(db.ValidationError):
             dataset.save()
 
-    def test_harvest_dataset_metadata_no_validation_dynamic(self):
-        # Adding a dynamic field (not defined in HarvestDatasetMetadata) does not raise error
-        # at validation time
-        harvest_metadata = HarvestDatasetMetadata(dynamic_created_at="maintenant")
-        dataset = DatasetFactory()
-        dataset.harvest = harvest_metadata
-        dataset.save()
-
     def test_harvest_dataset_metadata_past_modifed_at(self):
         dataset = DatasetFactory()
 
@@ -826,14 +818,6 @@ class HarvestMetadataTest:
         with pytest.raises(db.ValidationError):
             resource.validate()
 
-    def test_harvest_resource_metadata_no_validation_dynamic(self):
-        # Adding a dynamic field (not defined in HarvestResourceMetadata) does not raise error
-        # at validation time
-        harvest_metadata = HarvestResourceMetadata(dynamic_created_at="maintenant")
-        resource = ResourceFactory()
-        resource.harvest = harvest_metadata
-        resource.validate()
-
     def test_harvest_resource_metadata_future_modifed_at(self):
         resource = ResourceFactory()
         harvest_metadata = HarvestResourceMetadata(

{udata-10.8.2.dev37001.dist-info → udata-10.8.2.dev37076.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata
-Version: 10.8.2.dev37001
+Version: 10.8.2.dev37076
 Summary: Open data portal
 Home-page: https://github.com/opendatateam/udata
 Author: Opendata Team
@@ -151,6 +151,7 @@ It is collectively taken care of by members of the
 - Fix not returning 404 for harvest sources [#3382](https://github.com/opendatateam/udata/pull/3382/)
 - Add missing trailing slashes on some endpoints [#3392](https://github.com/opendatateam/udata/pull/3392/)
 - Make organization name filter case insensitive [#3392](https://github.com/opendatateam/udata/pull/3395)
+- Harvest metadata documents are no longer dynamic [#3396](https://github.com/opendatateam/udata/pull/3396)
 
 ## 10.8.1 (2025-07-25)
 

{udata-10.8.2.dev37001.dist-info → udata-10.8.2.dev37076.dist-info}/RECORD

@@ -106,7 +106,7 @@ udata/core/dataset/events.py,sha256=bSM0nFEX14r4JHc-bAM-7OOuD3JAxUIpw9GgXbOsUyw,
 udata/core/dataset/exceptions.py,sha256=uKiayLSpSzsnLvClObS6hOO0qXEqvURKN7_w8eimQNU,498
 udata/core/dataset/factories.py,sha256=fRDWDlybR_ud4pDs1-ntWuYHKtV9LMHeBOBp2SmTT6M,9006
 udata/core/dataset/forms.py,sha256=7KUxuFcEGT0MUe0cZCiZtsnZhvGgvEd68pe13NgeSMI,6292
-udata/core/dataset/models.py,sha256=kkzcuQAFZYT8MHFAGPsDm8atJIDJUaoYuP5_Ed_i5EM,43527
+udata/core/dataset/models.py,sha256=JBIORJcJf1UamIBCpBplKU2L_QjvzmG0FZ3XuhuI3Xw,43577
 udata/core/dataset/permissions.py,sha256=zXQ6kU-Ni3Pl5tDtat-ZPupug9InsNeCN7xRLc2Vcrc,1097
 udata/core/dataset/preview.py,sha256=qXmd4-nKvIPtcKLrMqC-aJne4-vMIiq_NubpX39D6I0,855
 udata/core/dataset/rdf.py,sha256=PFSzkpTFRdpa4cF-mxYU7IV1cJ73m80MmX-GZfWk3hg,30845
@@ -304,7 +304,6 @@ udata/harvest/backends/maaf.py,sha256=N7ty8ZWO9pfKPtZRk1wTaJ5pY6qi-0-GtF1p8jiYiY
 udata/harvest/backends/maaf.xsd,sha256=vEyG8Vqw7Yn_acjRdXjqUJgxOj4pv8bibep-FX-f3BQ,18322
 udata/harvest/backends/ckan/__init__.py,sha256=JE7Qa7kX7Yd8OvmJnAO_NupZe0tqYyhhkgJ-iGNxX64,35
 udata/harvest/backends/ckan/harvesters.py,sha256=WCnO80FJQqve3AaJgRs1tXrlcqmQwZBkTJy3P5xNkjo,10681
-udata/harvest/backends/ckan/models.py,sha256=r2cZKMXOkSfq0s0aHOrpB6EuHO0_7uQLYMj773RVxyA,432
 udata/harvest/backends/ckan/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/harvest/backends/ckan/schemas/ckan.py,sha256=59VQoIxPBb5vOHFzzMtdnPcxHL4qNOTkPYa4cqn-7xQ,2353
 udata/harvest/backends/ckan/schemas/dkan.py,sha256=RyQGPEaKqnunIrr9yApcz7WEfgye0IT1bV85wa2E8Lc,2774
@@ -320,7 +319,7 @@ udata/harvest/tests/test_models.py,sha256=f9NRR2_S4oZFgF8qOumg0vv-lpnEBJbI5vNtcw
 udata/harvest/tests/test_notifications.py,sha256=MMzTzkv-GXMNFeOwAi31rdTsAXyLCLOSna41zOtaJG0,816
 udata/harvest/tests/test_tasks.py,sha256=7wQGhaX7xJEPKGAF7dprIDj2_7Z2tTInzvAd5GiqaN8,681
 udata/harvest/tests/ckan/conftest.py,sha256=SNIRJ7uh6ssIB56mwIdDgteHKUY040yX4gzaGMREjJw,1893
-udata/harvest/tests/ckan/test_ckan_backend.py,sha256=1SNuIgzgLljirEc9TB-5jYxq52GHQy77EsQEkF2t5LE,21447
+udata/harvest/tests/ckan/test_ckan_backend.py,sha256=SiuUTquQ-MFYjGIJTJgry88-QIxQIb68sUvoox7l7VY,21439
 udata/harvest/tests/ckan/test_ckan_backend_errors.py,sha256=LbT-jFz9Fs_WECcQxpuOEEBm2FZ9PdqSbGYxMb1X32c,3847
 udata/harvest/tests/ckan/test_ckan_backend_filters.py,sha256=_cBXkHihCTvL1jsrH6ppfU6KmyfErXVqGplHVgIHMnU,3698
 udata/harvest/tests/ckan/test_dkan_backend.py,sha256=9Ym7bFLMj6bqbtdGCpDYm7nSwswbYQM3ksljfpi8z_g,2178
@@ -387,6 +386,7 @@ udata/migrations/2025-06-18-clean-spatial-coverages.py,sha256=aKg9HQwCmHDJSXAGD0
 udata/migrations/2025-07-18-compute-last-update.py,sha256=K3wSSl9oC8TRbHlvgKeekgPB9fMMVgll1NXtxGUiNyI,680
 udata/migrations/2025-07-18-redo-count-discussion.py,sha256=BOFTjf0zq_ahnVlYQC-d8fuoLixxyk2vP6JYJXhmvbs,941
 udata/migrations/2025-07-22-fix-duplicate-contact-points.py,sha256=ZEw2PX1E3jJiDNWLj6Y52zBAIHS-kZSLjBP6gUmA_ho,1789
+udata/migrations/2025-07-30-purge-old-harvest-dynamic-fields.py,sha256=ijeu6WvX6etOJrv0cEMCRJzj1pSetBguF-aUB1-2y_0,1017
 udata/migrations/__init__.py,sha256=RBCBDaTlLjuMs_Qzwji6Z6T4r7FCGXhESKoxQbT5qAA,11221
 udata/models/__init__.py,sha256=txbZwa-lRG3mq99eQ9E5YcFWiNUdjDVSyJJvlqUMFfs,1413
 udata/mongo/__init__.py,sha256=y4Rv-kq3o_kcEulcNpePLzocXPBNpx3Jd82G-VZPaMc,1421
@@ -650,7 +650,7 @@ udata/tests/api/test_auth_api.py,sha256=OMRlY0OQt60j5N4A-N3HdWTuffOjRlFHkz5a3jJF
 udata/tests/api/test_base_api.py,sha256=2w_vz0eEuq3P3aN-ByvxGc3VZAo7XtgatFfcrzf2uEU,2244
 udata/tests/api/test_contact_points.py,sha256=Sbb486RTN7HVycna_XB60OnURPSNc7xUity26XsYA4k,8766
 udata/tests/api/test_dataservices_api.py,sha256=PGl-817qFAHXmIjb3qzpFzp7sVXSq8gwEzG-g01Qnn8,31025
-udata/tests/api/test_datasets_api.py,sha256=R15D11LSNPZJx97UhYiGpSkgIUCz_zpSP9dhcgZW3zI,103167
+udata/tests/api/test_datasets_api.py,sha256=R4xX1wNTCwA97XUrVHgPwdPPRXICVCrzSknW9PU1V68,101118
 udata/tests/api/test_fields.py,sha256=OW85Z5MES5HeWOpapeem8OvR1cIcrqW-xMWpdZO4LZ8,1033
 udata/tests/api/test_follow_api.py,sha256=4nFXG5pZ_Hf2PJ4KEdHJX_uggjc9RpB8v0fidkAcw9I,5792
 udata/tests/api/test_me_api.py,sha256=YPd8zmR3zwJKtpSqz8nY1nOOMyXs66INeBwyhg5D0Us,13846
@@ -681,7 +681,7 @@ udata/tests/dataset/test_csv_adapter.py,sha256=7U83RiuSKNVgX-mV45Txp9a828mnWE4eK
 udata/tests/dataset/test_dataset_actions.py,sha256=bgDjVYjOvu3sX_FCTCzf2snZYSprsqor2nAhIVuokSs,722
 udata/tests/dataset/test_dataset_commands.py,sha256=zMPJG2wYwKBee2zI65kmboxf59Zqa84DDjT8V5wj9uo,801
 udata/tests/dataset/test_dataset_events.py,sha256=hlrpoOiBbnX_COUI9Pzdqlp45GZZDqu5piwupbnPiTI,3601
-udata/tests/dataset/test_dataset_model.py,sha256=av4RhOnT-52qs-WL1NCn6R2SMa_fzDcoV-06vjccsdw,33891
+udata/tests/dataset/test_dataset_model.py,sha256=5NYgIcl_7w4KJqWMwtNDb4OTXsjIA_gzECXmEoxXj84,33130
 udata/tests/dataset/test_dataset_rdf.py,sha256=j919psNAH0scn0DIwHNlqiZGFKd_lKOlyTisICNGeOE,44327
 udata/tests/dataset/test_dataset_recommendations.py,sha256=K52HXGXi9DuUSiSRQWpqTrsFCkYbv7K3upNp0fFp5v8,7068
 udata/tests/dataset/test_dataset_tasks.py,sha256=n1W2Pg0ez02d66zQG3N93kh7dpR2yLMRDqUI6PnPaI0,3088
@@ -757,9 +757,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=ogtEGsjwIkOEtzrI49e57izi-NUim5
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=1omSlqdMEdwYtzHGalXrtptVk14mlb6JS_8xR8BsznU,48428
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=V7unDjO31PNLiRhfEkYU3pU47wWWct-otdL0GDma0S4,28692
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=wVBipfpPl-DmYeVm5wrlzWWM3oEvuruDVbJHx4I1p8s,55170
-udata-10.8.2.dev37001.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-10.8.2.dev37001.dist-info/METADATA,sha256=kFNi1RaBP0RrU3VtSjl25Gee1M5-8Hz8Vkd1wrtad3k,152929
-udata-10.8.2.dev37001.dist-info/WHEEL,sha256=Kh9pAotZVRFj97E15yTA4iADqXdQfIVTHcNaZTjxeGM,110
-udata-10.8.2.dev37001.dist-info/entry_points.txt,sha256=v2u12qO11i2lyLNIp136WmLJ-NHT-Kew3Duu8J-AXPM,614
-udata-10.8.2.dev37001.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
-udata-10.8.2.dev37001.dist-info/RECORD,,
+udata-10.8.2.dev37076.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-10.8.2.dev37076.dist-info/METADATA,sha256=cDKSCXXtTtNalqZpf-abrvdlCtpSnkSp647y2x3dO3I,153037
+udata-10.8.2.dev37076.dist-info/WHEEL,sha256=Kh9pAotZVRFj97E15yTA4iADqXdQfIVTHcNaZTjxeGM,110
+udata-10.8.2.dev37076.dist-info/entry_points.txt,sha256=v2u12qO11i2lyLNIp136WmLJ-NHT-Kew3Duu8J-AXPM,614
+udata-10.8.2.dev37076.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
+udata-10.8.2.dev37076.dist-info/RECORD,,

udata/harvest/backends/ckan/models.py

@@ -1,10 +0,0 @@
-from udata.api import fields
-from udata.core.dataset.api_fields import dataset_harvest_fields
-
-# Register additional harvest fields to serve by api
-dataset_harvest_fields["ckan_name"] = fields.String(
-    description="The ckan name property for ckan harvested dataset", allow_null=True
-)
-dataset_harvest_fields["ckan_source"] = fields.String(
-    description="The ckan source property for ckan harvested dataset", allow_null=True
-)