udata 10.8.2.dev36980__py2.py3-none-any.whl → 10.8.2.dev37076__py2.py3-none-any.whl

udata/commands/db.py

@@ -2,6 +2,7 @@ import collections
 import copy
 import logging
 import os
+import sys
 import traceback
 from itertools import groupby
 from typing import Optional
@@ -312,15 +313,26 @@ def check_references(models_to_check):
                                     f"\t{model.__name__}#{obj.id} have a broken reference for `{reference['name']}`"
                                 )
                         elif reference["type"] == "list":
-                            attr_list = getattr(obj, reference["name"], [])
-                            for i, sub in enumerate(attr_list):
-                                # If it's still an instance of DBRef it means that it failed to
-                                # dereference the ID.
-                                if isinstance(sub, DBRef):
-                                    errors[model][key] += 1
-                                    print_and_save(
-                                        f"\t{model.__name__}#{obj.id} have a broken reference for {reference['name']}[{i}]"
-                                    )
+                            field_exists = (
+                                f"{reference['name']}__exists"  # Eg: "contact_points__exists"
+                            )
+                            if model.objects(id=obj.id, **{field_exists: True}).count() == 0:
+                                # See https://github.com/MongoEngine/mongoengine/issues/267#issuecomment-283065318
+                                # Setting it explicitely to an empty list actually removes the field, it shouldn't.
+                                errors[model][key] += 1
+                                print_and_save(
+                                    f"\t{model.__name__}#{obj.id} have a non existing field `{reference['name']}`, instead of an empty list"
+                                )
+                            else:
+                                attr_list = getattr(obj, reference["name"])
+                                for i, sub in enumerate(attr_list):
+                                    # If it's still an instance of DBRef it means that it failed to
+                                    # dereference the ID.
+                                    if isinstance(sub, DBRef):
+                                        errors[model][key] += 1
+                                        print_and_save(
+                                            f"\t{model.__name__}#{obj.id} have a broken reference for {reference['name']}[{i}]"
+                                        )
                         elif reference["type"] == "embed_list":
                             p1, p2 = reference["name"].split("__")
                             attr_list = getattr(obj, p1, [])
@@ -380,6 +392,7 @@ def check_references(models_to_check):
                 sentry_sdk.capture_message(f"{total} integrity errors", "fatal")
         except ImportError:
             print("`sentry_sdk` not installed. The errors weren't reported")
+        sys.exit(1)
 
 
 @grp.command()

udata/core/dataset/models.py

@@ -10,7 +10,6 @@ import requests
 from blinker import signal
 from dateutil.parser import parse as parse_dt
 from flask import current_app, url_for
-from mongoengine import DynamicEmbeddedDocument
 from mongoengine import ValidationError as MongoEngineValidationError
 from mongoengine.fields import DateTimeField
 from mongoengine.signals import post_save, pre_init, pre_save
@@ -78,7 +77,7 @@ def get_json_ld_extra(key, value):
     }
 
 
-class HarvestDatasetMetadata(DynamicEmbeddedDocument):
+class HarvestDatasetMetadata(db.EmbeddedDocument):
     backend = db.StringField()
     created_at = db.DateTimeField()
     modified_at = db.DateTimeField()
@@ -91,12 +90,15 @@ class HarvestDatasetMetadata(DynamicEmbeddedDocument):
     dct_identifier = db.StringField()
     archived_at = db.DateTimeField()
     archived = db.StringField()
+    ckan_name = db.StringField()
+    ckan_source = db.StringField()
 
 
-class HarvestResourceMetadata(DynamicEmbeddedDocument):
+class HarvestResourceMetadata(db.EmbeddedDocument):
     created_at = db.DateTimeField()
     modified_at = db.DateTimeField()
     uri = db.StringField()
+    dct_identifier = db.StringField()
 
 
 class Schema(db.EmbeddedDocument):

udata/harvest/tests/ckan/test_ckan_backend.py

@@ -460,7 +460,7 @@ def test_skip_no_resources(source, result):
 def test_ckan_url_is_url(data, result):
     dataset = dataset_for(result)
     assert dataset.harvest.remote_url == data["url"]
-    assert not hasattr(dataset.harvest, "ckan_source")
+    assert dataset.harvest.ckan_source is None
 
 
 @pytest.mark.ckan_data("ckan_url_is_a_string")

udata/migrations/2025-07-30-purge-old-harvest-dynamic-fields.py

@@ -0,0 +1,29 @@
+"""
+This migration removes legacy harvest dynamic fields
+"""
+
+import logging
+
+from mongoengine.connection import get_db
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    # Remove legacy fields (`ods_has_records`, `ods_url`, ...) from old harvested datasets and resources
+    dataset_legacy_fields = ["ods_has_records", "ods_url", "ods_geo"]
+    for field in dataset_legacy_fields:
+        result = get_db().dataset.update_many({}, {"$unset": {f"harvest.{field}": 1}})
+        log.info(
+            f"Harvest Dataset dynamic legacy fields ({field}) removed from {result.modified_count} objects"
+        )
+
+    resource_legacy_fields = ["ods_type"]
+    for field in resource_legacy_fields:
+        result = get_db().dataset.update_many(
+            {"resources": {"$exists": True, "$type": "array"}},
+            {"$unset": {f"resources.$[].harvest.{field}": 1}},
+        )
+        log.info(
+            f"Harvest Resource dynamic legacy fields ({field}) removed from {result.modified_count} objects"
+        )