udata 10.4.3.dev35680__py2.py3-none-any.whl → 10.4.3.dev35794__py2.py3-none-any.whl

udata/core/dataset/api.py

@@ -503,8 +503,6 @@ class ResourcesAPI(API):
             api.abort(400, "This endpoint only supports remote resources")
         form.populate_obj(resource)
         dataset.add_resource(resource)
-        dataset.last_modified_internal = datetime.utcnow()
-        dataset.save()
         return resource, 201
 
     @api.secure
@@ -571,8 +569,6 @@ class UploadNewDatasetResource(UploadMixin, API):
         infos = self.handle_upload(dataset)
         resource = Resource(**infos)
         dataset.add_resource(resource)
-        dataset.last_modified_internal = datetime.utcnow()
-        dataset.save()
         return resource, 201
 
 
@@ -625,8 +621,6 @@ class UploadDatasetResource(ResourceMixin, UploadMixin, API):
         for k, v in infos.items():
             resource[k] = v
         dataset.update_resource(resource)
-        dataset.last_modified_internal = datetime.utcnow()
-        dataset.save()
         if fs_filename_to_remove is not None:
             storages.resources.delete(fs_filename_to_remove)
         return resource
@@ -699,8 +693,6 @@ class ResourceAPI(ResourceMixin, API):
         form.populate_obj(resource)
         resource.last_modified_internal = datetime.utcnow()
         dataset.update_resource(resource)
-        dataset.last_modified_internal = datetime.utcnow()
-        dataset.save()
         return resource
 
     @api.secure
@@ -710,8 +702,6 @@ class ResourceAPI(ResourceMixin, API):
         dataset.permissions["edit_resources"].test()
         resource = self.get_resource_or_404(dataset, rid)
         dataset.remove_resource(resource)
-        dataset.last_modified_internal = datetime.utcnow()
-        dataset.save()
         return "", 204
 
 

udata/core/dataset/models.py

@@ -956,35 +956,54 @@ class Dataset(Auditable, WithMetrics, DatasetBadgeMixin, Owned, db.Document):
         obj = cls.objects(slug=id_or_slug).first()
         return obj or cls.objects.get_or_404(id=id_or_slug)
 
-    def add_resource(self, resource):
+    def add_resource(self, resource: Resource):
         """Perform an atomic prepend for a new resource"""
         resource.validate()
-        if resource.id in [r.id for r in self.resources]:
-            raise MongoEngineValidationError("Cannot add resource with already existing ID")
 
+        existing_resource = next((r for r in self.resources if r.id == resource.id), None)
+        if existing_resource:
+            raise MongoEngineValidationError(
+                f"Cannot add resource '{resource.title}'. A resource '{existing_resource.title}' already exists with ID '{existing_resource.id}'"
+            )
+
+        # only useful for compute_quality(), we will reload to have a clean object
         self.resources.insert(0, resource)
+
         self.update(
-            __raw__={
-                "$set": {
-                    "quality_cached": self.compute_quality(),
-                },
-                "$push": {
-                    "resources": {"$each": [resource.to_mongo()], "$position": 0},
-                },
-            }
+            set__quality_cached=self.compute_quality(),
+            push__resources={"$each": [resource.to_mongo()], "$position": 0},
+            set__last_modified_internal=datetime.utcnow(),
         )
+
         self.reload()
         self.on_resource_added.send(self.__class__, document=self, resource_id=resource.id)
 
     def update_resource(self, resource):
         """Perform an atomic update for an existing resource"""
-        index = self.resources.index(resource)
-        data = {"resources__{index}".format(index=index): resource}
-        self.update(**data)
+
+        # only useful for compute_quality(), we will reload to have a clean object
+        index = next(i for i, r in enumerate(self.resources) if r.id == resource.id)
+        self.resources[index] = resource
+
+        Dataset.objects(id=self.id, resources__id=resource.id).update_one(
+            set__quality_cached=self.compute_quality(),
+            set__resources__S=resource,
+            set__last_modified_internal=datetime.utcnow(),
+        )
+
         self.reload()
         self.on_resource_updated.send(self.__class__, document=self, resource_id=resource.id)
 
     def remove_resource(self, resource):
+        # only useful for compute_quality(), we will reload to have a clean object
+        self.resources = [r for r in self.resources if r.id != resource.id]
+
+        self.update(
+            set__quality_cached=self.compute_quality(),
+            pull__resources__id=resource.id,
+            set__last_modified_internal=datetime.utcnow(),
+        )
+
         # Deletes resource's file from file storage
         if resource.fs_filename is not None:
             try:
@@ -994,7 +1013,7 @@ class Dataset(Auditable, WithMetrics, DatasetBadgeMixin, Owned, db.Document):
                     f"File not found while deleting resource #{resource.id} in dataset {self.id}: {e}"
                 )
 
-        self.resources.remove(resource)
+        self.reload()
         self.on_resource_removed.send(self.__class__, document=self, resource_id=resource.id)
 
     @property

udata/core/dataset/rdf.py

@@ -536,7 +536,7 @@ def spatial_from_rdf(graph):
         spatial_coverage.clean()
         return spatial_coverage
     except ValidationError as e:
-        log.warning(f"Cannot save the spatial coverage {coordinates} (error was {e})")
+        log.warning(f"Cannot save the spatial coverage {polygons} (error was {e})")
         return None
 
 

udata/core/dataset/tasks.py

@@ -210,8 +210,9 @@ def export_csv_for_model(model, dataset):
         # add it to the dataset
         if created:
             dataset.add_resource(resource)
-        dataset.last_modified_internal = datetime.utcnow()
-        dataset.save()
+        else:
+            dataset.last_modified_internal = datetime.utcnow()
+            dataset.save()
     finally:
         csvfile.close()
         os.unlink(csvfile.name)

udata/core/spatial/models.py

@@ -1,3 +1,4 @@
+import geojson
 from flask import current_app
 from werkzeug.local import LocalProxy
 from werkzeug.utils import cached_property
@@ -174,3 +175,11 @@ class SpatialCoverage(db.EmbeddedDocument):
             raise db.ValidationError(
                 "The spatial coverage cannot contains a Geozone and a Geometry"
             )
+
+        if self.geom:
+            try:
+                geojson.loads(geojson.dumps(self.geom))
+            except (ValueError, TypeError) as err:
+                raise db.ValidationError(
+                    f"Invalid GeoJSON data `{self.geom}`: {err}.", field_name="geom"
+                )

udata/core/user/models.py

@@ -1,4 +1,5 @@
 import json
+import logging
 from copy import copy
 from datetime import datetime
 from itertools import chain
@@ -26,6 +27,8 @@ from .constants import AVATAR_SIZES
 
 __all__ = ("User", "Role", "datastore")
 
+log = logging.getLogger(__name__)
+
 
 # TODO: use simple text for role
 class Role(db.Document, RoleMixin):
@@ -250,11 +253,14 @@ class User(WithMetrics, UserMixin, db.Document):
 
     def mark_as_deleted(self, notify: bool = True, delete_comments: bool = False):
         if self.avatar.filename is not None:
-            storage = storages.avatars
-            storage.delete(self.avatar.filename)
-            storage.delete(self.avatar.original)
-            for key, value in self.avatar.thumbnails.items():
-                storage.delete(value)
+            try:
+                storage = storages.avatars
+                storage.delete(self.avatar.filename)
+                storage.delete(self.avatar.original)
+                for key, value in self.avatar.thumbnails.items():
+                    storage.delete(value)
+            except FileNotFoundError as e:
+                log.error(f"File not found while deleting user #{self.id} avatar: {e}")
 
         copied_user = copy(self)
         self.email = "{}@deleted".format(self.id)

udata/harvest/tests/dcat/bnodes.xml

@@ -35,6 +35,11 @@
             <schema:endDate rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-05T00:00:00</schema:endDate>
           </dcterms:PeriodOfTime>
         </dct:temporal>
+        <dct:spatial>
+            <ogc:Polygon>
+                <locn:geometry rdf:datatype="https://www.iana.org/assignments/media-types/application/vnd.geo+json"><![CDATA[{"type":"Polygon","coordinates":[[[4, 45], [4, NaN], [4, 46], [4, 45], [4, 45]]]}]]></locn:geometry>
+            </ogc:Polygon>
+        </dct:spatial>
       </dcat:Dataset>
     </dcat:dataset>
     <dcterms:title>Sample DCAT Catalog</dcterms:title>

udata/harvest/tests/test_dcat_backend.py

@@ -334,6 +334,7 @@ class DcatBackendTest:
                 [[[159, -25.0], [159, -11], [212, -11], [212, -25.0], [159, -25.0]]],
             ],
         }
+        # dataset-3 has a spatial with NaN values…
         assert datasets["3"].spatial is None
 
     @pytest.mark.options(SCHEMA_CATALOG_URL="https://example.com/schemas")

udata/migrations/2025-06-18-clean-spatial-coverages.py

@@ -0,0 +1,25 @@
+"""
+This migration updates Topic.featured to False when it is None.
+"""
+
+import logging
+
+import click
+
+from udata.core.dataset.models import Dataset
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    datasets = Dataset.objects(spatial__geom__exists=True)
+    count = Dataset.objects(spatial__geom__exists=True).count()
+
+    with click.progressbar(datasets, length=count) as datasets:
+        for dataset in datasets:
+            try:
+                dataset.spatial.clean()
+            except Exception as err:
+                log.error(f"Invalid spatial in dataset #{dataset.id} '{dataset.title}' {err}")
+                dataset.spatial = None
+                dataset.save()