udata 14.5.1.dev9__py3-none-any.whl → 14.6.1.dev5__py3-none-any.whl

udata/api_fields.py

@@ -79,13 +79,15 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
     user-supplied overrides, setting the readonly flag…), it's easier to have to do this only once at the end of the function.
 
     """
+    from udata.mongo.engine import db
+
     params: dict = {}
     params["required"] = field.required
 
     read_params: dict = {}
     write_params: dict = {}
 
-    constructor: Callable
+    constructor: Callable | None = None
     constructor_read: Callable | None = None
     constructor_write: Callable | None = None
 
@@ -204,13 +206,34 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
         def constructor_write(**kwargs):
             return restx_fields.List(field_write, **kwargs)
 
-    elif isinstance(
-        field, (mongo_fields.GenericReferenceField, mongoengine.fields.GenericLazyReferenceField)
-    ):
+    elif isinstance(field, mongoengine.fields.GenericLazyReferenceField):
 
         def constructor(**kwargs):
             return restx_fields.Nested(lazy_reference, **kwargs)
 
+    elif isinstance(field, mongo_fields.GenericReferenceField):
+        if field.choices:
+            generic_fields = {}
+            for cls in field.choices:
+                cls = db.resolve_model(cls) if isinstance(cls, str) else cls
+                generic_fields[cls.__name__] = convert_db_to_field(
+                    f"{key}.{cls.__name__}",
+                    # Instead of having GenericReferenceField() we'll create fields for each
+                    # of the subclasses with ReferenceField(Organization)…
+                    mongoengine.fields.ReferenceField(cls),
+                    info,
+                )
+
+            def constructor_read(**kwargs):
+                return GenericField({k: v[0].model for k, v in generic_fields.items()}, **kwargs)
+
+            def constructor_write(**kwargs):
+                return GenericField({k: v[1].model for k, v in generic_fields.items()}, **kwargs)
+        else:
+
+            def constructor(**kwargs):
+                return restx_fields.Nested(lazy_reference, **kwargs)
+
     elif isinstance(field, mongo_fields.ReferenceField | mongo_fields.LazyReferenceField):
         # For reference we accept while writing a String representing the ID of the referenced model.
         # For reading, if the user supplied a `nested_fields` (RestX model), we use it to convert

udata/core/badges/tests/test_tasks.py

@@ -1,5 +1,3 @@
-import udata.core.dataservices.tasks  # noqa
-import udata.core.dataset.tasks  # noqa
 from udata.core.badges.tasks import update_badges
 from udata.core.constants import HVD
 from udata.core.dataservices.factories import DataserviceFactory

udata/core/dataservices/apiv2.py

@@ -15,7 +15,7 @@ apiv2.inherit("AccessAudience (read)", AccessAudience.__read_fields__)
 
 ns = apiv2.namespace("dataservices", "Dataservice related operations")
 
-search_parser = DataserviceSearch.as_request_parser()
+search_parser = DataserviceSearch.as_request_parser(store_missing=False)
 
 
 @ns.route("/search/", endpoint="dataservice_search")

udata/core/dataset/models.py

@@ -15,17 +15,19 @@ from mongoengine.fields import DateTimeField
 from mongoengine.signals import post_save, pre_init, pre_save
 from werkzeug.utils import cached_property
 
-from udata.api_fields import field
+from udata.api_fields import field, generate_fields
 from udata.app import cache
 from udata.core import storages
 from udata.core.access_type.constants import AccessType
 from udata.core.access_type.models import WithAccessType, check_only_one_condition_per_role
 from udata.core.activity.models import Auditable
 from udata.core.constants import HVD
+from udata.core.dataset.api_fields import temporal_coverage_fields
 from udata.core.dataset.preview import TabularAPIPreview
 from udata.core.linkable import Linkable
 from udata.core.metrics.helpers import get_stock_metrics
 from udata.core.owned import Owned, OwnedQuerySet
+from udata.core.spatial.api_fields import spatial_coverage_fields
 from udata.frontend.markdown import mdstrip
 from udata.i18n import lazy_gettext as _
 from udata.models import Badge, BadgeMixin, BadgesList, SpatialCoverage, WithMetrics, db
@@ -89,6 +91,7 @@ def get_json_ld_extra(key, value):
     }
 
 
+@generate_fields()
 class HarvestDatasetMetadata(db.EmbeddedDocument):
     backend = db.StringField()
     created_at = db.DateTimeField()
@@ -114,6 +117,7 @@ class HarvestResourceMetadata(db.EmbeddedDocument):
     dct_identifier = db.StringField()
 
 
+@generate_fields()
 class Schema(db.EmbeddedDocument):
     """
     Schema can only be two things right now:
@@ -482,6 +486,7 @@ class ResourceMixin(object):
         return result
 
 
+@generate_fields()
 class Resource(ResourceMixin, WithMetrics, db.EmbeddedDocument):
     """
     Local file, remote file or API provided by the original provider of the
@@ -533,6 +538,7 @@ class DatasetBadgeMixin(BadgeMixin):
     __badges__ = BADGES
 
 
+@generate_fields()
 class Dataset(
     Auditable, WithMetrics, WithAccessType, DatasetBadgeMixin, Owned, Linkable, db.Document
 ):
@@ -560,8 +566,14 @@ class Dataset(
 
     frequency = field(db.EnumField(UpdateFrequency))
     frequency_date = field(db.DateTimeField(verbose_name=_("Future date of update")))
-    temporal_coverage = field(db.EmbeddedDocumentField(db.DateRange))
-    spatial = field(db.EmbeddedDocumentField(SpatialCoverage))
+    temporal_coverage = field(
+        db.EmbeddedDocumentField(db.DateRange),
+        nested_fields=temporal_coverage_fields,
+    )
+    spatial = field(
+        db.EmbeddedDocumentField(SpatialCoverage),
+        nested_fields=spatial_coverage_fields,
+    )
     schema = field(db.EmbeddedDocumentField(Schema))
 
     ext = field(db.MapField(db.GenericEmbeddedDocumentField()), auditable=False)

udata/core/dataset/rdf.py

@@ -5,7 +5,7 @@ This module centralize dataset helpers for RDF/DCAT serialization and parsing
 import calendar
 import json
 import logging
-from datetime import date, datetime
+from datetime import date
 
 from dateutil.parser import parse as parse_dt
 from flask import current_app
@@ -51,7 +51,7 @@ from udata.rdf import (
     themes_from_rdf,
     url_from_rdf,
 )
-from udata.utils import get_by, safe_unicode, to_naive_datetime
+from udata.utils import get_by, safe_harvest_datetime, safe_unicode
 
 from .constants import OGC_SERVICE_FORMATS, UpdateFrequency
 from .models import Checksum, Dataset, License, Resource
@@ -729,12 +729,10 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
         resource.harvest = HarvestResourceMetadata()
     resource.harvest.issued_at = issued_at
 
-    # In the past, we've encountered future `modified_at` during harvesting
-    # do not save it. :FutureHarvestModifiedAt
-    if modified_at and to_naive_datetime(modified_at) > datetime.utcnow():
-        log.warning(f"Future `DCT.modified` date '{modified_at}' in resource")
-    else:
-        resource.harvest.modified_at = modified_at
+    # :FutureHarvestModifiedAt
+    resource.harvest.modified_at = safe_harvest_datetime(
+        modified_at, "DCT.modified (resource)", refuse_future=True
+    )
 
     resource.harvest.dct_identifier = identifier
     resource.harvest.uri = uri
@@ -845,12 +843,10 @@ def dataset_from_rdf(
     dataset.harvest.created_at = created_at
     dataset.harvest.issued_at = issued_at
 
-    # In the past, we've encountered future `modified_at` during harvesting
-    # do not save it. :FutureHarvestModifiedAt
-    if modified_at and to_naive_datetime(modified_at) > datetime.utcnow():
-        log.warning(f"Future `DCT.modified` date '{modified_at}' in dataset")
-    else:
-        dataset.harvest.modified_at = modified_at
+    # :FutureHarvestModifiedAt
+    dataset.harvest.modified_at = safe_harvest_datetime(
+        modified_at, "DCT.modified (dataset)", refuse_future=True
+    )
 
     return dataset
 

udata/core/organization/apiv2.py

@@ -15,7 +15,7 @@ apiv2.inherit("ContactPoint", contact_point_fields)
 
 
 ns = apiv2.namespace("organizations", "Organization related operations")
-search_parser = OrganizationSearch.as_request_parser()
+search_parser = OrganizationSearch.as_request_parser(store_missing=False)
 
 DEFAULT_SORTING = "-created_at"
 

udata/core/organization/models.py

@@ -7,7 +7,7 @@ from flask_babel import LazyString
 from mongoengine.signals import post_save, pre_save
 from werkzeug.utils import cached_property
 
-from udata.api_fields import field
+from udata.api_fields import field, generate_fields
 from udata.core.activity.models import Auditable
 from udata.core.badges.models import Badge, BadgeMixin, BadgesList
 from udata.core.linkable import Linkable
@@ -21,6 +21,7 @@ from udata.uris import cdata_url
 
 from .constants import (
     ASSOCIATION,
+    BIGGEST_LOGO_SIZE,
     CERTIFIED,
     COMPANY,
     DEFAULT_ROLE,
@@ -44,6 +45,7 @@ BADGES: dict[str, LazyString] = {
 }
 
 
+@generate_fields()
 class Team(db.EmbeddedDocument):
     name = db.StringField(required=True)
     slug = db.SlugField(
@@ -54,6 +56,7 @@ class Team(db.EmbeddedDocument):
     members = db.ListField(db.ReferenceField("User"))
 
 
+@generate_fields()
 class Member(db.EmbeddedDocument):
     user = db.ReferenceField("User")
     role = db.StringField(choices=list(ORG_ROLES), default=DEFAULT_ROLE)
@@ -64,6 +67,7 @@ class Member(db.EmbeddedDocument):
         return ORG_ROLES[self.role]
 
 
+@generate_fields()
 class MembershipRequest(db.EmbeddedDocument):
     """
     Pending organization membership requests
@@ -113,18 +117,22 @@ class OrganizationBadge(Badge):
 
 
 class OrganizationBadgeMixin(BadgeMixin):
-    badges = field(BadgesList(OrganizationBadge), **BadgeMixin.default_badges_list_params)
+    badges = field(
+        BadgesList(OrganizationBadge), show_as_ref=True, **BadgeMixin.default_badges_list_params
+    )
     __badges__ = BADGES
 
 
+@generate_fields()
 class Organization(
     Auditable, WithMetrics, OrganizationBadgeMixin, Linkable, db.Datetimed, db.Document
 ):
-    name = field(db.StringField(required=True))
-    acronym = field(db.StringField(max_length=128))
+    name = field(db.StringField(required=True), show_as_ref=True)
+    acronym = field(db.StringField(max_length=128), show_as_ref=True)
     slug = field(
         db.SlugField(max_length=255, required=True, populate_from="name", update=True, follow=True),
         auditable=False,
+        show_as_ref=True,
     )
     description = field(
         db.StringField(required=True),
@@ -138,7 +146,11 @@ class Organization(
             basename=default_image_basename,
             max_size=LOGO_MAX_SIZE,
             thumbnails=LOGO_SIZES,
-        )
+        ),
+        show_as_ref=True,
+        thumbnail_info={
+            "size": BIGGEST_LOGO_SIZE,
+        },
     )
     business_number_id = field(db.StringField(max_length=ORG_BID_SIZE_LIMIT))
 
@@ -265,6 +277,14 @@ class Organization(
                 return request
         return None
 
+    @field(description="Link to the API endpoint for this organization", show_as_ref=True)
+    def uri(self, *args, **kwargs):
+        return self.self_api_url(*args, **kwargs)
+
+    @field(description="Link to the udata web page for this organization", show_as_ref=True)
+    def page(self, *args, **kwargs):
+        return self.self_web_url(*args, **kwargs)
+
     @classmethod
     def get(cls, id_or_slug):
         obj = cls.objects(slug=id_or_slug).first()

udata/core/post/models.py

@@ -4,6 +4,7 @@ from udata.api_fields import field, generate_fields
 from udata.core.dataset.api_fields import dataset_fields
 from udata.core.linkable import Linkable
 from udata.core.storages import default_image_basename, images
+from udata.core.user.api_fields import user_ref_fields
 from udata.i18n import lazy_gettext as _
 from udata.mongo import db
 from udata.uris import cdata_url
@@ -82,6 +83,7 @@ class Post(db.Datetimed, Linkable, db.Document):
 
     owner = field(
         db.ReferenceField("User"),
+        nested_fields=user_ref_fields,
         readonly=True,
         allow_null=True,
         description="The owner user",

udata/core/post/tests/test_api.py

@@ -56,9 +56,13 @@ class PostsAPITest(APITestCase):
 
     def test_post_api_get(self):
         """It should fetch a post from the API"""
-        post = PostFactory()
+        admin = AdminFactory()
+        post = PostFactory(owner=admin)
         response = self.get(url_for("api.post", post=post))
         assert200(response)
+        owner = response.json["owner"]
+        assert isinstance(owner, dict)
+        assert owner["id"] == str(admin.id)
 
     def test_post_api_create(self):
         """It should create a post from the API"""

udata/core/reuse/apiv2.py

@@ -11,7 +11,7 @@ apiv2.inherit("Reuse (read)", Reuse.__read_fields__)
 
 ns = apiv2.namespace("reuses", "Reuse related operations")
 
-search_parser = ReuseSearch.as_request_parser()
+search_parser = ReuseSearch.as_request_parser(store_missing=False)
 
 DEFAULT_SORTING = "-created_at"
 

udata/core/user/models.py

@@ -12,7 +12,7 @@ from flask_security import MongoEngineUserDatastore, RoleMixin, UserMixin
 from mongoengine.signals import post_save, pre_save
 from werkzeug.utils import cached_property
 
-from udata.api_fields import field
+from udata.api_fields import field, generate_fields
 from udata.core import storages
 from udata.core.discussions.models import Discussion
 from udata.core.linkable import Linkable
@@ -23,7 +23,7 @@ from udata.models import Follow, WithMetrics, db
 from udata.uris import cdata_url
 
 from . import mails
-from .constants import AVATAR_SIZES
+from .constants import AVATAR_SIZES, BIGGEST_AVATAR_SIZE
 
 __all__ = ("User", "Role", "datastore")
 
@@ -45,9 +45,12 @@ class UserSettings(db.EmbeddedDocument):
     prefered_language = db.StringField()
 
 
+@generate_fields()
 class User(WithMetrics, UserMixin, Linkable, db.Document):
     slug = field(
-        db.SlugField(max_length=255, required=True, populate_from="fullname"), auditable=False
+        db.SlugField(max_length=255, required=True, populate_from="fullname"),
+        auditable=False,
+        show_as_ref=True,
     )
     email = field(db.StringField(max_length=255, required=True, unique=True))
     password = field(db.StringField())
@@ -55,12 +58,16 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
     fs_uniquifier = field(db.StringField(max_length=64, unique=True, sparse=True))
     roles = field(db.ListField(db.ReferenceField(Role), default=[]))
 
-    first_name = field(db.StringField(max_length=255, required=True))
-    last_name = field(db.StringField(max_length=255, required=True))
+    first_name = field(db.StringField(max_length=255, required=True), show_as_ref=True)
+    last_name = field(db.StringField(max_length=255, required=True), show_as_ref=True)
 
     avatar_url = field(db.URLField())
     avatar = field(
-        db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES)
+        db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES),
+        show_as_ref=True,
+        thumbnail_info={
+            "size": BIGGEST_AVATAR_SIZE,
+        },
     )
     website = field(db.URLField())
     about = field(
@@ -199,6 +206,14 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
         """Return the number of followers of the user."""
         return self.metrics.get("followers", 0)
 
+    @field(description="Link to the API endpoint for this user", show_as_ref=True)
+    def uri(self, *args, **kwargs):
+        return self.self_api_url(*args, **kwargs)
+
+    @field(description="Link to the udata web page for this user", show_as_ref=True)
+    def page(self, *args, **kwargs):
+        return self.self_web_url(*args, **kwargs)
+
     def generate_api_key(self):
         payload = {
             "user": str(self.id),

udata/features/notifications/models.py

@@ -5,6 +5,7 @@ from udata.api_fields import field, generate_fields
 from udata.core.organization.notifications import MembershipRequestNotificationDetails
 from udata.core.user.api_fields import user_ref_fields
 from udata.core.user.models import User
+from udata.features.transfer.notifications import TransferRequestNotificationDetails
 from udata.models import db
 from udata.mongo.datetime_fields import Datetimed
 from udata.mongo.queryset import UDataQuerySet
@@ -51,6 +52,8 @@ class Notification(Datetimed, db.Document):
         filterable={},
     )
     details = field(
-        db.GenericEmbeddedDocumentField(choices=(MembershipRequestNotificationDetails,)),
+        db.GenericEmbeddedDocumentField(
+            choices=(MembershipRequestNotificationDetails, TransferRequestNotificationDetails)
+        ),
         generic=True,
     )

udata/features/transfer/models.py

@@ -1,6 +1,9 @@
 import logging
 from datetime import datetime
 
+from blinker import Signal
+from mongoengine.signals import post_save
+
 from udata.i18n import lazy_gettext as _
 from udata.mongo import db
 
@@ -30,6 +33,8 @@ class Transfer(db.Document):
     responder = db.ReferenceField("User")
     response_comment = db.StringField()
 
+    on_create = Signal()
+
     meta = {
         "indexes": [
             "owner",
@@ -38,3 +43,14 @@ class Transfer(db.Document):
             "status",
         ]
     }
+
+    @classmethod
+    def post_save(cls, sender, document, **kwargs):
+        """Handle post save signal for Transfer documents."""
+        # Only trigger on_create signal on creation, not on every save
+        if kwargs.get("created"):
+            cls.on_create.send(document)
+
+
+# Connect the post_save signal
+post_save.connect(Transfer.post_save, sender=Transfer)

udata/features/transfer/notifications.py

@@ -1,11 +1,85 @@
 import logging
 
+from udata.api_fields import field, generate_fields
+from udata.core.dataservices.models import Dataservice
+from udata.core.dataset.models import Dataset
+from udata.core.organization.models import Organization
+from udata.core.reuse.models import Reuse
+from udata.core.user.models import User
 from udata.features.notifications.actions import notifier
 from udata.models import Transfer
+from udata.mongo import db
 
 log = logging.getLogger(__name__)
 
 
+@generate_fields()
+class TransferRequestNotificationDetails(db.EmbeddedDocument):
+    transfer_owner = field(
+        db.GenericReferenceField(choices=(User, Organization), required=True),
+        readonly=True,
+        auditable=False,
+        allow_null=True,
+        filterable={},
+    )
+    transfer_recipient = field(
+        db.GenericReferenceField(choices=(User, Organization), required=True),
+        readonly=True,
+        auditable=False,
+        allow_null=True,
+        filterable={},
+    )
+    transfer_subject = field(
+        db.GenericReferenceField(choices=(Dataset, Dataservice, Reuse), required=True),
+        readonly=True,
+        auditable=False,
+        allow_null=True,
+        filterable={},
+    )
+
+
+@Transfer.on_create.connect
+def on_transfer_created(transfer, **kwargs):
+    """Create notification when a new transfer request is created"""
+
+    from udata.features.notifications.models import Notification
+
+    recipient = transfer.recipient
+    owner = transfer.owner
+    users = []
+
+    if isinstance(recipient, User):
+        users = [recipient]
+    elif isinstance(recipient, Organization):
+        users = [member.user for member in recipient.members if member.role == "admin"]
+
+    for user in users:
+        try:
+            existing = Notification.objects(
+                user=user,
+                details__transfer_recipient=recipient,
+                details__transfer_owner=owner,
+                details__transfer_subject=transfer.subject,
+            ).first()
+
+            if not existing:
+                notification = Notification(
+                    user=user,
+                    details=TransferRequestNotificationDetails(
+                        transfer_owner=owner,
+                        transfer_recipient=recipient,
+                        transfer_subject=transfer.subject,
+                    ),
+                )
+                notification.created_at = transfer.created
+                notification.save()
+        except Exception as e:
+            log.error(
+                f"Error creating notification for admin user {user.id} "
+                f"and recipient {recipient.id}: {e}"
+            )
+
+
 @notifier("transfer_request")
 def transfer_request_notifications(user):
     """Notify user about pending transfer requests"""

udata/harvest/backends/ckan/harvesters.py

@@ -3,6 +3,8 @@ import logging
 from urllib.parse import urljoin
 from uuid import UUID
 
+from dateutil.parser import ParserError
+
 from udata import uris
 from udata.core.dataset.constants import UpdateFrequency
 from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
@@ -202,10 +204,16 @@ class CkanBackend(BaseBackend):
                         log.debug("frequency value not handled: %s", value)
             # Temporal coverage start
             elif key == "temporal_start":
-                temporal_start = daterange_start(value)
+                try:
+                    temporal_start = daterange_start(value)
+                except ParserError:
+                    log.warning(f"Unparseable temporal_start value: '{value}'")
             # Temporal coverage end
             elif key == "temporal_end":
-                temporal_end = daterange_end(value)
+                try:
+                    temporal_end = daterange_end(value)
+                except ParserError:
+                    log.warning(f"Unparseable temporal_end value: '{value}'")
             else:
                 dataset.extras[extra["key"]] = value
 

udata/migrations/2021-08-17-harvest-integrity.py

@@ -5,6 +5,7 @@ Remove Harvest db integrity problems
 
 import logging
 
+import click
 import mongoengine
 
 from udata.core.jobs.models import PeriodicTask
@@ -16,29 +17,35 @@ log = logging.getLogger(__name__)
 def migrate(db):
     log.info("Processing HarvestJob source references.")
 
-    harvest_jobs = HarvestJob.objects().no_cache().all()
+    harvest_jobs = HarvestJob.objects().no_cache()
+    total = harvest_jobs.count()
     count = 0
-    for harvest_job in harvest_jobs:
-        try:
-            harvest_job.source.id
-        except mongoengine.errors.DoesNotExist:
-            count += 1
-            harvest_job.delete()
+    with click.progressbar(harvest_jobs, length=total, label="Checking sources refs") as jobs:
+        for harvest_job in jobs:
+            try:
+                if harvest_job.source is None:
+                    raise mongoengine.errors.DoesNotExist()
+                harvest_job.source.id
+            except mongoengine.errors.DoesNotExist:
+                count += 1
+                harvest_job.delete()
 
     log.info(f"Completed, removed {count} HarvestJob objects")
 
     log.info("Processing HarvestJob items references.")
 
-    harvest_jobs = HarvestJob.objects.filter(items__0__exists=True).no_cache().all()
+    harvest_jobs = HarvestJob.objects.filter(items__0__exists=True).no_cache()
+    total = harvest_jobs.count()
     count = 0
-    for harvest_job in harvest_jobs:
-        for item in harvest_job.items:
-            try:
-                item.dataset and item.dataset.id
-            except mongoengine.errors.DoesNotExist:
-                count += 1
-                item.dataset = None
-                harvest_job.save()
+    with click.progressbar(harvest_jobs, length=total, label="Checking items refs") as jobs:
+        for harvest_job in jobs:
+            for item in harvest_job.items:
+                try:
+                    item.dataset and item.dataset.id
+                except mongoengine.errors.DoesNotExist:
+                    count += 1
+                    item.dataset = None
+                    harvest_job.save()
 
     log.info(f"Completed, modified {count} HarvestJob objects")