udata 10.4.1.dev35201__py2.py3-none-any.whl → 10.4.2__py2.py3-none-any.whl

udata/__init__.py

@@ -4,5 +4,5 @@
 udata
 """
 
-__version__ = "10.4.1.dev"
+__version__ = "10.4.2"
 __description__ = "Open data portal"

udata/core/activity/__init__.py

@@ -6,6 +6,8 @@ log = logging.getLogger(__name__)
 def init_app(app):
     # Load all core actvitiess
     import udata.core.user.activities  # noqa
+    import udata.core.dataservices.activities  # noqa
     import udata.core.dataset.activities  # noqa
     import udata.core.reuse.activities  # noqa
     import udata.core.organization.activities  # noqa
+    import udata.core.topic.activities  # noqa

udata/core/activity/api.py

@@ -4,6 +4,7 @@ from bson import ObjectId
 from mongoengine.errors import DoesNotExist
 
 from udata.api import API, api, fields
+from udata.auth import current_user
 from udata.core.organization.api_fields import org_ref_fields
 from udata.core.user.api_fields import user_ref_fields
 from udata.models import Activity, db
@@ -46,6 +47,7 @@ activity_fields = api.model(
         "label": fields.String(description="The label of the activity", required=True),
         "key": fields.String(description="The key of the activity", required=True),
         "icon": fields.String(description="The icon of the activity", required=True),
+        "changes": fields.List(fields.String, description="Changed attributes as list"),
         "extras": fields.Raw(description="Extras attributes as key-value pairs"),
     },
 )
@@ -76,7 +78,7 @@ class SiteActivityAPI(API):
     @api.expect(activity_parser)
     @api.marshal_with(activity_page_fields)
     def get(self):
-        """Fetch site activity, optionally filtered by user of org."""
+        """Fetch site activity, optionally filtered by user or org."""
         args = activity_parser.parse_args()
         qs = Activity.objects
 
@@ -95,10 +97,11 @@ class SiteActivityAPI(API):
         qs = qs.order_by("-created_at")
         qs = qs.paginate(args["page"], args["page_size"])
 
-        # Filter out DBRefs
+        # - Filter out DBRefs
         # Always return a result even not complete
         # But log the error (ie. visible in sentry, silent for user)
         # Can happen when someone manually delete an object in DB (ie. without proper purge)
+        # - Filter out private items (except for sysadmin users)
         safe_items = []
         for item in qs.queryset.items:
             try:
@@ -106,6 +109,11 @@ class SiteActivityAPI(API):
             except DoesNotExist as e:
                 log.error(e, exc_info=True)
             else:
+                if hasattr(item.related_to, "private") and (
+                    current_user.is_anonymous or not current_user.sysadmin
+                ):
+                    if item.related_to.private:
+                        continue
                 safe_items.append(item)
         qs.queryset.items = safe_items
 

udata/core/activity/models.py

@@ -3,6 +3,7 @@ from datetime import datetime
 from blinker import Signal
 from mongoengine.signals import post_save
 
+from udata.api_fields import get_fields
 from udata.auth import current_user
 from udata.mongo import db
 
@@ -36,6 +37,7 @@ class Activity(db.Document, metaclass=EmitNewActivityMetaClass):
     organization = db.ReferenceField("Organization")
     related_to = db.ReferenceField(db.DomainModel, required=True)
     created_at = db.DateTimeField(default=datetime.utcnow, required=True)
+    changes = db.ListField(db.StringField())
 
     extras = db.ExtrasField()
 
@@ -65,11 +67,36 @@ class Activity(db.Document, metaclass=EmitNewActivityMetaClass):
         return cls.on_new.connect(func, sender=cls)
 
     @classmethod
-    def emit(cls, related_to, organization=None, extras=None):
+    def emit(cls, related_to, organization=None, changed_fields=None, extras=None):
         new_activity.send(
             cls,
             related_to=related_to,
             actor=current_user._get_current_object(),
             organization=organization,
+            changes=changed_fields,
             extras=extras,
         )
+
+
+class Auditable(object):
+    @classmethod
+    def post_save(cls, sender, document, **kwargs):
+        try:
+            auditable_fields = [
+                key for key, field, info in get_fields(cls) if info.get("auditable", True)
+            ]
+        except Exception:
+            # for backward compatibility, all fields are treated as auditable for classes not using field() function
+            auditable_fields = document._get_changed_fields()
+        changed_fields = [
+            field for field in document._get_changed_fields() if field in auditable_fields
+        ]
+        if "post_save" in kwargs.get("ignores", []):
+            return
+        cls.after_save.send(document)
+        if kwargs.get("created"):
+            cls.on_create.send(document)
+        elif len(changed_fields):
+            cls.on_update.send(document, changed_fields=changed_fields)
+        if getattr(document, "deleted_at", None) or getattr(document, "deleted", None):
+            cls.on_delete.send(document)

udata/core/activity/tasks.py

@@ -1,4 +1,5 @@
 import logging
+from collections.abc import Iterable
 
 from udata.models import Organization, User, db
 from udata.tasks import task
@@ -9,28 +10,36 @@ log = logging.getLogger(__name__)
 
 
 @new_activity.connect
-def delay_activity(cls, related_to, actor, organization=None, extras=None):
+def delay_activity(cls, related_to, actor, organization=None, changes=None, extras=None):
     emit_activity.delay(
         cls.__name__,
         str(actor.id),
         related_to_cls=related_to.__class__.__name__,
         related_to_id=str(related_to.id),
         organization_id=str(organization.id) if organization else None,
+        changes=changes,
         extras=extras,
     )
 
 
 @task
 def emit_activity(
-    classname, actor_id, related_to_cls, related_to_id, organization_id=None, extras=None
+    classname,
+    actor_id,
+    related_to_cls,
+    related_to_id,
+    organization_id=None,
+    changes=None,
+    extras=None,
 ):
     log.debug(
-        "Emit new activity: %s %s %s %s %s %s",
+        "Emit new activity: %s %s %s %s %s %s %s",
         classname,
         actor_id,
         related_to_cls,
         related_to_id,
         organization_id,
+        ", ".join(changes) if changes and isinstance(changes, Iterable) else "",
         extras,
     )
     cls = db.resolve_model(classname)
@@ -40,4 +49,10 @@ def emit_activity(
         organization = Organization.objects.get(pk=organization_id)
     else:
         organization = None
-    cls.objects.create(actor=actor, related_to=related_to, organization=organization, extras=extras)
+    cls.objects.create(
+        actor=actor,
+        related_to=related_to,
+        organization=organization,
+        changes=changes,
+        extras=extras,
+    )

udata/core/dataservices/activities.py

@@ -0,0 +1,53 @@
+from udata.auth import current_user
+from udata.i18n import lazy_gettext as _
+from udata.models import Activity, Dataservice, db
+
+__all__ = (
+    "UserCreatedDataservice",
+    "UserUpdatedDataservice",
+    "UserDeletedDataservice",
+    "DataserviceRelatedActivity",
+)
+
+
+class DataserviceRelatedActivity(object):
+    related_to = db.ReferenceField("Dataservice")
+
+
+class UserCreatedDataservice(DataserviceRelatedActivity, Activity):
+    key = "dataservice:created"
+    icon = "fa fa-plus"
+    badge_type = "success"
+    label = _("created a dataservice")
+
+
+class UserUpdatedDataservice(DataserviceRelatedActivity, Activity):
+    key = "dataservice:updated"
+    icon = "fa fa-pencil"
+    label = _("updated a dataservice")
+
+
+class UserDeletedDataservice(DataserviceRelatedActivity, Activity):
+    key = "dataservice:deleted"
+    icon = "fa fa-remove"
+    badge_type = "error"
+    label = _("deleted a dataservice")
+
+
+@Dataservice.on_create.connect
+def on_user_created_dataservice(dataservice):
+    if current_user and current_user.is_authenticated:
+        UserCreatedDataservice.emit(dataservice, dataservice.organization)
+
+
+@Dataservice.on_update.connect
+def on_user_updated_dataservice(dataservice, **kwargs):
+    changed_fields = kwargs.get("changed_fields", [])
+    if current_user and current_user.is_authenticated:
+        UserUpdatedDataservice.emit(dataservice, dataservice.organization, changed_fields)
+
+
+@Dataservice.on_delete.connect
+def on_user_deleted_dataservice(dataservice):
+    if current_user and current_user.is_authenticated:
+        UserDeletedDataservice.emit(dataservice, dataservice.organization)

udata/core/dataservices/api.py

@@ -1,7 +1,9 @@
 from datetime import datetime
+from typing import List
 
 import mongoengine
 from bson import ObjectId
+from feedgenerator.django.utils.feedgenerator import Atom1Feed
 from flask import make_response, redirect, request, url_for
 from flask_login import current_user
 
@@ -10,6 +12,9 @@ from udata.api_fields import patch
 from udata.core.dataservices.permissions import OwnablePermission
 from udata.core.dataset.models import Dataset
 from udata.core.followers.api import FollowAPI
+from udata.core.site.models import current_site
+from udata.frontend.markdown import md
+from udata.i18n import gettext as _
 from udata.rdf import RDF_EXTENSIONS, graph_response, negociate_content
 
 from .models import Dataservice
@@ -49,6 +54,44 @@ class DataservicesAPI(API):
         return dataservice, 201
 
 
+@ns.route("/recent.atom", endpoint="recent_dataservices_atom_feed")
+class DataservicesAtomFeedAPI(API):
+    @api.doc("recent_dataservices_atom_feed")
+    def get(self):
+        feed = Atom1Feed(
+            _("Latest APIs"), description=None, feed_url=request.url, link=request.url_root
+        )
+
+        dataservices: List[Dataservice] = (
+            Dataservice.objects.visible()
+            .order_by("-created_at_internal")
+            .limit(current_site.feed_size)
+        )
+        for dataservice in dataservices:
+            author_name = None
+            author_uri = None
+            if dataservice.organization:
+                author_name = dataservice.organization.name
+                author_uri = dataservice.organization.external_url
+            elif dataservice.owner:
+                author_name = dataservice.owner.fullname
+                author_uri = dataservice.owner.external_url
+            feed.add_item(
+                dataservice.title,
+                unique_id=dataservice.id,
+                description=dataservice.description,
+                content=md(dataservice.description),
+                author_name=author_name,
+                author_link=author_uri,
+                link=dataservice.url_for(external=True),
+                updateddate=dataservice.metadata_modified_at,
+                pubdate=dataservice.created_at,
+            )
+        response = make_response(feed.writeString("utf-8"))
+        response.headers["Content-Type"] = "application/atom+xml"
+        return response
+
+
 @ns.route("/<dataservice:dataservice>/", endpoint="dataservice")
 class DataserviceAPI(API):
     @api.doc("get_dataservice")

udata/core/dataservices/models.py

@@ -8,6 +8,7 @@ from mongoengine.signals import post_save
 import udata.core.contact_point.api_fields as contact_api_fields
 import udata.core.dataset.api_fields as datasets_api_fields
 from udata.api_fields import field, function_field, generate_fields
+from udata.core.activity.models import Auditable
 from udata.core.dataservices.constants import DATASERVICE_ACCESS_TYPES, DATASERVICE_FORMATS
 from udata.core.dataset.models import Dataset
 from udata.core.metrics.models import WithMetrics
@@ -105,7 +106,7 @@ class HarvestMetadata(db.EmbeddedDocument):
         {"key": "views", "value": "metrics.views"},
     ],
 )
-class Dataservice(WithMetrics, Owned, db.Document):
+class Dataservice(Auditable, WithMetrics, Owned, db.Document):
     meta = {
         "indexes": [
             "$title",
@@ -117,6 +118,11 @@ class Dataservice(WithMetrics, Owned, db.Document):
         "auto_create_index_on_save": True,
     }
 
+    after_save = Signal()
+    on_create = Signal()
+    on_update = Signal()
+    on_delete = Signal()
+
     verbose_name = _("dataservice")
 
     def __str__(self):
@@ -175,7 +181,10 @@ class Dataservice(WithMetrics, Owned, db.Document):
         description="Is the dataservice private to the owner or the organization",
     )
 
-    extras = field(db.ExtrasField())
+    extras = field(
+        db.ExtrasField(),
+        auditable=False,
+    )
 
     contact_points = field(
         db.ListField(
@@ -201,8 +210,9 @@ class Dataservice(WithMetrics, Owned, db.Document):
         ),
         readonly=True,
         sortable="last_modified",
+        auditable=False,
     )
-    deleted_at = field(db.DateTimeField())
+    deleted_at = field(db.DateTimeField(), auditable=False)
     archived_at = field(db.DateTimeField())
 
     datasets = field(
@@ -223,6 +233,7 @@ class Dataservice(WithMetrics, Owned, db.Document):
     harvest = field(
         db.EmbeddedDocumentField(HarvestMetadata),
         readonly=True,
+        auditable=False,
     )
 
     def url_for(self, *args, **kwargs):
@@ -250,26 +261,11 @@ class Dataservice(WithMetrics, Owned, db.Document):
 
     def count_discussions(self):
         self.metrics["discussions"] = Discussion.objects(subject=self, closed=None).count()
-        self.save()
+        self.save(signal_kwargs={"ignores": ["post_save"]})
 
     def count_followers(self):
         self.metrics["followers"] = Follow.objects(until=None).followers(self).count()
-        self.save()
-
-    on_create = Signal()
-    on_update = Signal()
-    on_delete = Signal()
-
-    @classmethod
-    def post_save(cls, sender, document, **kwargs):
-        if "post_save" in kwargs.get("ignores", []):
-            return
-        if kwargs.get("created"):
-            cls.on_create.send(document)
-        else:
-            cls.on_update.send(document)
-        if document.deleted_at:
-            cls.on_delete.send(document)
+        self.save(signal_kwargs={"ignores": ["post_save"]})
 
 
 post_save.connect(Dataservice.post_save, sender=Dataservice)

udata/core/dataset/activities.py

@@ -35,19 +35,66 @@ class UserDeletedDataset(DatasetRelatedActivity, Activity):
     label = _("deleted a dataset")
 
 
+class UserAddedResourceToDataset(DatasetRelatedActivity, Activity):
+    key = "dataset:resource:added"
+    icon = "fa fa-plus"
+    label = _("added a resource to a dataset")
+
+
+class UserUpdatedResource(DatasetRelatedActivity, Activity):
+    key = "dataset:resource:updated"
+    icon = "fa fa-pencil"
+    label = _("updated a resource")
+
+
+class UserRemovedResourceFromDataset(DatasetRelatedActivity, Activity):
+    key = "dataset:resource:deleted"
+    icon = "fa fa-remove"
+    label = _("removed a resource from a dataset")
+
+
+@Dataset.on_resource_added.connect
+def on_user_added_resource_to_dataset(sender, document, **kwargs):
+    if current_user and current_user.is_authenticated:
+        UserAddedResourceToDataset.emit(
+            document, document.organization, None, {"resource_id": str(kwargs["resource_id"])}
+        )
+
+
+@Dataset.on_resource_updated.connect
+def on_user_updated_resource(sender, document, **kwargs):
+    changed_fields = kwargs.get("changed_fields", [])
+    if current_user and current_user.is_authenticated:
+        UserUpdatedResource.emit(
+            document,
+            document.organization,
+            changed_fields,
+            {"resource_id": str(kwargs["resource_id"])},
+        )
+
+
+@Dataset.on_resource_removed.connect
+def on_user_removed_resource_from_dataset(sender, document, **kwargs):
+    if current_user and current_user.is_authenticated:
+        UserRemovedResourceFromDataset.emit(
+            document, document.organization, None, {"resource_id": str(kwargs["resource_id"])}
+        )
+
+
 @Dataset.on_create.connect
 def on_user_created_dataset(dataset):
-    if not dataset.private and current_user and current_user.is_authenticated:
+    if current_user and current_user.is_authenticated:
         UserCreatedDataset.emit(dataset, dataset.organization)
 
 
 @Dataset.on_update.connect
-def on_user_updated_dataset(dataset):
-    if not dataset.private and current_user and current_user.is_authenticated:
-        UserUpdatedDataset.emit(dataset, dataset.organization)
+def on_user_updated_dataset(dataset, **kwargs):
+    changed_fields = kwargs.get("changed_fields", [])
+    if current_user and current_user.is_authenticated:
+        UserUpdatedDataset.emit(dataset, dataset.organization, changed_fields)
 
 
 @Dataset.on_delete.connect
 def on_user_deleted_dataset(dataset):
-    if not dataset.private and current_user and current_user.is_authenticated:
+    if current_user and current_user.is_authenticated:
         UserDeletedDataset.emit(dataset, dataset.organization)

udata/core/dataset/api.py

@@ -20,9 +20,11 @@ These changes might lead to backward compatibility breakage meaning:
 import logging
 import os
 from datetime import datetime
+from typing import List
 
 import mongoengine
 from bson.objectid import ObjectId
+from feedgenerator.django.utils.feedgenerator import Atom1Feed
 from flask import abort, current_app, make_response, redirect, request, url_for
 from flask_restx.inputs import boolean
 from flask_security import current_user
@@ -39,8 +41,11 @@ from udata.core.dataset.models import CHECKSUM_TYPES
 from udata.core.followers.api import FollowAPI
 from udata.core.organization.models import Organization
 from udata.core.reuse.models import Reuse
+from udata.core.site.models import current_site
 from udata.core.storages.api import handle_upload, upload_parser
 from udata.core.topic.models import Topic
+from udata.frontend.markdown import md
+from udata.i18n import gettext as _
 from udata.linkchecker.checker import check_resource
 from udata.rdf import RDF_EXTENSIONS, graph_response, negociate_content
 from udata.utils import get_by
@@ -292,6 +297,45 @@ class DatasetListAPI(API):
         return dataset, 201
 
 
+@ns.route("/recent.atom", endpoint="recent_datasets_atom_feed")
+class DatasetsAtomFeedAPI(API):
+    @api.doc("recent_datasets_atom_feed")
+    def get(self):
+        feed = Atom1Feed(
+            _("Latest datasets"),
+            description=None,
+            feed_url=request.url,
+            link=request.url_root,
+        )
+
+        datasets: List[Dataset] = (
+            Dataset.objects.visible().order_by("-created_at_internal").limit(current_site.feed_size)
+        )
+        for dataset in datasets:
+            author_name = None
+            author_uri = None
+            if dataset.organization:
+                author_name = dataset.organization.name
+                author_uri = dataset.organization.external_url
+            elif dataset.owner:
+                author_name = dataset.owner.fullname
+                author_uri = dataset.owner.external_url
+            feed.add_item(
+                dataset.title,
+                unique_id=dataset.id,
+                description=dataset.description,
+                content=md(dataset.description),
+                author_name=author_name,
+                author_link=author_uri,
+                link=dataset.external_url,
+                updateddate=dataset.last_modified,
+                pubdate=dataset.created_at,
+            )
+        response = make_response(feed.writeString("utf-8"))
+        response.headers["Content-Type"] = "application/atom+xml"
+        return response
+
+
 @ns.route("/<dataset:dataset>/", endpoint="dataset", doc=common_doc)
 @api.response(404, "Dataset not found")
 @api.response(410, "Dataset has been deleted")

udata/core/dataset/csv.py

@@ -40,7 +40,6 @@ class DatasetCsvAdapter(csv.Adapter):
         ("resources_count", lambda o: len(o.resources)),
         ("main_resources_count", lambda o: len([r for r in o.resources if r.type == "main"])),
         ("resources_formats", lambda o: ",".join(set(r.format for r in o.resources if r.format))),
-        "downloads",
         ("harvest.backend", lambda r: r.harvest and r.harvest.backend),
         ("harvest.domain", lambda r: r.harvest and r.harvest.domain),
         ("harvest.created_at", lambda r: r.harvest and r.harvest.created_at),