udata 10.8.2.dev37076__py2.py3-none-any.whl → 10.8.3__py2.py3-none-any.whl

udata/__init__.py

@@ -4,5 +4,5 @@
 udata
 """
 
-__version__ = "10.8.2.dev"
+__version__ = "10.8.3"
 __description__ = "Open data portal"

udata/core/activity/models.py

@@ -1,11 +1,13 @@
 from datetime import datetime
 
 from blinker import Signal
+from mongoengine.errors import DoesNotExist
 from mongoengine.signals import post_save
 
 from udata.api_fields import get_fields
 from udata.auth import current_user
 from udata.mongo import db
+from udata.utils import get_field_value_from_path
 
 from .signals import new_activity
 
@@ -79,6 +81,25 @@ class Activity(db.Document, metaclass=EmitNewActivityMetaClass):
 
 
 class Auditable(object):
+    def clean(self, **kwargs):
+        super().clean()
+        """
+        Fetch original document changed fields values before the new one erase it.
+        """
+        changed_fields = self._get_changed_fields()
+        if changed_fields:
+            try:
+                # `only` does not support having nested list as expressed in changed fields, ex resources.0.title
+                # thus we only strip to direct attributes for simplicity
+                direct_attributes = set(field.split(".")[0] for field in changed_fields)
+                old_document = self.__class__.objects.only(*direct_attributes).get(pk=self.pk)
+                self._previous_changed_fields = {}
+                for field_path in changed_fields:
+                    field_value = get_field_value_from_path(old_document, field_path)
+                    self._previous_changed_fields[field_path] = field_value
+            except DoesNotExist:
+                pass
+
     @classmethod
     def post_save(cls, sender, document, **kwargs):
         try:
@@ -97,6 +118,7 @@ class Auditable(object):
         if kwargs.get("created"):
             cls.on_create.send(document)
         elif len(changed_fields):
-            cls.on_update.send(document, changed_fields=changed_fields)
+            previous = getattr(document, "_previous_changed_fields", None)
+            cls.on_update.send(document, changed_fields=changed_fields, previous=previous)
         if getattr(document, "deleted_at", None) or getattr(document, "deleted", None):
             cls.on_delete.send(document)

udata/core/dataset/api_fields.py

@@ -266,6 +266,7 @@ DEFAULT_MASK = ",".join(
         "acronym",
         "slug",
         "description",
+        "description_short",
         "created_at",
         "last_modified",
         "deleted",
@@ -327,6 +328,7 @@ dataset_fields = api.model(
         "description": fields.Markdown(
             description="The dataset description in markdown", required=True
         ),
+        "description_short": fields.String(description="The dataset short description"),
         "created_at": fields.ISODateTime(
             description="This date is computed between harvested creation date if any and site's internal creation date",
             required=True,

udata/core/dataset/apiv2.py

@@ -44,6 +44,7 @@ DEFAULT_MASK_APIV2 = ",".join(
         "acronym",
         "slug",
         "description",
+        "description_short",
         "created_at",
         "last_modified",
         "deleted",
@@ -105,6 +106,9 @@ dataset_fields = apiv2.model(
         "description": fields.Markdown(
             description="The dataset description in markdown", required=True
         ),
+        "description_short": fields.String(
+            description="The dataset short description", required=False
+        ),
         "created_at": fields.ISODateTime(
             description="The dataset creation date", required=True, readonly=True
         ),

udata/core/dataset/constants.py

@@ -89,5 +89,6 @@ SCHEMA_CACHE_DURATION = 60 * 5  # In seconds
 
 TITLE_SIZE_LIMIT = 350
 DESCRIPTION_SIZE_LIMIT = 100000
+DESCRIPTION_SHORT_SIZE_LIMIT = 200
 
 FULL_OBJECTS_HEADER = "X-Get-Datasets-Full-Objects"

udata/core/dataset/csv.py

@@ -26,6 +26,7 @@ class DatasetCsvAdapter(csv.Adapter):
         ("owner_id", "owner.id"),
         # 'contact_point', #  ?
         "description",
+        "description_short",
         "frequency",
         "license",
         "temporal_coverage.start",

udata/core/dataset/forms.py

@@ -7,6 +7,7 @@ from udata.mongo.errors import FieldValidationError
 from .constants import (
     CHECKSUM_TYPES,
     DEFAULT_FREQUENCY,
+    DESCRIPTION_SHORT_SIZE_LIMIT,
     DESCRIPTION_SIZE_LIMIT,
     LEGACY_FREQUENCIES,
     RESOURCE_FILETYPES,
@@ -151,6 +152,11 @@ class DatasetForm(ModelForm):
         [validators.DataRequired(), validators.Length(max=DESCRIPTION_SIZE_LIMIT)],
         description=_("The details about the dataset (collection process, specifics...)."),
     )
+    description_short = fields.StringField(
+        _("Short description"),
+        [validators.Length(max=DESCRIPTION_SHORT_SIZE_LIMIT)],
+        description=_("A short description of the dataset."),
+    )
     license = fields.ModelSelectField(_("License"), model=License, allow_blank=True)
     frequency = fields.SelectField(
         _("Update frequency"),

udata/core/dataset/metrics.py

@@ -0,0 +1,34 @@
+from udata.core.dataservices.models import Dataservice
+from udata.core.reuse.models import Reuse
+
+from .models import Dataset
+
+
+@Reuse.on_create.connect
+@Reuse.on_update.connect
+@Reuse.on_delete.connect
+def update_dataset_reuses_metric(reuse, **kwargs):
+    previous = kwargs.get("previous")
+    if previous and previous.get("datasets"):
+        datasets_delta = set(dat.id for dat in reuse.datasets).symmetric_difference(
+            set(dat.id for dat in previous["datasets"])
+        )
+    else:
+        datasets_delta = set(dat.id for dat in reuse.datasets)
+    for dataset in datasets_delta:
+        Dataset.get(dataset).count_reuses()
+
+
+@Dataservice.on_create.connect
+@Dataservice.on_update.connect
+@Dataservice.on_delete.connect
+def update_dataset_dataservices_metric(dataservice, **kwargs):
+    previous = kwargs.get("previous")
+    if previous and previous.get("datasets"):
+        datasets_delta = set(dat.id for dat in dataservice.datasets).symmetric_difference(
+            set(dat.id for dat in previous["datasets"])
+        )
+    else:
+        datasets_delta = set(dat.id for dat in dataservice.datasets)
+    for dataset in datasets_delta:
+        Dataset.get(dataset).count_dataservices()

udata/core/dataset/models.py

@@ -35,6 +35,7 @@ from .constants import (
     CHECKSUM_TYPES,
     CLOSED_FORMATS,
     DEFAULT_LICENSE,
+    DESCRIPTION_SHORT_SIZE_LIMIT,
     LEGACY_FREQUENCIES,
     MAX_DISTANCE,
     PIVOTAL_DATA,
@@ -560,6 +561,7 @@ class Dataset(Auditable, WithMetrics, DatasetBadgeMixin, Owned, Linkable, db.Doc
         auditable=False,
     )
     description = field(db.StringField(required=True, default=""))
+    description_short = field(db.StringField(max_length=DESCRIPTION_SHORT_SIZE_LIMIT))
     license = field(db.ReferenceField("License"))
 
     tags = field(db.TagListField())
@@ -616,6 +618,7 @@ class Dataset(Auditable, WithMetrics, DatasetBadgeMixin, Owned, Linkable, db.Doc
         "discussions_open",
         "reuses",
         "reuses_by_months",
+        "dataservices",
         "followers",
         "followers_by_months",
         "views",
@@ -628,6 +631,7 @@ class Dataset(Auditable, WithMetrics, DatasetBadgeMixin, Owned, Linkable, db.Doc
             "created_at_internal",
             "last_modified_internal",
             "metrics.reuses",
+            "metrics.dataservices",
             "metrics.followers",
             "metrics.views",
             "slug",
@@ -1100,6 +1104,12 @@ class Dataset(Auditable, WithMetrics, DatasetBadgeMixin, Owned, Linkable, db.Doc
         self.metrics["reuses_by_months"] = get_stock_metrics(Reuse.objects(datasets=self).visible())
         self.save(signal_kwargs={"ignores": ["post_save"]})
 
+    def count_dataservices(self):
+        from udata.core.dataservices.models import Dataservice
+
+        self.metrics["dataservices"] = Dataservice.objects(datasets=self).visible().count()
+        self.save(signal_kwargs={"ignores": ["post_save"]})
+
     def count_followers(self):
         from udata.models import Follow
 

udata/core/dataset/tasks.py

@@ -123,17 +123,6 @@ def send_frequency_reminder(self):
     print("Done")
 
 
-@job("update-datasets-reuses-metrics")
-def update_datasets_reuses_metrics(self):
-    all_datasets = Dataset.objects.visible().timeout(False)
-    for dataset in all_datasets:
-        try:
-            dataset.count_reuses()
-        except Exception as e:
-            log.error(f"Error for dataset {dataset} during reuses metrics update: {e}")
-            continue
-
-
 def get_queryset(model_cls):
     # special case for resources
     if model_cls.__name__ == "Resource":

udata/core/metrics/__init__.py

@@ -6,6 +6,7 @@ def init_app(app):
     import udata.core.user.metrics  # noqa
     import udata.core.organization.metrics  # noqa
     import udata.core.discussions.metrics  # noqa
+    import udata.core.dataset.metrics  # noqa
     import udata.core.reuse.metrics  # noqa
     import udata.core.followers.metrics  # noqa
 

udata/core/metrics/commands.py

@@ -74,6 +74,7 @@ def update(
                         dataset.metrics.clear()
                     dataset.count_discussions()
                     dataset.count_reuses()
+                    dataset.count_dataservices()
                     dataset.count_followers()
                 except Exception as e:
                     log.info(f"Error during update: {e}")
@@ -119,6 +120,7 @@ def update(
                         organization.metrics.clear()
                     organization.count_datasets()
                     organization.count_reuses()
+                    organization.count_dataservices()
                     organization.count_followers()
                     organization.count_members()
                 except Exception as e:
@@ -135,6 +137,7 @@ def update(
                         user.metrics.clear()
                     user.count_datasets()
                     user.count_reuses()
+                    user.count_dataservices()
                     user.count_followers()
                     user.count_following()
                 except Exception as e:

udata/core/organization/csv.py

@@ -4,6 +4,13 @@ from udata.core.dataset.models import Dataset
 from .models import Organization
 
 
+def get_resource_download_count(organization: Organization) -> int:
+    return sum(
+        dat.metrics.get("resources_downloads", 0) or 0
+        for dat in Dataset.objects(organization=organization).only("metrics").visible()
+    )
+
+
 @csv.adapter(Organization)
 class OrganizationCsvAdapter(csv.Adapter):
     downloads_counts = None
@@ -21,32 +28,8 @@ class OrganizationCsvAdapter(csv.Adapter):
         "last_modified",
         "business_number_id",
         ("members_count", lambda o: len(o.members)),
+        ("downloads", get_resource_download_count),
     )
 
     def dynamic_fields(self):
-        return csv.metric_fields(Organization) + self.get_dynamic_field_downloads()
-
-    def get_dynamic_field_downloads(self):
-        downloads_counts = self.get_downloads_counts()
-        return [("downloads", lambda o: downloads_counts.get(str(o.id), 0))]
-
-    def get_downloads_counts(self):
-        """
-        Prefetch all the resources' downloads for all selected organization into memory
-        """
-        if self.downloads_counts is not None:
-            return self.downloads_counts
-
-        self.downloads_counts = {}
-
-        ids = [o.id for o in self.queryset]
-        for dataset in Dataset.objects(organization__in=ids):
-            org_id = str(dataset.organization.id)
-            if self.downloads_counts.get(org_id) is None:
-                self.downloads_counts[org_id] = 0
-
-            self.downloads_counts[org_id] += sum(
-                resource.metrics.get("views", 0) for resource in dataset.resources
-            )
-
-        return self.downloads_counts
+        return csv.metric_fields(Organization)

udata/core/organization/metrics.py

@@ -35,3 +35,5 @@ def update_org_metrics(document, previous):
         previous.count_datasets()
     elif isinstance(document, Reuse):
         previous.count_reuses()
+    elif isinstance(document, Dataservice):
+        previous.count_dataservices()

udata/core/organization/models.py

@@ -188,6 +188,10 @@ class Organization(
     after_delete = Signal()
     on_delete = Signal()
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.compute_aggregate_metrics = True
+
     @classmethod
     def pre_save(cls, sender, document, **kwargs):
         cls.before_save.send(document)
@@ -307,15 +311,16 @@ class Organization(
         from udata.models import Dataset, Follow, Reuse
 
         self.metrics["datasets"] = Dataset.objects(organization=self).visible().count()
-        self.metrics["datasets_by_months"] = get_stock_metrics(
-            Dataset.objects(organization=self).visible(), date_label="created_at_internal"
-        )
-        self.metrics["datasets_followers_by_months"] = get_stock_metrics(
-            Follow.objects(following__in=Dataset.objects(organization=self)), date_label="since"
-        )
-        self.metrics["datasets_reuses_by_months"] = get_stock_metrics(
-            Reuse.objects(datasets__in=Dataset.objects(organization=self)).visible()
-        )
+        if self.compute_aggregate_metrics:
+            self.metrics["datasets_by_months"] = get_stock_metrics(
+                Dataset.objects(organization=self).visible(), date_label="created_at_internal"
+            )
+            self.metrics["datasets_followers_by_months"] = get_stock_metrics(
+                Follow.objects(following__in=Dataset.objects(organization=self)), date_label="since"
+            )
+            self.metrics["datasets_reuses_by_months"] = get_stock_metrics(
+                Reuse.objects(datasets__in=Dataset.objects(organization=self)).visible()
+            )
 
         self.save(signal_kwargs={"ignores": ["post_save"]})
 

udata/core/user/metrics.py

@@ -42,3 +42,5 @@ def update_owner_metrics(document, previous):
         previous.count_datasets()
     elif isinstance(document, Reuse):
         previous.count_reuses()
+    elif isinstance(document, Dataservice):
+        previous.count_dataservices()