udata 10.8.3.dev37191__py2.py3-none-any.whl → 10.8.3.dev37263__py2.py3-none-any.whl

udata/core/organization/csv.py

@@ -4,6 +4,13 @@ from udata.core.dataset.models import Dataset
 from .models import Organization
 
 
+def get_resource_download_count(organization: Organization) -> int:
+    return sum(
+        dat.metrics.get("resources_downloads", 0) or 0
+        for dat in Dataset.objects(organization=organization).only("metrics").visible()
+    )
+
+
 @csv.adapter(Organization)
 class OrganizationCsvAdapter(csv.Adapter):
     downloads_counts = None
@@ -21,32 +28,8 @@ class OrganizationCsvAdapter(csv.Adapter):
         "last_modified",
         "business_number_id",
         ("members_count", lambda o: len(o.members)),
+        ("downloads", get_resource_download_count),
     )
 
     def dynamic_fields(self):
-        return csv.metric_fields(Organization) + self.get_dynamic_field_downloads()
-
-    def get_dynamic_field_downloads(self):
-        downloads_counts = self.get_downloads_counts()
-        return [("downloads", lambda o: downloads_counts.get(str(o.id), 0))]
-
-    def get_downloads_counts(self):
-        """
-        Prefetch all the resources' downloads for all selected organization into memory
-        """
-        if self.downloads_counts is not None:
-            return self.downloads_counts
-
-        self.downloads_counts = {}
-
-        ids = [o.id for o in self.queryset]
-        for dataset in Dataset.objects(organization__in=ids):
-            org_id = str(dataset.organization.id)
-            if self.downloads_counts.get(org_id) is None:
-                self.downloads_counts[org_id] = 0
-
-            self.downloads_counts[org_id] += sum(
-                resource.metrics.get("views", 0) for resource in dataset.resources
-            )
-
-        return self.downloads_counts
+        return csv.metric_fields(Organization)

udata/core/organization/models.py

@@ -188,6 +188,10 @@ class Organization(
     after_delete = Signal()
     on_delete = Signal()
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.compute_aggregate_metrics = True
+
     @classmethod
     def pre_save(cls, sender, document, **kwargs):
         cls.before_save.send(document)
@@ -307,15 +311,16 @@ class Organization(
         from udata.models import Dataset, Follow, Reuse
 
         self.metrics["datasets"] = Dataset.objects(organization=self).visible().count()
-        self.metrics["datasets_by_months"] = get_stock_metrics(
-            Dataset.objects(organization=self).visible(), date_label="created_at_internal"
-        )
-        self.metrics["datasets_followers_by_months"] = get_stock_metrics(
-            Follow.objects(following__in=Dataset.objects(organization=self)), date_label="since"
-        )
-        self.metrics["datasets_reuses_by_months"] = get_stock_metrics(
-            Reuse.objects(datasets__in=Dataset.objects(organization=self)).visible()
-        )
+        if self.compute_aggregate_metrics:
+            self.metrics["datasets_by_months"] = get_stock_metrics(
+                Dataset.objects(organization=self).visible(), date_label="created_at_internal"
+            )
+            self.metrics["datasets_followers_by_months"] = get_stock_metrics(
+                Follow.objects(following__in=Dataset.objects(organization=self)), date_label="since"
+            )
+            self.metrics["datasets_reuses_by_months"] = get_stock_metrics(
+                Reuse.objects(datasets__in=Dataset.objects(organization=self)).visible()
+            )
 
         self.save(signal_kwargs={"ignores": ["post_save"]})
 

udata/harvest/backends/dcat.py

@@ -66,6 +66,10 @@ def extract_graph(source, target, node, specs):
 class DcatBackend(BaseBackend):
     display_name = "DCAT"
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.organizations_to_update = set()
+
     def inner_harvest(self):
         fmt = self.get_format()
         self.job.data = {"format": fmt}
@@ -76,6 +80,10 @@ class DcatBackend(BaseBackend):
             self.process_one_datasets_page(page_number, page)
             pages.append((page_number, page))
 
+        for org in self.organizations_to_update:
+            org.compute_aggregate_metrics = True
+            org.count_datasets()
+
         # We do a second pass to have all datasets in memory and attach datasets
         # to dataservices. It could be better to be one pass of graph walking and
         # then one pass of attaching datasets to dataservices.
@@ -217,7 +225,11 @@ class DcatBackend(BaseBackend):
 
         dataset = self.get_dataset(item.remote_id)
         remote_url_prefix = self.get_extra_config_value("remote_url_prefix")
-        return dataset_from_rdf(page, dataset, node=node, remote_url_prefix=remote_url_prefix)
+        dataset = dataset_from_rdf(page, dataset, node=node, remote_url_prefix=remote_url_prefix)
+        if dataset.organization:
+            dataset.organization.compute_aggregate_metrics = False
+            self.organizations_to_update.add(dataset.organization)
+        return dataset
 
     def inner_process_dataservice(self, item: HarvestItem, page_number: int, page: Graph, node):
         item.kwargs["page_number"] = page_number