udata 10.4.2.dev35441__py2.py3-none-any.whl → 10.4.2.dev35451__py2.py3-none-any.whl

udata/core/metrics/commands.py

@@ -60,6 +60,7 @@ def update(
             site.count_max_org_followers()
             site.count_max_org_reuses()
             site.count_max_org_datasets()
+            site.count_stock_metrics()
         except Exception as e:
             log.info(f"Error during update: {e}")
 

udata/core/metrics/helpers.py

@@ -0,0 +1,102 @@
+import logging
+from collections import OrderedDict
+from datetime import datetime, timedelta
+from typing import Dict, List, Union
+
+import requests
+from bson import ObjectId
+from dateutil.rrule import MONTHLY, rrule
+from flask import current_app
+from mongoengine import QuerySet
+from pymongo.command_cursor import CommandCursor
+
+log = logging.getLogger(__name__)
+
+
+def get_last_13_months() -> List[str]:
+    dstart = datetime.today().replace(day=1) - timedelta(days=365)
+    months = rrule(freq=MONTHLY, count=13, dtstart=dstart)
+    return [month.strftime("%Y-%m") for month in months]
+
+
+def compute_monthly_metrics(metrics_data: List[Dict], metrics_labels: List[str]) -> OrderedDict:
+    # Initialize default monthly_metrics
+    monthly_metrics = OrderedDict(
+        (month, {label: 0 for label in metrics_labels}) for month in get_last_13_months()
+    )
+    # Update monthly_metrics with metrics_data values
+    for entry in metrics_data:
+        entry_month = entry["metric_month"]
+        if entry_month in monthly_metrics:
+            for metric_label in metrics_labels:
+                label = f"monthly_{metric_label}"
+                monthly_metrics[entry_month][metric_label] = entry.get(label) or 0
+    return monthly_metrics
+
+
+def metrics_by_label(monthly_metrics: Dict, metrics_labels: List[str]) -> List[OrderedDict]:
+    metrics_by_label = []
+    for label in metrics_labels:
+        metrics_by_label.append(
+            OrderedDict((month, monthly_metrics[month][label]) for month in monthly_metrics)
+        )
+    return metrics_by_label
+
+
+def get_metrics_for_model(
+    model: str, id: Union[str, ObjectId, None], metrics_labels: List[str]
+) -> List[OrderedDict]:
+    """
+    Get distant metrics for a particular model object
+    """
+    if not current_app.config["METRICS_API"]:
+        # TODO: How to best deal with no METRICS_API, prevent calling or return empty?
+        # raise ValueError("missing config METRICS_API to use this function")
+        return [{} for _ in range(len(metrics_labels))]
+    models = model + "s" if id else model  # TODO: not clean of a hack
+    model_metrics_api = f"{current_app.config['METRICS_API']}/{models}/data/"
+    try:
+        params = {"metric_month__sort": "desc"}
+        if id:
+            params[f"{model}_id__exact"] = id
+        res = requests.get(model_metrics_api, params)
+        res.raise_for_status()
+        monthly_metrics = compute_monthly_metrics(res.json()["data"], metrics_labels)
+        return metrics_by_label(monthly_metrics, metrics_labels)
+    except requests.exceptions.RequestException as e:
+        log.exception(f"Error while getting metrics for {model}({id}): {e}")
+        return [{} for _ in range(len(metrics_labels))]
+
+
+def compute_monthly_aggregated_metrics(aggregation_res: CommandCursor) -> OrderedDict:
+    monthly_metrics = OrderedDict((month, 0) for month in get_last_13_months())
+    for monthly_count in aggregation_res:
+        year, month = monthly_count["_id"].split("-")
+        monthly_label = year + "-" + month.zfill(2)
+        if monthly_label in monthly_metrics:
+            monthly_metrics[monthly_label] = monthly_count["count"]
+    return monthly_metrics
+
+
+def get_stock_metrics(objects: QuerySet, date_label: str = "created_at") -> OrderedDict:
+    """
+    Get stock metrics for a particular model object
+    """
+    pipeline = [
+        {"$match": {date_label: {"$gte": datetime.now() - timedelta(days=365)}}},
+        {
+            "$group": {
+                "_id": {
+                    "$concat": [
+                        {"$substr": [{"$year": f"${date_label}"}, 0, 4]},
+                        "-",
+                        {"$substr": [{"$month": f"${date_label}"}, 0, 12]},
+                    ]
+                },
+                "count": {"$sum": 1},
+            }
+        },
+    ]
+    aggregation_res = objects.aggregate(*pipeline)
+
+    return compute_monthly_aggregated_metrics(aggregation_res)

udata/core/metrics/tasks.py

@@ -24,5 +24,6 @@ def compute_site_metrics(self):
     site.count_max_org_followers()
     site.count_max_org_reuses()
     site.count_max_org_datasets()
+    site.count_stock_metrics()
     # Sending signal
     on_site_metrics_computed.send(site)

udata/core/site/models.py

@@ -3,6 +3,7 @@ from werkzeug.local import LocalProxy
 
 from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.models import Dataset
+from udata.core.metrics.helpers import get_metrics_for_model, get_stock_metrics
 from udata.core.organization.models import Organization
 from udata.core.reuse.models import Reuse
 from udata.models import WithMetrics, db
@@ -36,15 +37,23 @@ class Site(WithMetrics, db.Document):
         "max_org_reuses",
         "max_org_datasets",
         "datasets",
+        "datasets_visits_by_months",
         "discussions",
         "followers",
         "organizations",
         "public-service",
         "resources",
+        "resources_downloads_by_months",
         "reuses",
         "dataservices",
         "users",
         "harvesters",
+        "users_by_months",
+        "datasets_by_months",
+        "harvesters_by_months",
+        "reuses_by_months",
+        "organizations_by_months",
+        "discussions_by_months",
     ]
 
     def __str__(self):
@@ -72,6 +81,9 @@ class Site(WithMetrics, db.Document):
         from udata.models import Dataset
 
         self.metrics["datasets"] = Dataset.objects.visible().count()
+        self.metrics["datasets_visits_by_months"] = get_metrics_for_model(
+            "site", None, ["visit_dataset"]
+        )[0]
         self.save()
 
     def count_resources(self):
@@ -83,6 +95,9 @@ class Site(WithMetrics, db.Document):
             ),
             {},
         ).get("count", 0)
+        self.metrics["resources_downloads_by_months"] = get_metrics_for_model(
+            "site", None, ["download_resource"]
+        )[0]
         self.save()
 
     def count_reuses(self):
@@ -172,6 +187,24 @@ class Site(WithMetrics, db.Document):
         self.metrics["max_org_datasets"] = org.metrics["datasets"] if org else 0
         self.save()
 
+    def count_stock_metrics(self):
+        from udata.harvest.models import HarvestSource
+        from udata.models import Discussion, User
+
+        self.metrics["users_by_months"] = get_stock_metrics(User.objects())
+        self.metrics["datasets_by_months"] = get_stock_metrics(
+            Dataset.objects().visible(), date_label="created_at_internal"
+        )
+        self.metrics["harvesters_by_months"] = get_stock_metrics(HarvestSource.objects())
+        self.metrics["reuses_by_months"] = get_stock_metrics(Reuse.objects().visible())
+        self.metrics["organizations_by_months"] = get_stock_metrics(
+            Organization.objects().visible()
+        )
+        self.metrics["discussions_by_months"] = get_stock_metrics(
+            Discussion.objects(), date_label="created"
+        )
+        self.save()
+
 
 def get_current_site():
     if getattr(g, "site", None) is None:

udata/settings.py

@@ -574,6 +574,10 @@ class Defaults(object):
     ###########################################################################
     MAX_RESOURCES_IN_JSON_LD = 20
 
+    # Metrics settings
+    ###########################################################################
+    METRICS_API = None
+
 
 class Testing(object):
     """Sane values for testing. Should be applied as override"""