udata 10.4.2.dev35427__py2.py3-none-any.whl → 10.4.2.dev35451__py2.py3-none-any.whl

udata/core/metrics/commands.py

@@ -60,6 +60,7 @@ def update(
             site.count_max_org_followers()
             site.count_max_org_reuses()
             site.count_max_org_datasets()
+            site.count_stock_metrics()
         except Exception as e:
             log.info(f"Error during update: {e}")
 

udata/core/metrics/helpers.py

@@ -0,0 +1,102 @@
+import logging
+from collections import OrderedDict
+from datetime import datetime, timedelta
+from typing import Dict, List, Union
+
+import requests
+from bson import ObjectId
+from dateutil.rrule import MONTHLY, rrule
+from flask import current_app
+from mongoengine import QuerySet
+from pymongo.command_cursor import CommandCursor
+
+log = logging.getLogger(__name__)
+
+
+def get_last_13_months() -> List[str]:
+    dstart = datetime.today().replace(day=1) - timedelta(days=365)
+    months = rrule(freq=MONTHLY, count=13, dtstart=dstart)
+    return [month.strftime("%Y-%m") for month in months]
+
+
+def compute_monthly_metrics(metrics_data: List[Dict], metrics_labels: List[str]) -> OrderedDict:
+    # Initialize default monthly_metrics
+    monthly_metrics = OrderedDict(
+        (month, {label: 0 for label in metrics_labels}) for month in get_last_13_months()
+    )
+    # Update monthly_metrics with metrics_data values
+    for entry in metrics_data:
+        entry_month = entry["metric_month"]
+        if entry_month in monthly_metrics:
+            for metric_label in metrics_labels:
+                label = f"monthly_{metric_label}"
+                monthly_metrics[entry_month][metric_label] = entry.get(label) or 0
+    return monthly_metrics
+
+
+def metrics_by_label(monthly_metrics: Dict, metrics_labels: List[str]) -> List[OrderedDict]:
+    metrics_by_label = []
+    for label in metrics_labels:
+        metrics_by_label.append(
+            OrderedDict((month, monthly_metrics[month][label]) for month in monthly_metrics)
+        )
+    return metrics_by_label
+
+
+def get_metrics_for_model(
+    model: str, id: Union[str, ObjectId, None], metrics_labels: List[str]
+) -> List[OrderedDict]:
+    """
+    Get distant metrics for a particular model object
+    """
+    if not current_app.config["METRICS_API"]:
+        # TODO: How to best deal with no METRICS_API, prevent calling or return empty?
+        # raise ValueError("missing config METRICS_API to use this function")
+        return [{} for _ in range(len(metrics_labels))]
+    models = model + "s" if id else model  # TODO: not clean of a hack
+    model_metrics_api = f"{current_app.config['METRICS_API']}/{models}/data/"
+    try:
+        params = {"metric_month__sort": "desc"}
+        if id:
+            params[f"{model}_id__exact"] = id
+        res = requests.get(model_metrics_api, params)
+        res.raise_for_status()
+        monthly_metrics = compute_monthly_metrics(res.json()["data"], metrics_labels)
+        return metrics_by_label(monthly_metrics, metrics_labels)
+    except requests.exceptions.RequestException as e:
+        log.exception(f"Error while getting metrics for {model}({id}): {e}")
+        return [{} for _ in range(len(metrics_labels))]
+
+
+def compute_monthly_aggregated_metrics(aggregation_res: CommandCursor) -> OrderedDict:
+    monthly_metrics = OrderedDict((month, 0) for month in get_last_13_months())
+    for monthly_count in aggregation_res:
+        year, month = monthly_count["_id"].split("-")
+        monthly_label = year + "-" + month.zfill(2)
+        if monthly_label in monthly_metrics:
+            monthly_metrics[monthly_label] = monthly_count["count"]
+    return monthly_metrics
+
+
+def get_stock_metrics(objects: QuerySet, date_label: str = "created_at") -> OrderedDict:
+    """
+    Get stock metrics for a particular model object
+    """
+    pipeline = [
+        {"$match": {date_label: {"$gte": datetime.now() - timedelta(days=365)}}},
+        {
+            "$group": {
+                "_id": {
+                    "$concat": [
+                        {"$substr": [{"$year": f"${date_label}"}, 0, 4]},
+                        "-",
+                        {"$substr": [{"$month": f"${date_label}"}, 0, 12]},
+                    ]
+                },
+                "count": {"$sum": 1},
+            }
+        },
+    ]
+    aggregation_res = objects.aggregate(*pipeline)
+
+    return compute_monthly_aggregated_metrics(aggregation_res)

udata/core/metrics/tasks.py

@@ -24,5 +24,6 @@ def compute_site_metrics(self):
     site.count_max_org_followers()
     site.count_max_org_reuses()
     site.count_max_org_datasets()
+    site.count_stock_metrics()
     # Sending signal
     on_site_metrics_computed.send(site)

udata/core/site/models.py

@@ -3,6 +3,7 @@ from werkzeug.local import LocalProxy
 
 from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.models import Dataset
+from udata.core.metrics.helpers import get_metrics_for_model, get_stock_metrics
 from udata.core.organization.models import Organization
 from udata.core.reuse.models import Reuse
 from udata.models import WithMetrics, db
@@ -36,15 +37,23 @@ class Site(WithMetrics, db.Document):
         "max_org_reuses",
         "max_org_datasets",
         "datasets",
+        "datasets_visits_by_months",
         "discussions",
         "followers",
         "organizations",
         "public-service",
         "resources",
+        "resources_downloads_by_months",
         "reuses",
         "dataservices",
         "users",
         "harvesters",
+        "users_by_months",
+        "datasets_by_months",
+        "harvesters_by_months",
+        "reuses_by_months",
+        "organizations_by_months",
+        "discussions_by_months",
     ]
 
     def __str__(self):
@@ -72,6 +81,9 @@ class Site(WithMetrics, db.Document):
         from udata.models import Dataset
 
         self.metrics["datasets"] = Dataset.objects.visible().count()
+        self.metrics["datasets_visits_by_months"] = get_metrics_for_model(
+            "site", None, ["visit_dataset"]
+        )[0]
         self.save()
 
     def count_resources(self):
@@ -83,6 +95,9 @@ class Site(WithMetrics, db.Document):
             ),
             {},
         ).get("count", 0)
+        self.metrics["resources_downloads_by_months"] = get_metrics_for_model(
+            "site", None, ["download_resource"]
+        )[0]
         self.save()
 
     def count_reuses(self):
@@ -172,6 +187,24 @@ class Site(WithMetrics, db.Document):
         self.metrics["max_org_datasets"] = org.metrics["datasets"] if org else 0
         self.save()
 
+    def count_stock_metrics(self):
+        from udata.harvest.models import HarvestSource
+        from udata.models import Discussion, User
+
+        self.metrics["users_by_months"] = get_stock_metrics(User.objects())
+        self.metrics["datasets_by_months"] = get_stock_metrics(
+            Dataset.objects().visible(), date_label="created_at_internal"
+        )
+        self.metrics["harvesters_by_months"] = get_stock_metrics(HarvestSource.objects())
+        self.metrics["reuses_by_months"] = get_stock_metrics(Reuse.objects().visible())
+        self.metrics["organizations_by_months"] = get_stock_metrics(
+            Organization.objects().visible()
+        )
+        self.metrics["discussions_by_months"] = get_stock_metrics(
+            Discussion.objects(), date_label="created"
+        )
+        self.save()
+
 
 def get_current_site():
     if getattr(g, "site", None) is None:

udata/core/user/activities.py

@@ -30,11 +30,6 @@ class DiscussActivity(object):
     badge_type = "warning"
 
 
-class UserStarredOrganization(FollowActivity, OrgRelatedActivity, Activity):
-    key = "organization:followed"
-    label = _("followed an organization")
-
-
 class UserFollowedUser(FollowActivity, Activity):
     key = "user:followed"
     label = _("followed a user")

udata/migrations/2025-05-22-purge-duplicate-activities.py

@@ -0,0 +1,101 @@
+"""
+This migration updates Topic.featured to False when it is None.
+"""
+
+import logging
+from datetime import datetime, timedelta
+
+from mongoengine.connection import get_db
+
+from udata.core.dataset.activities import UserCreatedDataset, UserDeletedDataset, UserUpdatedDataset
+from udata.core.organization.activities import UserUpdatedOrganization
+from udata.core.reuse.activities import UserCreatedReuse, UserDeletedReuse, UserUpdatedReuse
+from udata.core.user.activities import (
+    UserDiscussedDataset,
+    UserDiscussedReuse,
+    UserFollowedDataset,
+    UserFollowedOrganization,
+    UserFollowedReuse,
+)
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    # Remove legacy fields (`as_organization`, `kwargs`) from old activities
+    result = get_db().activity.update_many({}, {"$unset": {"as_organization": ""}})
+    log.info(
+        f"Legacy field `as_organization` removed from {result.modified_count} activity objects"
+    )
+
+    result = get_db().activity.update_many({}, {"$unset": {"kwargs": ""}})
+    log.info(f"Legacy field `kwargs` removed from {result.modified_count} activity objects")
+
+    # Clean duplicate activities in case of discussion or following
+    # - remove the "updated" activity on the discussed/followed object
+    # - remove the activity on the organization
+    # The heuristic is to look for specific activities by the same actor on the targeted object
+    # within a -1 +1 second timespan
+    for action_related_activity, object_updated_activity in [
+        (UserDiscussedDataset, UserUpdatedDataset),
+        (UserDiscussedReuse, UserUpdatedReuse),
+        (UserFollowedDataset, UserUpdatedDataset),
+        (UserFollowedReuse, UserUpdatedReuse),
+    ]:
+        org_activity_count = 0
+        object_activity_count = 0
+        activities = (
+            action_related_activity.objects()
+            .no_dereference()  # We use no_dereference in query to prevent DBref DoesNotExist errors
+            .no_cache()
+            .timeout(False)
+        )
+        log.info(
+            f"{datetime.utcnow()}: Processing {activities.count()} {action_related_activity} activities..."
+        )
+        for act in activities:
+            object_activity_count += object_updated_activity.objects(
+                actor=act.actor.id,
+                related_to=act.related_to.id,
+                created_at__gte=act.created_at - timedelta(seconds=1),
+                created_at__lte=act.created_at + timedelta(seconds=1),
+            ).delete()
+            if act.organization:
+                org_activity_count += UserUpdatedOrganization.objects(
+                    actor=act.actor.id,
+                    related_to=act.organization,
+                    created_at__gte=act.created_at - timedelta(seconds=1),
+                    created_at__lte=act.created_at + timedelta(seconds=1),
+                ).delete()
+        log.info(
+            f"{datetime.utcnow()}: Deleted {object_activity_count} {object_updated_activity} and {org_activity_count} UserUpdatedOrganization activities"
+        )
+
+    # Clean duplicated UserUpdatedOrganization activities on organization for any object related activity
+    for object_related_activity in [
+        UserCreatedDataset,
+        UserUpdatedDataset,
+        UserDeletedDataset,
+        UserCreatedReuse,
+        UserUpdatedReuse,
+        UserDeletedReuse,
+        UserFollowedOrganization,
+    ]:
+        count = 0
+        activities = (
+            object_related_activity.objects(organization__exists=True)
+            .no_dereference()  # We use no_dereference in query to prevent DBref DoesNotExist errors
+            .no_cache()
+            .timeout(False)
+        )
+        log.info(
+            f"{datetime.utcnow()}: Processing {activities.count()} {object_related_activity} activities..."
+        )
+        for act in activities:
+            count += UserUpdatedOrganization.objects(
+                actor=act.actor.id,
+                related_to=act.organization,
+                created_at__gte=act.created_at - timedelta(seconds=1),
+                created_at__lte=act.created_at + timedelta(seconds=1),
+            ).delete()
+        log.info(f"{datetime.utcnow()}: Deleted {count} UserUpdatedOrganization activities")

udata/settings.py

@@ -574,6 +574,10 @@ class Defaults(object):
     ###########################################################################
     MAX_RESOURCES_IN_JSON_LD = 20
 
+    # Metrics settings
+    ###########################################################################
+    METRICS_API = None
+
 
 class Testing(object):
     """Sane values for testing. Should be applied as override"""