wbfdm 2.2.1__py2.py3-none-any.whl

wbfdm/contrib/metric/migrations/0001_initial.py

@@ -0,0 +1,88 @@
+# Generated by Django 5.0.6 on 2024-05-27 08:56
+
+import django.core.serializers.json
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = [
+        ("contenttypes", "0002_remove_content_type_name"),
+        ("wbfdm", "0021_delete_instrumentdailystatistics"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="InstrumentMetric",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("basket_id", models.PositiveIntegerField()),
+                ("basket_repr", models.CharField(max_length=256)),
+                (
+                    "date",
+                    models.DateField(
+                        blank=True,
+                        help_text="If date is null, the metric is considered static",
+                        null=True,
+                        verbose_name="Metric Date",
+                    ),
+                ),
+                ("key", models.CharField(max_length=255, verbose_name="Metric Key")),
+                (
+                    "metrics",
+                    models.JSONField(
+                        default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder, verbose_name="Metrics"
+                    ),
+                ),
+                (
+                    "basket_content_type",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="instrument_metrics",
+                        to="contenttypes.contenttype",
+                    ),
+                ),
+                (
+                    "instrument",
+                    models.ForeignKey(
+                        blank=True,
+                        help_text="Instrument where this metric belongs to. If null, the metric is applicable to all instruments related to the basket",
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="metrics",
+                        to="wbfdm.instrument",
+                    ),
+                ),
+                (
+                    "parent_metric",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="dependent_metrics",
+                        to="metric.instrumentmetric",
+                        verbose_name="Parent Metric",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Instrument Metric",
+                "verbose_name_plural": "Instrument Metrics",
+                "indexes": [
+                    models.Index(
+                        fields=["basket_content_type", "basket_id", "key", "date"],
+                        name="metric_inst_basket__30dcda_idx",
+                    )
+                ],
+            },
+        ),
+        migrations.AddConstraint(
+            model_name="instrumentmetric",
+            constraint=models.UniqueConstraint(
+                fields=("basket_content_type", "basket_id", "instrument", "date", "key"),
+                name="unique_instrument_metric",
+            ),
+        ),
+    ]

wbfdm/contrib/metric/migrations/0002_remove_instrumentmetric_unique_instrument_metric_and_more.py

@@ -0,0 +1,26 @@
+# Generated by Django 5.0.9 on 2024-11-06 13:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("contenttypes", "0002_remove_content_type_name"),
+        ("metric", "0001_initial"),
+        ("wbfdm", "0027_remove_instrument_unique_ric_and_more"),
+    ]
+
+    operations = [
+        migrations.RemoveConstraint(
+            model_name="instrumentmetric",
+            name="unique_instrument_metric",
+        ),
+        migrations.AddConstraint(
+            model_name="instrumentmetric",
+            constraint=models.UniqueConstraint(
+                fields=("basket_content_type", "basket_id", "instrument", "date", "key"),
+                name="unique_instrument_metric",
+                nulls_distinct=False,
+            ),
+        ),
+    ]

wbfdm/contrib/metric/models.py

@@ -0,0 +1,180 @@
+from datetime import date as date_lib
+from typing import Self, Type
+
+from django.contrib.contenttypes.fields import GenericForeignKey
+from django.contrib.contenttypes.models import ContentType
+from django.core.serializers.json import DjangoJSONEncoder
+from django.db import models
+from django.db.models.fields.json import KeyTextTransform
+from django.db.models.functions import Cast, NullIf
+from wbcore.models import WBModel
+
+from .backends.base import AbstractBackend
+from .dto import Metric, MetricKey
+from .registry import backend_registry
+
+
+class InstrumentMetric(models.Model):
+    basket_content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, related_name="instrument_metrics")
+    basket_id = models.PositiveIntegerField()
+    basket_repr = models.CharField(max_length=256)
+
+    basket = GenericForeignKey("basket_content_type", "basket_id")
+
+    instrument = models.ForeignKey(
+        "wbfdm.Instrument",
+        on_delete=models.CASCADE,
+        related_name="metrics",
+        null=True,
+        blank=True,
+        help_text="Instrument where this metric belongs to. If null, the metric is applicable to all instruments related to the basket",
+    )
+    date = models.DateField(
+        verbose_name="Metric Date", null=True, blank=True, help_text="If date is null, the metric is considered static"
+    )
+    key = models.CharField(max_length=255, verbose_name="Metric Key")
+    metrics = models.JSONField(default=dict, verbose_name="Metrics", encoder=DjangoJSONEncoder)
+
+    parent_metric = models.ForeignKey(
+        "self",
+        related_name="dependent_metrics",
+        blank=True,
+        null=True,
+        on_delete=models.SET_NULL,
+        verbose_name="Parent Metric",
+    )
+
+    def save(self, *args, **kwargs):
+        self.basket_repr = str(self.basket)
+        super().save(*args, **kwargs)
+
+    @classmethod
+    def update_or_create_from_metric(cls, metric: Metric, parent_instrument_metric: Self | None = None):
+        """
+        Update or create an InstrumentMetric instance based on a given Metric DTO object.
+
+        Args:
+            metric (Metric): The DTO to base the creation or update on.
+            parent_instrument_metric (Optional[InstrumentMetric]): The parent of the created/updated InstrumentMetric instance, if any. Defaults to None.
+
+        Returns:
+            None
+
+        Side Effects:
+            - Creates or updates an InstrumentMetric instance in the database.
+            - Recursively creates or updates dependent InstrumentMetric instances.
+        """
+        base_metric, _ = InstrumentMetric.objects.update_or_create(
+            basket_id=metric.basket_id,
+            basket_content_type_id=metric.basket_content_type_id,
+            instrument_id=metric.instrument_id,
+            date=metric.date,
+            key=metric.key,
+            defaults={
+                "metrics": metric.metrics,
+                "parent_metric": parent_instrument_metric,
+            },
+        )
+        for dependency_metric in metric.dependency_metrics:
+            cls.update_or_create_from_metric(dependency_metric, parent_instrument_metric=base_metric)
+
+    @classmethod
+    def annotate_with_metrics(
+        cls,
+        queryset: models.QuerySet,
+        metric_key: MetricKey,
+        metric_basket_class: Type[WBModel],
+        val_date: date_lib | None = None,
+        basket_label: str = "id",
+        instrument_label: str | None = None,
+    ) -> models.QuerySet:
+        """
+        Annotate metrics to a queryset related to the given metric key and basket class.
+
+        Args:
+            queryset (models.QuerySet): The queryset to be annotated.
+            metric_key (MetricKey): The metric key or MetricKey instance to use for annotation.
+            metric_basket_class (Type[WBModel]): The basket class associated with the metric.
+            val_date (date, optional): The date for the metrics. Defaults to None.
+            basket_label (str, optional): The label to identify the basket in the subquery queryset. Defaults to "id".
+            instrument_label (Union[str, None], optional): The label to identify the instrument in the subquery queryset, if any. Defaults to None.
+
+        Returns:
+            models.QuerySet: The annotated queryset with the metrics included. (with field keys as returned by MetricField.get_fields)
+
+        Side Effects:
+            - Annotates the given queryset with subqueries for the specified metrics.
+        """
+
+        backend_class: AbstractBackend = backend_registry[metric_key, metric_basket_class]
+
+        content_type = ContentType.objects.get_for_model(backend_class.BASKET_MODEL_CLASS)
+        subquery = InstrumentMetric.objects.filter(
+            basket_content_type=content_type,
+            basket_id=models.OuterRef(basket_label),
+            key=metric_key.key,
+            date=val_date,
+        )
+        if instrument_label:
+            subquery = subquery.filter(instrument=models.OuterRef(instrument_label))
+        queryset = queryset.annotate(
+            **{
+                k: models.Subquery(
+                    subquery.annotate(
+                        casted_metric=Cast(
+                            NullIf(models.F("metrics__" + v), models.Value("null")),
+                            output_field=models.FloatField(),
+                        )
+                    ).values("casted_metric")[:1]
+                )
+                for k, v in metric_key.subfields_filter_map.items()
+            }
+        )
+
+        # annotate extra subfields
+        queryset = queryset.annotate(
+            **{
+                f"{metric_key.key}_{extra_subfield.key}": models.Subquery(
+                    subquery.annotate(
+                        casted_metric=Cast(
+                            KeyTextTransform(extra_subfield.key, "metrics"), output_field=extra_subfield.field_type()
+                        ),
+                    ).values("casted_metric")[:1]
+                )
+                for extra_subfield in metric_key.extra_subfields
+            }
+        )
+
+        return queryset
+
+    @classmethod
+    def get_endpoint_basename(cls) -> str:
+        return "metric:instrumentmetric"
+
+    @classmethod
+    def get_representation_endpoint(cls) -> str:
+        return "metric:instrumentmetricrepresentation-list"
+
+    @classmethod
+    def get_representation_value_key(cls) -> str:
+        return "id"
+
+    @classmethod
+    def get_representation_label_key(cls) -> str:
+        return "{{key}} - {{date}} - {{basket_repr}}"
+
+    class Meta:
+        verbose_name = "Instrument Metric"
+        verbose_name_plural = "Instrument Metrics"
+        constraints = [
+            models.UniqueConstraint(
+                name="unique_instrument_metric",
+                fields=["basket_content_type", "basket_id", "instrument", "date", "key"],
+                nulls_distinct=False,
+            )
+        ]
+        indexes = [
+            models.Index(
+                fields=["basket_content_type", "basket_id", "key", "date"],
+            )
+        ]

wbfdm/contrib/metric/orchestrators.py

@@ -0,0 +1,94 @@
+from contextlib import suppress
+from datetime import date
+from typing import Any, Generator
+
+from django.db import models
+from django.db.utils import DataError
+from tqdm import tqdm
+
+from .backends.base import AbstractBackend
+from .dto import Metric
+from .exceptions import MetricInvalidParameterException
+from .models import InstrumentMetric
+from .registry import backend_registry
+
+
+class MetricOrchestrator:
+    """
+    Orchestrator class responsible to gather the proper metric backends and their respective queryset of baskets
+
+    For each parameters, the orchestrator is responsible to compute and store the generated metrics
+    """
+
+    def __init__(self, val_date: date | None, key: str | None = None, basket: Any | None = None, **kwargs):
+        self.basket = basket
+        self.val_date = val_date
+        self.backend_classes = backend_registry.get(metric_key=key, model_class=basket.__class__ if basket else None)
+
+    def _iterate_baskets(self, backend: AbstractBackend) -> models.QuerySet:
+        """
+        Retrieve and filter the queryset of baskets from the given backend.
+
+        Args:
+            backend (AbstractBackend): The backend from which to retrieve the queryset of baskets.
+
+        Returns:
+            models.QuerySet: A queryset of baskets, filtered by the current basket if specified.
+        """
+        qs = backend.get_queryset()
+        if self.basket:
+            qs = qs.filter(id=self.basket.id)
+        return qs
+
+    def _get_parameters(self) -> list[tuple[AbstractBackend, Any]]:
+        """
+        Generate a list of parameters consisting of metric backend instances and baskets.
+
+        This method initializes backends for each backend class with the validation date,
+        iterates through the baskets, and creates a list of tuples containing the backend
+        and each basket.
+
+        Returns:
+            List[Tuple[AbstractBackend, Any]]: A list of tuples where each tuple contains
+            an instance of a instantiated metric backend and a corresponding basket to compute metrics from.
+        """
+        parameters = []
+        for backend_class in self.backend_classes:
+            backend = backend_class(self.val_date)
+            for basket in self._iterate_baskets(backend):
+                parameters.append((backend, basket))
+        return parameters
+
+    def get_results(self, debug: bool = False) -> Generator[Metric, None, None]:
+        """
+        Compute and yield metrics based on the parameters obtained from `_get_parameters`
+
+        Args:
+            debug (bool, optional): If True, wraps the parameters list in a tqdm generator for progress tracking. Defaults to False.
+
+        Yields:
+            Metric: Each DTO metric computed by the backend for the given baskets.
+        """
+        parameters = self._get_parameters()
+        if debug:
+            # if debug mode is enabled, we wrap the parameters list into a tqdm generator
+            parameters = tqdm(parameters)
+        for parameters in parameters:
+            with suppress(MetricInvalidParameterException):
+                yield from parameters[0].compute_metrics(parameters[1])
+
+    def process(self, debug: bool = False):
+        """
+        Process the computation of metrics and update or create InstrumentMetric instances accordingly.
+
+        Side Effects:
+            - Updates or creates InstrumentMetric instances based on the computed metrics.
+        """
+        # we need to see how one threaded loop is fast enough
+        errors = []
+        for metric in self.get_results(debug=debug):
+            try:
+                InstrumentMetric.update_or_create_from_metric(metric)
+            except DataError:
+                errors.append(metric)
+        return errors

wbfdm/contrib/metric/registry.py

@@ -0,0 +1,80 @@
+from collections import OrderedDict, defaultdict
+from typing import TYPE_CHECKING, Type
+
+from django.db.models import Model
+
+from .dto import MetricKey
+
+if TYPE_CHECKING:
+    from wbfdm.contrib.metric.backends.base import AbstractBackend
+
+
+# The cached metric backend registry as a dictionary of dictionary (e.g. {key: {model_class: {backend_class}})
+class BackendRegistry:
+    _key_label_map: dict[str, str] = dict()
+    _metric_key_map: dict[str, MetricKey] = dict()
+    _overrode_model_class: dict[MetricKey, list[Type[Model]]] = defaultdict(list)
+    registry: dict[MetricKey, OrderedDict] = defaultdict(OrderedDict)
+
+    def __getitem__(self, index):
+        key = index[0]
+        if isinstance(key, str):
+            key = self._metric_key_map[key]
+        return self.registry[key][index[1]]
+
+    def get_choices(self, keys: list[MetricKey] | None = None) -> list[tuple[str, str]]:
+        if not keys:
+            keys = list(self.registry.keys())
+        return [(key.key, key.label) for key in keys]
+
+    def set(
+        self,
+        metric_key: MetricKey,
+        model_class: Type[Model],
+        backend: Type,
+        move_first: bool = False,
+        override_backend: bool = False,
+    ):
+        self._metric_key_map[metric_key.key] = metric_key
+        self._key_label_map[metric_key.key] = metric_key.label
+        # we ensure that the registered key and model class pair are not already registered and lock as override
+        if model_class not in self._overrode_model_class[metric_key]:
+            self.registry[metric_key][model_class] = backend
+            if move_first:
+                self.registry[metric_key].move_to_end(model_class, last=False)
+        if override_backend:
+            self._overrode_model_class[metric_key].append(model_class)
+
+    def get(
+        self, metric_key: MetricKey | str | None = None, model_class: Type[Model] | None = None
+    ) -> list[Type["AbstractBackend"]]:
+        # Initialize the backend classes list to iterate over
+        registry = self.registry
+        if isinstance(metric_key, str):
+            metric_key = self._metric_key_map[metric_key]
+        if metric_key:
+            # if key is provided, we return only the backends associated with that metric key
+            if metric_key not in self.registry:
+                raise ValueError(f"key {metric_key.key} does not belong to a registered backend")
+            registry = {metric_key: registry[metric_key]}
+
+        if model_class:
+            # if the metric needs to be computed only for a specific basket, we filter out the backend classes related to his basket class
+            registry = {
+                key: {_class: _dataclass}
+                for key, d in registry.items()
+                for _class, _dataclass in d.items()
+                if _class == model_class
+            }
+        backends = []
+        for _, ordered_dict in registry.items():
+            for _, backend in ordered_dict.items():
+                if backend not in backends:
+                    backends.append(backend)
+        return backends
+
+    def keys(self):
+        return self._metric_key_map.keys()
+
+
+backend_registry = BackendRegistry()

wbfdm/contrib/metric/serializers.py

@@ -0,0 +1,44 @@
+from rest_framework.reverse import reverse
+from wbcore import serializers as wb_serializers
+from wbcore.content_type.serializers import ContentTypeRepresentationSerializer
+from wbfdm.contrib.metric.models import InstrumentMetric
+from wbfdm.serializers.instruments import InstrumentRepresentationSerializer
+
+
+class InstrumentMetricRepresentationSerializer(wb_serializers.RepresentationSerializer):
+    class Meta:
+        model = InstrumentMetric
+        fields = ("id", "key", "date", "basket_repr")
+
+
+class InstrumentMetricModelSerializer(wb_serializers.ModelSerializer):
+    _basket_content_type = ContentTypeRepresentationSerializer(source="basket_content_type")
+    _instrument = InstrumentRepresentationSerializer(source="instrument")
+    _parent_metric = InstrumentMetricRepresentationSerializer(source="parent_metric")
+    _group_key = wb_serializers.CharField(read_only=True)
+
+    @wb_serializers.register_resource()
+    def additional_resources(self, instance, request, user):
+        return {
+            "children_metrics": f'{reverse("metric:instrumentmetric-list", args=[], request=request)}?parent_metric={instance.id}'
+        }
+
+    class Meta:
+        model = InstrumentMetric
+        fields = (
+            "id",
+            "basket_content_type",
+            "_basket_content_type",
+            "basket_repr",
+            "basket_id",
+            "instrument",
+            "_instrument",
+            "date",
+            "key",
+            "metrics",
+            "parent_metric",
+            "_parent_metric",
+            "_group_key",
+            "_additional_resources",
+        )
+        read_only_fields = fields

wbfdm/contrib/metric/tasks.py

@@ -0,0 +1,27 @@
+from datetime import date
+
+from celery import shared_task
+from django.contrib.contenttypes.models import ContentType
+from wbfdm.contrib.metric.orchestrators import MetricOrchestrator
+
+
+@shared_task(queue="portfolio")
+def compute_metrics_as_task(
+    val_date: date | None = None,
+    key: str | None = None,
+    basket_content_type_id: int | None = None,
+    basket_id: int | None = None,
+    **kwargs
+):
+    """
+    Run the orchestrator as a async task periodically for all keys and baskets
+
+    Args:
+        val_date (date): Compute the metric for the given date (Default to None). If None, let the metric backends decide what is the last valide date to use
+
+    """
+    basket = None
+    if basket_content_type_id and basket_id:
+        basket = ContentType.objects.get(id=basket_content_type_id).get_object_for_this_type(pk=basket_id)
+    routine = MetricOrchestrator(val_date, key=key, basket=basket, **kwargs)
+    routine.process()