wbfdm 1.46.9__tar.gz → 1.46.11__tar.gz

{wbfdm-1.46.9 → wbfdm-1.46.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wbfdm
-Version: 1.46.9
+Version: 1.46.11
 Summary: The workbench module ensures rapid access to diverse financial data (market, fundamental, forecasts, ESG), with features for storing instruments, classifying them, and conducting financial analysis.
 Author-email: Christopher Wittlinger <c.wittlinger@stainly.com>
 Requires-Dist: roman==4.*

{wbfdm-1.46.9 → wbfdm-1.46.11}/wbfdm/factories/instrument_prices.py

@@ -42,7 +42,9 @@ class InstrumentPriceFactory(factory.django.DjangoModelFactory):
         )[0]
     )
 
-    net_value = factory.Faker("pydecimal", min_value=10, max_value=100, right_digits=6)
+    net_value = factory.Faker(
+        "pydecimal", min_value=90, max_value=110, right_digits=6
+    )  # we narrow down the range to simulate more realistic returns
     gross_value = factory.LazyAttribute(lambda o: o.net_value + Decimal(random.random()))
 
     calculated = False

{wbfdm-1.46.9 → wbfdm-1.46.11}/wbfdm/models/instruments/instrument_prices.py

@@ -408,16 +408,17 @@ class InstrumentPrice(
                 )
 
     def compute_and_update_statistics(self, min_period: int = 20):
-        prices_df = (
+        df = (
             pd.DataFrame(
                 InstrumentPrice.objects.filter_only_valid_prices()
                 .filter(instrument=self.instrument, date__lte=self.date)
-                .values_list("date", "net_value"),
-                columns=["date", "net_value"],
+                .values_list("date", "net_value", "volume"),
+                columns=["date", "net_value", "volume"],
             )
             .set_index("date")
-            .sort_index()["net_value"]
+            .sort_index()
         )
+        prices_df = df["net_value"]
         if not prices_df.empty and prices_df.shape[0] >= min_period:
             financial_statistics = FinancialStatistics(prices_df)
             min_date = prices_df.index.min()
@@ -451,6 +452,9 @@ class InstrumentPrice(
                     self.correlation = correlation
 
             self.annualized_daily_volatility = financial_statistics.get_annualized_daily_volatility()
+        if not (volume_df := df["volume"]).empty:
+            self.volume_50d = volume_df.tail(50).mean()
+            self.volume_200d = volume_df.tail(200).mean()
 
     @classmethod
     def subquery_closest_value(

{wbfdm-1.46.9 → wbfdm-1.46.11}/wbfdm/models/instruments/instruments.py

@@ -844,7 +844,7 @@ class Instrument(ComplexToStringMixin, TagModelMixin, ImportMixin, InstrumentPMS
     def technical_benchmark_analysis(self, from_date: date | None = None, to_date: date | None = None):
         return TechnicalAnalysis.init_close_from_instrument(self, from_date, to_date)
 
-    def get_price_objects(self, start: date | None = None, end: date | None = None, clear: bool = False):
+    def import_prices(self, start: date | None = None, end: date | None = None, **kwargs):
         if not self.is_leaf_node():
             raise ValueError("Cannot import price on a non-leaf node")
         if not start:
@@ -861,11 +861,8 @@ class Instrument(ComplexToStringMixin, TagModelMixin, ImportMixin, InstrumentPMS
         # we detect when was the last date price imported before start and switch the start date from there
         with suppress(ObjectDoesNotExist):
             start = self.prices.filter(date__lte=start).latest("date").date
-        yield from self._get_price_objects(start, end, clear=clear)
-
-    def import_prices(self, start: date | None = None, end: date | None = None, clear: bool = False):
         # Import instrument prices
-        objs = list(self.get_price_objects(start=start, end=end, clear=clear))
+        objs = list(self.__class__.objects.filter(id=self.id).get_instrument_prices_from_market_data(start, end))
         self.bulk_save_instrument_prices(objs)
         # compute daily statistics & performances
         self.update_last_valuation_date()

{wbfdm-1.46.9 → wbfdm-1.46.11}/wbfdm/models/instruments/mixin/instruments.py

@@ -1,15 +1,12 @@
 import calendar
-import math
 from collections import defaultdict
-from contextlib import suppress
-from datetime import date, timedelta
+from datetime import date
 from decimal import Decimal
-from typing import Dict, Iterable, Optional
+from typing import Dict, Optional
 
 import numpy as np
 import pandas as pd
 from pandas.tseries.offsets import BDay
-from wbcore.contrib.currency.models import CurrencyFXRates
 
 from wbfdm.analysis.financial_analysis.financial_statistics_analysis import (
     FinancialStatistics,
@@ -199,91 +196,6 @@ class InstrumentPMSMixin:
 
         return df
 
-    def _get_price_objects(self, from_date: date, to_date: date, clear: bool = False) -> Iterable[InstrumentPrice]:
-        df = pd.DataFrame(
-            self.__class__.objects.filter(id=self.id).dl.market_data(
-                from_date=from_date
-                - timedelta(
-                    days=90
-                ),  # we make sure to at least import the last 80 days to be sure to be able to compute the volume 50d
-                to_date=to_date,
-                values=[MarketData.CLOSE, MarketData.VOLUME, MarketData.MARKET_CAPITALIZATION],
-            )
-        )
-        if not df.empty:
-            df["calculated"] = False
-            df = df.set_index("valuation_date").sort_index()
-
-            # # if market cap is not found, maybe we have better chance on the primary exhange
-            if not df.market_capitalization.notnull().any() and self.parent and (company := self.parent.get_root()):
-                with suppress(KeyError):
-                    df["market_capitalization"] = pd.DataFrame(
-                        self.__class__.objects.filter(id=company.id).dl.market_data(
-                            from_date=from_date,
-                            to_date=to_date,
-                            values=[MarketData.MARKET_CAPITALIZATION],
-                        )
-                    ).set_index("valuation_date")["market_capitalization"]
-
-            ts = pd.date_range(df.index.min(), df.index.max(), freq="B")
-            # fill forward missing data
-            df = df.reindex(ts)
-            df[["close", "market_capitalization"]] = df[["close", "market_capitalization"]].astype(float).ffill()
-            df.volume = df.volume.astype(float).fillna(0)
-            df.calculated = df.calculated.astype(bool).fillna(
-                True
-            )  # we do not ffill calculated but set the to True to mark them as "estimated"/"not real"
-            df["volume_50d"] = df["volume"].rolling(50).mean()
-            df = df[df.index.date >= from_date]  # we import only from the requested from_date
-            df = df.reset_index().dropna(subset=["index", "close"])
-            df = df.replace([np.inf, -np.inf, np.nan], None)
-
-            for row in df.to_dict("records"):
-                if (_date := row.get("index")) and (close := row.get("close", None)):
-                    # we make sure data does not exist 10 digits (for db constraint)
-                    if int(math.log10(close)) + 1 < 10:
-                        try:
-                            try:
-                                p = InstrumentPrice.objects.get(instrument=self, date=_date, calculated=False)
-                            except InstrumentPrice.DoesNotExist:
-                                p = InstrumentPrice.objects.get(instrument=self, date=_date, calculated=True)
-
-                            # update only if net value is different with existing instrument price
-                            if (
-                                round(p.net_value, 2) != round(close, 2)
-                                or (p.market_capitalization != row.get("market_capitalization"))
-                                or (p.volume != row.get("volume"))
-                                or (p.calculated != row.get("calculated"))
-                                or clear
-                            ):
-                                p.net_value = close
-                                p.gross_value = close
-                                p.calculated = row["calculated"]
-                                p.volume = row.get("volume", p.volume)
-                                p.volume_50d = row.get("volume_50d", p.volume_50d)
-                                p.market_capitalization = row.get("market_capitalization", p.market_capitalization)
-                                p.market_capitalization_consolidated = p.market_capitalization
-                                p.set_dynamic_field(False)
-                                p.id = None
-                                yield p
-                        except InstrumentPrice.DoesNotExist:
-                            with suppress(CurrencyFXRates.DoesNotExist):
-                                p = InstrumentPrice(
-                                    currency_fx_rate_to_usd=CurrencyFXRates.objects.get(  # we need to get the currency rate because we bulk create the object, and thus save is not called
-                                        date=_date, currency=self.currency
-                                    ),
-                                    instrument=self,
-                                    date=_date,
-                                    calculated=row["calculated"],
-                                    net_value=close,
-                                    gross_value=close,
-                                    volume=row.get("volume", None),
-                                    market_capitalization=row.get("market_capitalization", None),
-                                    volume_50d=row.get("volume_50d", None),
-                                )
-                                p.set_dynamic_field(False)
-                                yield p
-
     @classmethod
     def bulk_save_instrument_prices(cls, objs):
         InstrumentPrice.objects.bulk_create(
@@ -297,7 +209,6 @@ class InstrumentPMSMixin:
                 "market_capitalization",
                 "market_capitalization_consolidated",
                 "calculated",
-                "volume_50d",
             ],
             batch_size=1000,
         )

wbfdm-1.46.11/wbfdm/models/instruments/querysets.py

@@ -0,0 +1,156 @@
+import math
+from contextlib import suppress
+from datetime import date
+
+import numpy as np
+import pandas as pd
+from django.db.models import (
+    AutoField,
+    Exists,
+    ExpressionWrapper,
+    F,
+    OuterRef,
+    QuerySet,
+    Subquery,
+)
+from django.db.models.functions import Coalesce
+from wbcore.contrib.currency.models import CurrencyFXRates
+
+from wbfdm.enums import MarketData
+
+
+class InstrumentQuerySet(QuerySet):
+    def annotate_classification_for_group(
+        self, classification_group, classification_height: int = 0, **kwargs
+    ) -> QuerySet:
+        return classification_group.annotate_queryset(
+            self, classification_height, "", annotation_label="ancestor_classifications", **kwargs
+        )
+
+    def annotate_base_data(self):
+        base_qs = InstrumentQuerySet(self.model, using=self._db)
+        return self.annotate(
+            is_investable=~Exists(base_qs.filter(parent=OuterRef("pk"))),
+            root=Subquery(base_qs.filter(tree_id=OuterRef("tree_id"), level=0).values("id")[:1]),
+            primary_security=ExpressionWrapper(
+                Coalesce(
+                    Subquery(
+                        base_qs.filter(
+                            parent=OuterRef("pk"),
+                            is_primary=True,
+                            is_security=True,
+                        ).values("id")[:1]
+                    ),
+                    F("id"),
+                ),
+                output_field=AutoField(),
+            ),
+            primary_quote=ExpressionWrapper(
+                Coalesce(
+                    Subquery(
+                        base_qs.filter(
+                            parent=OuterRef("primary_security"),
+                            is_primary=True,
+                        ).values("id")[:1]
+                    ),
+                    F("primary_security"),
+                ),
+                output_field=AutoField(),
+            ),
+        )
+
+    def annotate_all(self):
+        return self.annotate_base_data()
+
+    @property
+    def dl(self):
+        """Provides access to the dataloader proxy for the entities in the QuerySet.
+
+        This method allows for easy retrieval of the DataloaderProxy instance
+        associated with the QuerySet. It enables the utilization of dataloader
+        functionalities directly from the QuerySet, facilitating data fetching and
+        processing tasks.
+
+        Returns:
+            DataloaderProxy: An instance of DataloaderProxy associated with the
+                entities in the QuerySet.
+        """
+        return self.model.dl_proxy(self)
+
+    def get_instrument_prices_from_market_data(self, from_date: date, to_date: date):
+        from wbfdm.models import InstrumentPrice
+
+        def _dict_to_object(instrument, row):
+            if (
+                (price_date := row.get("date"))
+                and (close := row.get("close", None))
+                and int(math.log10(close)) + 1 < 10
+            ):
+                try:
+                    try:
+                        p = InstrumentPrice.objects.get(instrument=instrument, date=price_date, calculated=False)
+                    except InstrumentPrice.DoesNotExist:
+                        p = InstrumentPrice.objects.get(instrument=instrument, date=price_date, calculated=True)
+                    p.net_value = close
+                    p.gross_value = close
+                    p.calculated = row["calculated"]
+                    p.volume = row.get("volume", p.volume)
+                    p.market_capitalization = row.get("market_capitalization", p.market_capitalization)
+                    p.market_capitalization_consolidated = p.market_capitalization
+                    p.set_dynamic_field(False)
+                    p.id = None
+                    return p
+                except InstrumentPrice.DoesNotExist:
+                    with suppress(CurrencyFXRates.DoesNotExist):
+                        p = InstrumentPrice(
+                            currency_fx_rate_to_usd=CurrencyFXRates.objects.get(
+                                # we need to get the currency rate because we bulk create the object, and thus save is not called
+                                date=price_date,
+                                currency=instrument.currency,
+                            ),
+                            instrument=instrument,
+                            date=price_date,
+                            calculated=row["calculated"],
+                            net_value=close,
+                            gross_value=close,
+                            volume=row.get("volume", None),
+                            market_capitalization=row.get("market_capitalization", None),
+                        )
+                        p.set_dynamic_field(False)
+                        return p
+
+        df = pd.DataFrame(
+            self.dl.market_data(
+                from_date=from_date,
+                to_date=to_date,
+                values=[MarketData.CLOSE, MarketData.VOLUME, MarketData.MARKET_CAPITALIZATION],
+            )
+        )
+        if not df.empty:
+            df["valuation_date"] = pd.to_datetime(df["valuation_date"])
+            df = df.rename(columns={"valuation_date": "date"}).set_index(["instrument_id", "date"]).sort_index()
+            df = df.drop(columns=df.columns.difference(["calculated", "close", "market_capitalization", "volume"]))
+            df["calculated"] = False
+            idx = pd.MultiIndex.from_product(
+                [
+                    df.index.levels[0],
+                    pd.date_range(
+                        df.index.get_level_values("date").min(), df.index.get_level_values("date").max(), freq="B"
+                    ),
+                ],
+                names=["instrument_id", "date"],
+            )
+            df = df.reindex(idx)
+            df[["close", "market_capitalization"]] = df[["close", "market_capitalization"]].astype(float).ffill()
+            df.volume = df.volume.astype(float).fillna(0)
+            df.calculated = df.calculated.astype(bool).fillna(
+                True
+            )  # we do not ffill calculated but set the to True to mark them as "estimated"/"not real"
+            df = df.reset_index("date").dropna(subset=["close"])
+            df = df.replace([np.inf, -np.inf, np.nan], None)
+
+            for instrument_id, dff in df.groupby(level=0):
+                instrument = self.get(id=instrument_id)
+                yield from filter(
+                    lambda x: x, map(lambda row: _dict_to_object(instrument, row), dff.to_dict("records"))
+                )

{wbfdm-1.46.9 → wbfdm-1.46.11}/wbfdm/tasks.py

@@ -19,7 +19,10 @@ from .signals import investable_universe_updated
 
 @shared_task(queue="portfolio")
 def update_of_investable_universe_data(
-    start: date | None = None, end: date | None = None, clear: bool = False, with_background_tasks: bool = True
+    start: date | None = None,
+    end: date | None = None,
+    with_background_tasks: bool = True,
+    instrument_ids: list[int] | None = None,
 ):
     """
     Update the investable universe data on a daily basis.
@@ -27,6 +30,8 @@ def update_of_investable_universe_data(
     Parameters:
     - start (date | None): The start date for updating data. If None, defaults to three business days before 'end'.
     - end (date | None): The end date for updating data. If None, defaults to the current date.
+    - with_background_tasks (bool | True): If true, will trigger the post import background tasks automatically.
+    - instrument_ids (list[int] | None): if specified, narrow down the instrument queryset with this list ids
 
     Notes:
     - The function resets the investable universe by marking all instruments as not in the investable universe.
@@ -51,12 +56,9 @@ def update_of_investable_universe_data(
         Q(is_managed=True)
         | Q(dl_parameters__market_data__path="wbfdm.contrib.internal.dataloaders.market_data.MarketDataDataloader")
     )  # we exclude product and index managed to avoid circular import
-    prices = []
-    for instrument in tqdm(instruments, total=instruments.count()):
-        try:
-            prices.extend(list(instrument.get_price_objects(start=start, end=end, clear=clear)))
-        except Exception as e:
-            print(f"Connection error while processing instrument {instrument}: {str(e)}")  # noqa
+    if instrument_ids:
+        instruments = instruments.filter(id__in=instrument_ids)
+    prices = list(instruments.get_instrument_prices_from_market_data(start, end))
     Instrument.bulk_save_instrument_prices(prices)
     investable_universe_updated.send(sender=Instrument, end_date=end)
     if with_background_tasks:
@@ -105,5 +107,7 @@ def daily_update_instrument_price_statistics(from_date: date = None, to_date: da
         p.compute_and_update_statistics()
         objs.append(p)
     InstrumentPrice.objects.bulk_update(
-        objs, fields=["sharpe_ratio", "correlation", "beta", "annualized_daily_volatility"], batch_size=1000
+        objs,
+        fields=["sharpe_ratio", "correlation", "beta", "annualized_daily_volatility", "volume_50d", "volume_200d"],
+        batch_size=1000,
     )

{wbfdm-1.46.9 → wbfdm-1.46.11}/wbfdm/tests/models/test_instruments.py

@@ -157,12 +157,14 @@ class TestInstrumentModel:
                 "close": 1,
                 "volume": 1,
                 "market_capitalization": 1,
+                "instrument_id": instrument.id,
             },
             {
                 "valuation_date": to_date,
                 "close": 2,
                 "volume": 2,
                 "market_capitalization": 2,
+                "instrument_id": instrument.id,
             },
         ]
         instrument.import_prices(from_date, to_date + timedelta(days=1))

wbfdm-1.46.9/wbfdm/models/instruments/querysets.py

@@ -1,73 +0,0 @@
-from django.db.models import (
-    AutoField,
-    Exists,
-    ExpressionWrapper,
-    F,
-    OuterRef,
-    QuerySet,
-    Subquery,
-)
-from django.db.models.functions import Coalesce
-
-
-class InstrumentQuerySet(QuerySet):
-    def annotate_classification_for_group(
-        self, classification_group, classification_height: int = 0, **kwargs
-    ) -> QuerySet:
-        return classification_group.annotate_queryset(
-            self, classification_height, "", annotation_label="ancestor_classifications", **kwargs
-        )
-
-    def annotate_base_data(self):
-        base_qs = InstrumentQuerySet(self.model, using=self._db)
-        return self.annotate(
-            is_investable=~Exists(base_qs.filter(parent=OuterRef("pk"))),
-            root=Subquery(base_qs.filter(tree_id=OuterRef("tree_id"), level=0).values("id")[:1]),
-            primary_security=ExpressionWrapper(
-                Coalesce(
-                    Subquery(
-                        base_qs.filter(
-                            parent=OuterRef("pk"),
-                            is_primary=True,
-                            is_security=True,
-                        ).values(
-                            "id"
-                        )[:1]
-                    ),
-                    F("id"),
-                ),
-                output_field=AutoField(),
-            ),
-            primary_quote=ExpressionWrapper(
-                Coalesce(
-                    Subquery(
-                        base_qs.filter(
-                            parent=OuterRef("primary_security"),
-                            is_primary=True,
-                        ).values(
-                            "id"
-                        )[:1]
-                    ),
-                    F("primary_security"),
-                ),
-                output_field=AutoField(),
-            ),
-        )
-
-    def annotate_all(self):
-        return self.annotate_base_data()
-
-    @property
-    def dl(self):
-        """Provides access to the dataloader proxy for the entities in the QuerySet.
-
-        This method allows for easy retrieval of the DataloaderProxy instance
-        associated with the QuerySet. It enables the utilization of dataloader
-        functionalities directly from the QuerySet, facilitating data fetching and
-        processing tasks.
-
-        Returns:
-            DataloaderProxy: An instance of DataloaderProxy associated with the
-                entities in the QuerySet.
-        """
-        return self.model.dl_proxy(self)