wbfdm 1.54.22__tar.gz → 1.55.4__tar.gz

{wbfdm-1.54.22 → wbfdm-1.55.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wbfdm
-Version: 1.54.22
+Version: 1.55.4
 Summary: The workbench module ensures rapid access to diverse financial data (market, fundamental, forecasts, ESG), with features for storing instruments, classifying them, and conducting financial analysis.
 Author-email: Christopher Wittlinger <c.wittlinger@stainly.com>
 Requires-Dist: roman==4.*

wbfdm-1.55.4/wbfdm/contrib/qa/dataloaders/fx_rates.py

@@ -0,0 +1,49 @@
+from datetime import date, timedelta
+from typing import Iterator
+
+import pypika as pk
+from django.db import connections
+from pypika import Case
+from pypika import functions as fn
+from pypika.enums import Order, SqlTypes
+from wbcore.contrib.dataloader.dataloaders import Dataloader
+from wbcore.contrib.dataloader.utils import dictfetchall
+
+from wbfdm.dataloaders.protocols import FXRateProtocol
+from wbfdm.dataloaders.types import FXRateDict
+
+
+class DatastreamFXRatesDataloader(FXRateProtocol, Dataloader):
+    def fx_rates(
+        self,
+        from_date: date,
+        to_date: date,
+        target_currency: str,
+    ) -> Iterator[FXRateDict]:
+        currencies = list(self.entities.values_list("currency__key", flat=True))
+        # Define tables
+        fx_rate = pk.Table("DS2FxRate")
+        fx_code = pk.Table("DS2FxCode")
+
+        # Base query to get data we always need unconditionally
+        query = (
+            pk.MSSQLQuery.from_(fx_rate)
+            # We join on _codes, which removes all instruments not in _codes - implicit where
+            .join(fx_code)
+            .on(fx_rate.ExRateIntCode == fx_code.ExRateIntCode)
+            .where((fx_rate.ExRateDate >= from_date) & (fx_rate.ExRateDate <= to_date + timedelta(days=1)))
+            .where(
+                (fx_code.ToCurrCode == target_currency)
+                & (fx_code.FromCurrCode.isin(currencies))
+                & (fx_code.RateTypeCode == "SPOT")
+            )
+            .orderby(fx_rate.ExRateDate, order=Order.desc)
+            .select(
+                fn.Cast(fx_rate.ExRateDate, SqlTypes.DATE).as_("fx_date"),
+                fn.Concat(fx_code.FromCurrCode, fx_code.ToCurrCode).as_("currency_pair"),
+                (Case().when(fx_code.FromCurrCode == target_currency, 1).else_(1 / fx_rate.midrate)).as_("fx_rate"),
+            )
+        )
+        with connections["qa"].cursor() as cursor:
+            cursor.execute(query.get_sql())
+            yield from dictfetchall(cursor, FXRateDict)

{wbfdm-1.54.22 → wbfdm-1.55.4}/wbfdm/contrib/qa/dataloaders/market_data.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from contextlib import suppress
 from datetime import date
 from enum import Enum
@@ -7,13 +8,14 @@ from typing import TYPE_CHECKING, Iterator
 
 import pypika as pk
 from django.db import ProgrammingError, connections
-from pypika import Case, Column, MSSQLQuery
+from pypika import Column, MSSQLQuery
 from pypika import functions as fn
 from pypika.enums import Order, SqlTypes
 from pypika.terms import ValueWrapper
 from wbcore.contrib.dataloader.dataloaders import Dataloader
 from wbcore.contrib.dataloader.utils import dictfetchall
 
+from wbfdm.contrib.qa.dataloaders.fx_rates import DatastreamFXRatesDataloader
 from wbfdm.contrib.qa.dataloaders.utils import create_table
 from wbfdm.dataloaders.protocols import MarketDataProtocol
 from wbfdm.dataloaders.types import MarketDataDict
@@ -60,10 +62,19 @@ class DatastreamMarketDataDataloader(MarketDataProtocol, Dataloader):
         Returns:
             Iterator[MarketDataDict]: An iterator of dictionaries conforming to the DailyValuationDict.
         """
-
         lookup = {
             f"{k[0]},{k[1]}": v for k, v in self.entities.values_list("dl_parameters__market_data__parameters", "id")
         }
+        fx_rates = defaultdict(dict)
+        if target_currency:
+            if exact_date:
+                from_date = exact_date
+                to_date = exact_date
+            if from_date and to_date:
+                for fx_rate in DatastreamFXRatesDataloader(self.entities).fx_rates(
+                    from_date, to_date, target_currency
+                ):
+                    fx_rates[fx_rate["currency_pair"]][fx_rate["fx_date"]] = fx_rate["fx_rate"]
 
         # Define tables
         pricing = pk.Table("vw_DS2Pricing")
@@ -90,33 +101,7 @@ class DatastreamMarketDataDataloader(MarketDataProtocol, Dataloader):
             .orderby(pricing.MarketDate, order=Order.desc)
         )
 
-        # if a target currency is required, we join on the fx tables and set the currency to the desired one
-        # otherwise we just set the currency to whatever the currency is from the instrument
-        fx_rate = None
-        if target_currency:
-            query = query.select(ValueWrapper(target_currency).as_("currency"))
-            fx_code = pk.Table("DS2FxCode")
-            fx_rate = pk.Table("DS2FxRate")
-            query = (
-                query.select(
-                    (Case().when(pricing.Currency == target_currency, 1).else_(1 / fx_rate.midrate)).as_("fx_rate")
-                )
-                # Join FX code table matching currencies and ensuring SPOT rate type
-                .left_join(fx_code)
-                .on(
-                    (fx_code.FromCurrCode == pricing.Currency)
-                    & (fx_code.ToCurrCode == target_currency)
-                    & (fx_code.RateTypeCode == "SPOT")
-                )
-                # Join FX rate table matching internal code and date
-                .left_join(fx_rate)
-                .on((fx_rate.ExRateIntCode == fx_code.ExRateIntCode) & (fx_rate.ExRateDate == pricing.MarketDate))
-                # We filter out rows which do not have a proper fx rate (we exclude same currency conversions)
-                .where((Case().when(pricing.Currency == target_currency, 1).else_(fx_rate.midrate).isnotnull()))
-            )
-
-        else:
-            query = query.select(pricing.Currency.as_("currency"))
+        query = query.select(pricing.Currency.as_("currency"))
 
         # if market cap or shares outstanding are required we need to join with an additional table
         if MarketData.MARKET_CAPITALIZATION in values or MarketData.SHARES_OUTSTANDING in values:
@@ -130,9 +115,6 @@ class DatastreamMarketDataDataloader(MarketDataProtocol, Dataloader):
             )
 
             value = pricing_2.Close_
-            if fx_rate and apply_fx_rate:
-                value /= Case().when(pricing_2.Currency == target_currency, 1).else_(fx_rate.midrate)
-
             query = query.select(value.as_("undadjusted_close"))
             query = query.select(
                 MSSQLQuery.from_(num_shares)
@@ -148,9 +130,6 @@ class DatastreamMarketDataDataloader(MarketDataProtocol, Dataloader):
         ):
             ds2_value = DS2MarketData[market_data.name].value
             value = getattr(pricing, ds2_value)
-            if fx_rate and apply_fx_rate and market_data is not MarketData.SHARES_OUTSTANDING:
-                value /= Case().when(pricing.Currency == target_currency, 1).else_(fx_rate.midrate)
-
             query = query.select(value.as_(market_data.value))
 
         # Add conditional where clauses
@@ -189,7 +168,21 @@ class DatastreamMarketDataDataloader(MarketDataProtocol, Dataloader):
 
                 if MarketData.SHARES_OUTSTANDING in values:
                     row["outstanding_shares"] = (row["market_capitalization"] / row["close"]) if row["close"] else None
-                row["fx_rate"] = row.get("fx_rate", 1.0)
+                if target_currency:
+                    try:
+                        if target_currency == row["currency"]:
+                            fx_rate = 1.0
+                        else:
+                            fx_rate = fx_rates[f'{row["currency"]}{target_currency}'][row["valuation_date"]] or 1.0
+                        if apply_fx_rate:
+                            for e in MarketData:
+                                if e != MarketData.SHARES_OUTSTANDING and e.value in row and row[e.value]:
+                                    row[e.value] = row[e.value] * fx_rate
+                            row["currency"] = target_currency
+                        row["fx_rate"] = fx_rate
+                    except KeyError:
+                        # if we don't find the fx rate but we asked for it, we invalid that row and do not return it
+                        continue
                 yield row
 
             cursor.execute(MSSQLQuery.drop_table(mapping).get_sql())

{wbfdm-1.54.22 → wbfdm-1.55.4}/wbfdm/contrib/qa/dataloaders/statements.py

@@ -239,6 +239,10 @@ class RKDStatementsDataloader(StatementsProtocol, Dataloader):
     ) -> Iterator[StatementDataDict]:
         lookup = {k: v for k, v in self.entities.values_list("dl_parameters__statements__parameters", "id")}
         sql = reported_sql if data_type is DataType.REPORTED else standardized_sql
+        if not financials:
+            financials = []
+        external_codes = [RKDFinancial[fin.name].value for fin in financials if fin.name in RKDFinancial.__members__]
+
         query, bind_params = JinjaSql(param_style="format").prepare_query(
             sql,
             {
@@ -249,7 +253,7 @@ class RKDStatementsDataloader(StatementsProtocol, Dataloader):
                 "from_date": from_date,
                 "to_date": to_date,
                 "period_type": period_type.value,
-                "external_codes": [RKDFinancial[fin.name].value for fin in financials or []],
+                "external_codes": external_codes,
             },
         )
         with connections["qa"].cursor() as cursor:
@@ -264,5 +268,8 @@ class RKDStatementsDataloader(StatementsProtocol, Dataloader):
                 row["year"] = int(row["year"] or row["period_end_date"].year)
                 row["instrument_id"] = lookup[row["external_identifier"]]
                 if financials:
-                    row["financial"] = Financial[RKDFinancial(row["external_code"]).name].value
+                    try:
+                        row["financial"] = Financial[RKDFinancial(row["external_code"]).name].value
+                    except (ValueError, KeyError):
+                        continue
                 yield row

{wbfdm-1.54.22 → wbfdm-1.55.4}/wbfdm/dataloaders/protocols.py

@@ -7,6 +7,7 @@ from wbfdm.dataloaders.types import (
     ESGControversyDataDict,
     ESGDataDict,
     FinancialDataDict,
+    FXRateDict,
     MarketDataDict,
     OfficerDataDict,
     ReportDateDataDict,
@@ -36,6 +37,15 @@ class AdjustmentsProtocol(Protocol):
     ) -> Iterator[AdjustmentDataDict]: ...
 
 
+class FXRateProtocol(Protocol):
+    def fx_rates(
+        self,
+        from_date: date,
+        to_date: date,
+        target_currency: str,
+    ) -> Iterator[FXRateDict]: ...
+
+
 class MarketDataProtocol(Protocol):
     def market_data(
         self,

{wbfdm-1.54.22 → wbfdm-1.55.4}/wbfdm/dataloaders/proxies.py

@@ -9,6 +9,7 @@ from wbfdm.dataloaders.protocols import (
     ESGControversyProtocol,
     ESGProtocol,
     FinancialsProtocol,
+    FXRateProtocol,
     MarketDataProtocol,
     OfficersProtocol,
     ReportDateProtocol,
@@ -20,6 +21,7 @@ from wbfdm.dataloaders.types import (
     ESGControversyDataDict,
     ESGDataDict,
     FinancialDataDict,
+    FXRateDict,
     MarketDataDict,
     OfficerDataDict,
     ReportDateDataDict,
@@ -53,7 +55,8 @@ def _market_data_row_parser(row):
 
 class InstrumentDataloaderProxy(
     DataloaderProxy[
-        AdjustmentsProtocol
+        FXRateProtocol
+        | AdjustmentsProtocol
         | MarketDataProtocol
         | CorporateActionsProtocol
         | OfficersProtocol
@@ -68,6 +71,15 @@ class InstrumentDataloaderProxy(
         for dl in self.iterate_dataloaders("reporting_dates"):
             yield from dl.reporting_dates(only_next=only_next)
 
+    def fx_rates(
+        self,
+        from_date: date,
+        to_date: date,
+        target_currency: str,
+    ) -> Iterator[FXRateDict]:
+        for dl in self.iterate_dataloaders("fx_rates"):
+            yield from dl.fx_rates(from_date, to_date, target_currency)
+
     def adjustments(self, from_date: date | None = None, to_date: date | None = None) -> Iterator[AdjustmentDataDict]:
         for dl in self.iterate_dataloaders("adjustments"):
             yield from dl.adjustments(from_date=from_date, to_date=to_date)

{wbfdm-1.54.22 → wbfdm-1.55.4}/wbfdm/dataloaders/types.py

@@ -26,6 +26,12 @@ class BaseDict(TypedDict):
     currency: NotRequired[str]
 
 
+class FXRateDict(TypedDict):
+    currency_pair: str
+    fx_date: date
+    fx_rate: float
+
+
 class MarketDataDict(BaseDict):
     """
     Represents a dictionary for daily valuation data.

{wbfdm-1.54.22 → wbfdm-1.55.4}/wbfdm/factories/instruments.py

@@ -21,7 +21,7 @@ class InstrumentFactory(factory.django.DjangoModelFactory):
     instrument_type = factory.SubFactory(InstrumentTypeFactory)
     inception_date = factory.Faker("past_date")
     delisted_date = None
-    currency = factory.SubFactory("wbcore.contrib.currency.factories.CurrencyFactory")
+    currency = factory.SubFactory("wbcore.contrib.currency.factories.CurrencyUSDFactory")
     country = factory.SubFactory("wbcore.contrib.geography.factories.CountryFactory")
     exchange = factory.SubFactory("wbfdm.factories.exchanges.ExchangeFactory")
 
@@ -54,7 +54,7 @@ class InstrumentFactory(factory.django.DjangoModelFactory):
 
 class CashFactory(factory.django.DjangoModelFactory):
     is_cash = True
-    currency = factory.SubFactory("wbcore.contrib.currency.factories.CurrencyFactory")
+    currency = factory.SubFactory("wbcore.contrib.currency.factories.CurrencyUSDFactory")
     name = factory.LazyAttribute(lambda o: o.currency.title)
     instrument_type = factory.LazyAttribute(
         lambda o: InstrumentType.objects.get_or_create(key="cash", defaults={"name": "Cash", "short_name": "Cash"})[0]

{wbfdm-1.54.22 → wbfdm-1.55.4}/wbfdm/import_export/handlers/option.py

@@ -26,7 +26,7 @@ class OptionAggregateImportHandler(ImportExportHandler):
 
     def _get_instance(self, data: Dict[str, Any], history: Optional[models.QuerySet] = None, **kwargs) -> models.Model:
         with suppress(ObjectDoesNotExist):
-            return self.model.objects.filter(instrument=data["instrument"], date=data["date"], type=data["type"])
+            return self.model.objects.get(instrument=data["instrument"], date=data["date"], type=data["type"])
 
 
 class OptionImportHandler(ImportExportHandler):
@@ -46,7 +46,7 @@ class OptionImportHandler(ImportExportHandler):
 
     def _get_instance(self, data: Dict[str, Any], history: Optional[models.QuerySet] = None, **kwargs) -> models.Model:
         with suppress(ObjectDoesNotExist):
-            return self.model.objects.filter(
+            return self.model.objects.get(
                 instrument=data["instrument"],
                 contract_identifier=data["contract_identifier"],
                 date=data["date"],

wbfdm-1.55.4/wbfdm/migrations/0032_alter_instrumentprice_outstanding_shares.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.14 on 2025-08-27 07:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('wbfdm', '0031_exchange_apply_round_lot_size_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='instrumentprice',
+            name='outstanding_shares',
+            field=models.DecimalField(blank=True, decimal_places=4, help_text='The amount of outstanding share for this instrument', max_digits=16, null=True, verbose_name='Outstanding Shares'),
+        ),
+    ]

{wbfdm-1.54.22 → wbfdm-1.55.4}/wbfdm/models/instruments/instrument_prices.py

@@ -195,9 +195,11 @@ class InstrumentPrice(
         verbose_name="Value (Gross)",
     )  # TODO: I think we need to remove this field that is not really used here.
 
-    outstanding_shares = DynamicDecimalField(
+    outstanding_shares = models.DecimalField(
         decimal_places=4,
         max_digits=16,
+        blank=True,
+        null=True,
         verbose_name="Outstanding Shares",
         help_text="The amount of outstanding share for this instrument",
     )

{wbfdm-1.54.22 → wbfdm-1.55.4}/wbfdm/models/instruments/mixin/financials_computed.py

@@ -697,10 +697,6 @@ import pandas as pd
 #
 #
 class InstrumentPriceComputedMixin:
-    def _compute_outstanding_shares(self):
-        if self.outstanding_shares is None and (previous_price := self.previous_price):
-            return previous_price.outstanding_shares
-
     def _compute_outstanding_shares_consolidated(self):
         if self.outstanding_shares_consolidated is None and self.outstanding_shares is not None:
             return self.outstanding_shares

{wbfdm-1.54.22 → wbfdm-1.55.4}/wbfdm/models/instruments/querysets.py

@@ -200,7 +200,7 @@ class InstrumentQuerySet(QuerySet):
         padded_from_date = from_date - timedelta(days=15)
         padded_to_date = to_date + timedelta(days=3)
         logger.info(
-            f"Loading returns from {from_date:%Y-%m-%d} (padded to {padded_from_date:%Y-%m-%d}) to {to_date:%Y-%m-%d} (padded to {padded_to_date:%Y-%m-%d})"
+            f"Loading returns from {from_date:%Y-%m-%d} (padded to {padded_from_date:%Y-%m-%d}) to {to_date:%Y-%m-%d} (padded to {padded_to_date:%Y-%m-%d}) for {self.count()} instruments"
         )
 
         if use_dl:
@@ -248,13 +248,12 @@ class InstrumentQuerySet(QuerySet):
                 (df.index <= pd.Timestamp(to_date)) & (df.index >= pd.Timestamp(from_date))
             ]  # ensure the returned df corresponds to requested date range
             prices_df = df["close"]
-
             if "fx_rate" in df.columns:
                 fx_rate_df = df["fx_rate"].fillna(1.0)
             else:
                 fx_rate_df = pd.DataFrame(np.ones(prices_df.shape), index=prices_df.index, columns=prices_df.columns)
             returns = prices_to_returns(fx_rate_df * prices_df, drop_inceptions_nan=False, fill_nan=True)
-            return {ts.date(): row for ts, row in prices_df.to_dict("index").items()}, returns.replace(
-                [np.inf, -np.inf, np.nan], 0
-            )
+            return {
+                ts.date(): row for ts, row in prices_df.replace([np.nan], None).to_dict("index").items()
+            }, returns.replace([np.inf, -np.inf, np.nan], 0)
         return {}, pd.DataFrame()

{wbfdm-1.54.22 → wbfdm-1.55.4}/wbfdm/tasks.py

@@ -1,8 +1,9 @@
+from contextlib import suppress
 from datetime import date, timedelta
 
 from celery import shared_task
 from django.db import transaction
-from django.db.models import Q
+from django.db.models import ProtectedError, Q
 from pandas.tseries.offsets import BDay
 from tqdm import tqdm
 
@@ -89,7 +90,8 @@ def full_synchronization_as_task():
         (Q(name="") & Q(name_repr="")) | (Q(source__in=["qa-ds2-security", "qa-ds2-quote"]) & Q(parent__isnull=True))
     )
     for instrument in qs:
-        instrument.delete()
+        with suppress(ProtectedError):
+            instrument.delete()
     initialize_exchanges()
     initialize_instruments()
     with transaction.atomic():

{wbfdm-1.54.22 → wbfdm-1.55.4}/wbfdm/tests/models/test_instrument_prices.py

@@ -1,10 +1,8 @@
 from datetime import date
-from decimal import Decimal
 
 import pandas as pd
 import pytest
 from faker import Faker
-from pandas.tseries.offsets import BDay
 from wbcore.models import DynamicDecimalField, DynamicFloatField
 
 from wbfdm.models import Instrument, InstrumentPrice, RelatedInstrumentThroughModel
@@ -168,18 +166,6 @@ class TestInstrumentPriceModel:
         assert isinstance(instrument_price._meta.get_field("gross_value"), DynamicDecimalField)
         assert instrument_price.gross_value == instrument_price.net_value
 
-    @pytest.mark.parametrize("instrument_price__outstanding_shares", [Decimal(10)])
-    def test_compute_outstanding_shares(self, instrument_price, instrument_price_factory):
-        next_price = instrument_price_factory.create(
-            instrument=instrument_price.instrument,
-            date=instrument_price.date + BDay(1),
-            outstanding_shares=None,
-            calculated=instrument_price.calculated,
-        )
-        assert hasattr(instrument_price, "_compute_outstanding_shares")
-        assert isinstance(instrument_price._meta.get_field("outstanding_shares"), DynamicDecimalField)
-        assert next_price.outstanding_shares == instrument_price.outstanding_shares
-
     @pytest.mark.parametrize("instrument_price__volume_50d", [None])
     def test_compute_volume_50d(self, instrument_price, instrument_price_factory):
         assert hasattr(instrument_price, "_compute_volume_50d")

{wbfdm-1.54.22 → wbfdm-1.55.4}/wbfdm/viewsets/instruments/instrument_prices.py

@@ -174,22 +174,22 @@ class InstrumentPriceInstrumentStatisticsChartView(InstrumentMixin, viewsets.Cha
 
         if self.instrument.related_instruments.count() > 0:
             reference = self.instrument.related_instruments.first().name_repr
-            risk = self.instrument.primary_risk_instrument.name_repr
             df = pd.DataFrame(queryset.values("date", "sharpe_ratio", "correlation", "beta")).replace(
                 [np.inf, -np.inf], np.nan
             )
 
             if not df.empty:
                 df = df.set_index("date").sort_index().dropna()
-                fig.add_trace(
-                    go.Scatter(
-                        x=df.index,
-                        y=df.sharpe_ratio,
-                        mode="lines",
-                        name=f"Sharpe Ratio ({risk})",
-                        hovertemplate=get_hovertemplate_timeserie(currency=""),
+                if risk_instrument := self.instrument.primary_risk_instrument:
+                    fig.add_trace(
+                        go.Scatter(
+                            x=df.index,
+                            y=df.sharpe_ratio,
+                            mode="lines",
+                            name=f"Sharpe Ratio ({risk_instrument.name_repr})",
+                            hovertemplate=get_hovertemplate_timeserie(currency=""),
+                        )
                     )
-                )
                 fig.add_trace(
                     go.Scatter(
                         x=df.index,