wbfdm 1.46.4__tar.gz → 1.46.6__tar.gz

{wbfdm-1.46.4 → wbfdm-1.46.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wbfdm
-Version: 1.46.4
+Version: 1.46.6
 Summary: The workbench module ensures rapid access to diverse financial data (market, fundamental, forecasts, ESG), with features for storing instruments, classifying them, and conducting financial analysis.
 Author-email: Christopher Wittlinger <c.wittlinger@stainly.com>
 Requires-Dist: roman==4.*

{wbfdm-1.46.4 → wbfdm-1.46.6}/wbfdm/contrib/qa/dataloaders/market_data.py

@@ -3,11 +3,11 @@ from datetime import date
 from enum import Enum
 from functools import reduce
 from itertools import batched
-from typing import Iterator
+from typing import TYPE_CHECKING, Iterator
 
 import pypika as pk
 from django.db import ProgrammingError, connections
-from pypika import Column, MSSQLQuery
+from pypika import Case, Column, MSSQLQuery
 from pypika import functions as fn
 from pypika.enums import Order, SqlTypes
 from pypika.terms import LiteralValue, ValueWrapper
@@ -19,24 +19,27 @@ from wbfdm.dataloaders.protocols import MarketDataProtocol
 from wbfdm.dataloaders.types import MarketDataDict
 from wbfdm.enums import Frequency, MarketData
 
+if TYPE_CHECKING:
+    pass
+
 
 class DS2MarketData(Enum):
-    OPEN = ("Open_", 1)
-    CLOSE = ("Close_", 1)
-    HIGH = ("High", 1)
-    LOW = ("Low", 1)
-    BID = ("Bid", 1)
-    ASK = ("Ask", 1)
-    VWAP = ("VWAP", 1)
-    VOLUME = ("Volume", 1)
-    MARKET_CAPITALIZATION = ("ConsolMktVal", 1_000_000)
-    SHARES_OUTSTANDING = ("ConsolNumShrs", 1_000)
+    OPEN = "Open_"
+    CLOSE = "Close_"
+    HIGH = "High"
+    LOW = "Low"
+    BID = "Bid"
+    ASK = "Ask"
+    VWAP = "VWAP"
+    VOLUME = "Volume"
+    MARKET_CAPITALIZATION = "MktCap"
+    SHARES_OUTSTANDING = "NumShrs"
 
 
 class DatastreamMarketDataDataloader(MarketDataProtocol, Dataloader):
     def market_data(
         self,
-        values: list[MarketData] | None = [MarketData.CLOSE],
+        values: list[MarketData] = [MarketData.CLOSE],
         from_date: date | None = None,
         to_date: date | None = None,
         exact_date: date | None = None,
@@ -60,17 +63,10 @@ class DatastreamMarketDataDataloader(MarketDataProtocol, Dataloader):
         lookup = {
             f"{k[0]},{k[1]}": v for k, v in self.entities.values_list("dl_parameters__market_data__parameters", "id")
         }
-        value_mapping = [(DS2MarketData[x.name].value, x.value) for x in values or []]
 
         # Define tables
         pricing = pk.Table("vw_DS2Pricing")
-        market_val = pk.Table("DS2MktVal")
-        fx_code = pk.Table("DS2FxCode")
-        fx = pk.Table("DS2FxRate")
-
-        quotes = pk.Table("DS2CtryQtInfo")
-        securities = pk.Table("DS2Security")
-        securities2 = pk.Table("DS2Security", alias="securities2")
+        market_val = pk.Table("vw_DS2MktCap")
 
         mapping, create_mapping_table = create_table(
             "#ds2infoexchcode", Column("InfoCode", SqlTypes.INTEGER), Column("ExchIntCode", SqlTypes.INTEGER)
@@ -87,14 +83,6 @@ class DatastreamMarketDataDataloader(MarketDataProtocol, Dataloader):
                 fn.Cast(pricing.MarketDate, SqlTypes.DATE).as_("valuation_date"),
                 ValueWrapper("qa-ds2").as_("source"),
             )
-            .join(quotes)
-            .on_field("InfoCode")
-            .join(securities)
-            .on(quotes.DsSecCode == securities.DsSecCode)
-            .join(securities2)
-            .on((securities2.DsCmpyCode == securities.DsCmpyCode) & securities2.IsMajorSec == "Y")
-            .left_join(market_val)
-            .on((securities2.PrimQtInfoCode == market_val.InfoCode) & (pricing.MarketDate == market_val.ValDate))
             # We join on _codes, which removes all instruments not in _codes - implicit where
             .join(mapping)
             .on((pricing.InfoCode == mapping.InfoCode) & (pricing.ExchIntCode == mapping.ExchIntCode))
@@ -102,31 +90,46 @@ class DatastreamMarketDataDataloader(MarketDataProtocol, Dataloader):
             .orderby(pricing.MarketDate, order=Order.desc)
         )
 
-        # If we need to convert to a target currency, we need the fx rate table and multiply all values with the fx rate
+        # if a target currency is required, we join on the fx tables and set the currency to the desired one
+        # otherwise we just set the currency to whatever the currency is from the instrument
+        fx_rate = None
         if target_currency:
+            query = query.select(ValueWrapper(target_currency).as_("currency"))
+            fx_code = pk.Table("DS2FxCode")
+            fx_rate = pk.Table("DS2FxRate")
             query = (
-                query.select(
-                    ValueWrapper(target_currency).as_("currency"),
-                    *[
-                        (LiteralValue(value[0][0]) * fn.Coalesce(fx.midrate, 1) * value[0][1]).as_(value[1])
-                        for value in value_mapping
-                    ],
-                )
+                query
+                # Join FX code table matching currencies and ensuring SPOT rate type
                 .left_join(fx_code)
                 .on(
-                    (fx_code.FromCurrCode == target_currency)
-                    & (fx_code.ToCurrCode == pricing.Currency)
+                    (fx_code.FromCurrCode == pricing.Currency)
+                    & (fx_code.ToCurrCode == target_currency)
                     & (fx_code.RateTypeCode == "SPOT")
                 )
-                .left_join(fx)
-                .on((fx_code.ExRateIntCode == fx.ExRateIntCode) & (fx.ExRateDate == pricing.MarketDate))
+                # Join FX rate table matching internal code and date
+                .left_join(fx_rate)
+                .on((fx_rate.ExRateIntCode == fx_code.ExRateIntCode) & (fx_rate.ExRateDate == pricing.MarketDate))
+                # We filter out rows which do not have a proper fx rate (we exclude same currency conversions)
+                .where((Case().when(pricing.Currency == target_currency, 1).else_(fx_rate.midrate).isnotnull()))
             )
+
         else:
-            query = query.select(
-                pricing.Currency.as_("currency"),
-                *[(LiteralValue(value[0][0]) * value[0][1]).as_(value[1]) for value in value_mapping],
+            query = query.select(pricing.Currency.as_("currency"))
+
+        # if market cap or shares outstanding are required we need to join with an additional table
+        if MarketData.MARKET_CAPITALIZATION in values or MarketData.SHARES_OUTSTANDING in values:
+            query = query.join(market_val).on(
+                (pricing.InfoCode == market_val.InfoCode) & (pricing.MarketDate == market_val.MarketDate)
             )
 
+        for market_data in values:
+            ds2_value = DS2MarketData[market_data.name].value
+            value = LiteralValue(ds2_value)
+            if fx_rate and market_data is not MarketData.SHARES_OUTSTANDING:
+                value /= Case().when(pricing.Currency == target_currency, 1).else_(fx_rate.midrate)
+
+            query = query.select(value.as_(market_data.value))
+
         # Add conditional where clauses
         if from_date:
             query = query.where(pricing.MarketDate >= from_date)

wbfdm-1.46.6/wbfdm/contrib/qa/dataloaders/utils.py

@@ -0,0 +1,68 @@
+from typing import TYPE_CHECKING
+
+from pypika import Case, Column, Criterion, Field, MSSQLQuery, Table
+from pypika.terms import ValueWrapper
+
+if TYPE_CHECKING:
+    from pypika.queries import CreateQueryBuilder, QueryBuilder
+    from pypika.terms import Term
+
+
+def compile_source(source: str) -> "Term":
+    return ValueWrapper(source).as_("source")
+
+
+SOURCE_DS2 = compile_source("qa-ds2")
+SOURCE_RKD = compile_source("qa-rkd")
+SOURCE_IBES = compile_source("qa-ibes")
+
+
+def create_table(tablename: "str", *columns: "Column") -> tuple[Table, "CreateQueryBuilder"]:
+    table = Table(tablename)
+    return table, MSSQLQuery.create_table(table).columns(*columns)
+
+
+def get_currency_fx_rate(
+    query: "QueryBuilder", alias_prefix: str, from_ccy: str | Field, to_ccy: str | Field, fx_date: str | Criterion
+) -> tuple["QueryBuilder", Criterion]:
+    """Gets the currency exchange rate with fallback to 1 if rate not found.
+
+    Extends a query with FX table joins to retrieve currency conversion rates. Uses left joins
+    and provides a default rate of 1 when no conversion rate exists, effectively making the
+    conversion neutral in such cases.
+
+    Warning:
+        The alias_prefix must be unique across the entire query chain. Reusing the same
+        prefix in multiple calls will cause table alias conflicts, leading to incorrect
+        results or SQL errors. Each call to this function should use a distinct prefix.
+
+    Args:
+        query (QueryBuilder): Base query to extend with FX-related joins
+        alias_prefix (str): Prefix for table aliases to prevent naming conflicts in joins.
+                          Must be unique across the entire query chain.
+        from_ccy (str | pk.Field): Source currency code or Field to convert from
+        to_ccy (str | pk.Field): Target currency code or Field to convert to
+        fx_date (Any): Date for which to get the exchange rate
+
+    Returns:
+        tuple[QueryBuilder, pk.Criterion]: A tuple containing:
+            - Modified query with FX table joins
+            - Coalesced midrate (defaults to 1 if no rate found)
+    """
+    # Create aliased FX tables to avoid naming conflicts in complex queries
+    fx_code = Table("DS2FxCode", alias=f"{alias_prefix}DS2FxCode")
+    fx_rate = Table("DS2FxRate", alias=f"{alias_prefix}DS2FxRate")
+    # Build query with FX table joins
+    query = (
+        query
+        # Join FX code table matching currencies and ensuring SPOT rate type
+        .left_join(fx_code)
+        .on((fx_code.FromCurrCode == from_ccy) & (fx_code.ToCurrCode == to_ccy) & (fx_code.RateTypeCode == "SPOT"))
+        # Join FX rate table matching internal code and date
+        .left_join(fx_rate)
+        .on((fx_rate.ExRateIntCode == fx_code.ExRateIntCode) & (fx_rate.ExRateDate == fx_date))
+    )
+
+    # Return query and coalesced rate (falls back to 1 if no rate found)
+    fx = Case().when(from_ccy == to_ccy, 1).else_(fx_rate.midrate)
+    return (query, fx)

{wbfdm-1.46.4 → wbfdm-1.46.6}/wbfdm/contrib/qa/sync/utils.py

@@ -114,7 +114,19 @@ def convert_data(data, parent_source: str | None = None) -> dict:
 def get_instrument_from_data(data, parent_source: str | None = None) -> Instrument | None:
     if len(data.keys()) == 0:
         return None
-    instrument = Instrument(**convert_data(data, parent_source))
+
+    defaults = convert_data(data, parent_source)
+    try:
+        instrument = Instrument.objects.get(source=defaults["source"], source_id=defaults["source_id"])
+        instrument.dl_parameters.update(
+            defaults.pop("dl_parameters")
+        )  # we need to update the existing dl parameters to not override its existing value
+        for k, v in defaults.items():
+            setattr(instrument, k, v)
+
+    except Instrument.DoesNotExist:
+        instrument = Instrument(**defaults)
+
     instrument.pre_save()
     instrument.computed_str = instrument.compute_str()
     if (
@@ -198,10 +210,7 @@ def update_or_create_item(
         source_id=external_id,
         defaults=defaults,
     )
-
-    for key, value in dl_parameters.items():
-        instrument.dl_parameters[key] = value
-
+    instrument.dl_parameters.update(dl_parameters)
     instrument.save()
 
     return instrument

{wbfdm-1.46.4 → wbfdm-1.46.6}/wbfdm/dataloaders/protocols.py

@@ -39,7 +39,7 @@ class AdjustmentsProtocol(Protocol):
 class MarketDataProtocol(Protocol):
     def market_data(
         self,
-        values: list[MarketData] | None = [MarketData.CLOSE],
+        values: list[MarketData] = [MarketData.CLOSE],
         from_date: date | None = None,
         to_date: date | None = None,
         exact_date: date | None = None,

{wbfdm-1.46.4 → wbfdm-1.46.6}/wbfdm/filters/classifications.py

@@ -1,5 +1,6 @@
 from wbcore import filters as wb_filters
 
+from wbfdm.filters.utils import _get_default_classification_group_id
 from wbfdm.models import (
     Classification,
     ClassificationGroup,
@@ -88,6 +89,7 @@ class InstrumentClassificationThroughModelViewFilterSet(wb_filters.FilterSet):
         endpoint=ClassificationGroup.get_representation_endpoint(),
         value_key=ClassificationGroup.get_representation_value_key(),
         label_key=ClassificationGroup.get_representation_label_key(),
+        default=_get_default_classification_group_id,
     )
 
     class Meta:

{wbfdm-1.46.4 → wbfdm-1.46.6}/wbfdm/filters/instruments.py

@@ -5,7 +5,7 @@ from psycopg.types.range import DateRange
 from wbcore import filters as wb_filters
 from wbcore.contrib.tags.filters import TagFilterMixin
 
-from wbfdm.filters.utils import get_earliest_date, get_latest_date
+from wbfdm.filters.utils import _get_default_classification_group_id, get_earliest_date, get_latest_date
 from wbfdm.models.instruments import (
     ClassificationGroup,
     Instrument,
@@ -13,7 +13,6 @@ from wbfdm.models.instruments import (
     InstrumentFavoriteGroup,
 )
 from wbfdm.models.instruments.classifications import Classification
-from wbfdm.preferences import get_default_classification_group
 
 
 def get_default_favorite_group(field, request, view):
@@ -156,12 +155,6 @@ class InstrumentFilterSet(TagFilterMixin, InstrumentFavoriteGroupFilterSet):
         hidden_fields = ["id__in", "is_managed", "is_security", "level"]
 
 
-def _get_default_classification_group_id(*args, **kwargs):
-    if group := get_default_classification_group():
-        return group.id
-    return None
-
-
 class BaseClassifiedInstrumentFilterSet(TagFilterMixin, wb_filters.FilterSet):
     classification_group = wb_filters.ModelChoiceFilter(
         label="Classification Group",
@@ -169,7 +162,7 @@ class BaseClassifiedInstrumentFilterSet(TagFilterMixin, wb_filters.FilterSet):
         endpoint=ClassificationGroup.get_representation_endpoint(),
         value_key=ClassificationGroup.get_representation_value_key(),
         label_key=ClassificationGroup.get_representation_label_key(),
-        default=_get_default_classification_group_id(),
+        default=_get_default_classification_group_id,
         method="fake_filter",
         required=True,
     )

{wbfdm-1.46.4 → wbfdm-1.46.6}/wbfdm/filters/utils.py

@@ -4,7 +4,16 @@ from pandas.tseries.offsets import BDay, BMonthBegin, BMonthEnd
 from psycopg.types.range import DateRange
 from wbcore.utils.date import current_quarter_date_start
 
-from wbfdm.models import Instrument
+from wbfdm.models import Classification, Instrument
+from wbfdm.preferences import get_default_classification_group
+
+
+def _get_default_classification_group_id(field, request, view, **kwargs):
+    if classification_id := view.kwargs.get("classification_id", None):
+        return Classification.objects.get(id=classification_id).group.id
+    if group := get_default_classification_group():
+        return group.id
+    return None
 
 
 def get_earliest_date(field, request, view):

wbfdm-1.46.6/wbfdm/migrations/0030_alter_relatedinstrumentthroughmodel_related_type.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.12 on 2025-02-20 14:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('wbfdm', '0029_alter_instrumentprice_volume'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='relatedinstrumentthroughmodel',
+            name='related_type',
+            field=models.CharField(choices=[('BENCHMARK', 'Benchmark'), ('PEER', 'Peer'), ('RISK_INSTRUMENT', 'Risk Instrument'), ('TRACKER', 'Tracker')], default='BENCHMARK', max_length=32),
+        ),
+    ]

{wbfdm-1.46.4 → wbfdm-1.46.6}/wbfdm/models/instruments/instrument_relationships.py

@@ -140,6 +140,7 @@ class RelatedInstrumentThroughModel(models.Model):
         BENCHMARK = "BENCHMARK", "Benchmark"
         PEER = "PEER", "Peer"
         RISK_INSTRUMENT = "RISK_INSTRUMENT", "Risk Instrument"
+        TRACKER = "TRACKER", "Tracker"
 
     instrument = models.ForeignKey(
         "wbfdm.Instrument",

{wbfdm-1.46.4 → wbfdm-1.46.6}/wbfdm/tasks.py

@@ -55,8 +55,8 @@ def update_of_investable_universe_data(
     for instrument in tqdm(instruments, total=instruments.count()):
         try:
             prices.extend(list(instrument.get_price_objects(start=start, end=end, clear=clear)))
-        except ConnectionError:
-            print(f"Connection error while processing instrument {instrument}")  # noqa
+        except Exception as e:
+            print(f"Connection error while processing instrument {instrument}: {str(e)}")  # noqa
     Instrument.bulk_save_instrument_prices(prices)
     investable_universe_updated.send(sender=Instrument, end_date=end)
     if with_background_tasks:

wbfdm-1.46.4/wbfdm/contrib/qa/dataloaders/utils.py

@@ -1,22 +0,0 @@
-from typing import TYPE_CHECKING
-
-from pypika import Column, MSSQLQuery, Table
-from pypika.terms import ValueWrapper
-
-if TYPE_CHECKING:
-    from pypika.queries import CreateQueryBuilder
-    from pypika.terms import Term
-
-
-def compile_source(source: str) -> "Term":
-    return ValueWrapper(source).as_("source")
-
-
-SOURCE_DS2 = compile_source("qa-ds2")
-SOURCE_RKD = compile_source("qa-rkd")
-SOURCE_IBES = compile_source("qa-ibes")
-
-
-def create_table(tablename: "str", *columns: "Column") -> tuple[Table, "CreateQueryBuilder"]:
-    table = Table(tablename)
-    return table, MSSQLQuery.create_table(table).columns(*columns)