wbfdm 1.49.5__py2.py3-none-any.whl → 1.59.4__py2.py3-none-any.whl

wbfdm/models/fields.py

@@ -23,7 +23,7 @@ class CompositeKey(models.AutoField):
     def __init__(self, columns: list[str], db_column_ref: str | None = None, *args, **kwargs):
         self.columns = columns
         self.db_column_ref = db_column_ref
-        super().__init__(primary_key=True, *args, **kwargs)
+        super().__init__(*args, primary_key=True, **kwargs)
 
     def contribute_to_class(self, cls, name, private_only=False):
         self.set_attributes_from_name(name)
@@ -108,7 +108,7 @@ class Exact(models.Lookup):
 
         lookups = [
             lookup_class(field.get_col(self.lhs.alias), self.rhs[column])
-            for lookup_class, field, column in zip(lookup_classes, fields, self.lhs.field.columns)
+            for lookup_class, field, column in zip(lookup_classes, fields, self.lhs.field.columns, strict=False)
         ]
 
         value_constraint = WhereNode()

wbfdm/models/fk_fields.py

@@ -93,7 +93,7 @@ class CompositeForeignKey(ForeignObject):
             kwargs["on_delete"] = self.override_on_delete(kwargs["on_delete"])
 
         kwargs["to_fields"], kwargs["from_fields"] = zip(
-            *((k, v.value) for k, v in self._raw_fields.items() if v.is_local_field)
+            *((k, v.value) for k, v in self._raw_fields.items() if v.is_local_field), strict=False
         )
         super().__init__(to, **kwargs)
 
@@ -220,7 +220,7 @@ class CompositeForeignKey(ForeignObject):
 
     def _check_nullifequal_fields_exists(self):
         res = []
-        for field_name, value in self.null_if_equal:
+        for field_name, _ in self.null_if_equal:
             try:
                 self.model._meta.get_field(field_name)
             except FieldDoesNotExist:
@@ -269,7 +269,7 @@ class CompositeForeignKey(ForeignObject):
 
         return OrderedDict(
             (k, (v if isinstance(v, CompositePart) else LocalFieldValue(v)))
-            for k, v in (to_fields.items() if isinstance(to_fields, dict) else zip(to_fields, to_fields))
+            for k, v in (to_fields.items() if isinstance(to_fields, dict) else zip(to_fields, to_fields, strict=False))
         )
 
     def db_type(self, connection):

wbfdm/models/instruments/instrument_lists.py

@@ -68,6 +68,7 @@ class InstrumentList(WBModel):
     class InstrumentListType(models.TextChoices):
         WATCH = "WATCH", "Watch List"
         EXCLUSION = "EXCLUSION", "Exclusion List"
+        INCLUSION = "INCLUSION", "Inclusion List"
 
     name = models.CharField(max_length=255)
     identifier = models.CharField(max_length=255, unique=True, blank=True)

wbfdm/models/instruments/instrument_prices.py

@@ -195,9 +195,11 @@ class InstrumentPrice(
         verbose_name="Value (Gross)",
     )  # TODO: I think we need to remove this field that is not really used here.
 
-    outstanding_shares = DynamicDecimalField(
+    outstanding_shares = models.DecimalField(
         decimal_places=4,
         max_digits=16,
+        blank=True,
+        null=True,
         verbose_name="Outstanding Shares",
         help_text="The amount of outstanding share for this instrument",
     )
@@ -340,6 +342,11 @@ class InstrumentPrice(
 
         if self.market_capitalization_consolidated is None:
             self.market_capitalization_consolidated = self.market_capitalization
+
+        # if the instrument is of type cash, we enforce the net value to 1
+        if self.instrument.is_cash or self.instrument.is_cash_equivalent:
+            self.net_value = Decimal("1")
+            self.gross_value = Decimal("1")
         super().save(*args, **kwargs)
 
     def __str__(self):

wbfdm/models/instruments/instrument_relationships.py

@@ -162,6 +162,9 @@ class RelatedInstrumentThroughModel(models.Model):
     class Meta:
         unique_together = ("instrument", "related_instrument", "is_primary", "related_type")
 
+    def __str__(self) -> str:
+        return f"{self.instrument} - {self.related_instrument} ({self.related_type})"
+
     def save(self, *args, **kwargs):
         qs = RelatedInstrumentThroughModel.objects.filter(
             instrument=self.instrument, related_type=self.related_type, is_primary=True

wbfdm/models/instruments/instruments.py

@@ -1,8 +1,10 @@
+import logging
 import re
 from contextlib import suppress
 from datetime import date, timedelta
 from typing import Any, Generator, Iterator, Self, TypeVar
 
+import pandas as pd
 from celery import shared_task
 from colorfield.fields import ColorField
 from django.contrib import admin
@@ -44,13 +46,17 @@ from wbfdm.preferences import get_default_classification_group
 from wbfdm.signals import (
     add_instrument_to_investable_universe,
     instrument_price_imported,
+    investable_universe_updated,
 )
 
+from ...analysis.financial_analysis.change_point_detection import outlier_detection, statistical_change_point_detection
 from ...dataloaders.proxies import InstrumentDataloaderProxy
 from .instrument_relationships import RelatedInstrumentThroughModel
 from .mixin.instruments import InstrumentPMSMixin
 from .querysets import InstrumentQuerySet
-from .utils import re_bloomberg, re_isin, re_mnemonic, re_ric
+from .utils import clean_ric, re_bloomberg, re_isin, re_mnemonic, re_ric
+
+logger = logging.getLogger("pms")
 
 
 class InstrumentManager(TreeManager):
@@ -133,6 +139,12 @@ class InstrumentManager(TreeManager):
     def filter_active_at_date(self, val_date: date):
         return self.get_queryset().filter_active_at_date(val_date)
 
+    def get_instrument_prices_from_market_data(self, **kwargs):
+        return self.get_queryset().get_instrument_prices_from_market_data(**kwargs)
+
+    def get_returns_df(self, **kwargs) -> tuple[dict[date, dict[int, float]], pd.DataFrame]:
+        return self.get_queryset().get_returns_df(**kwargs)
+
 
 class SecurityInstrumentManager(InstrumentManager):
     def get_queryset(self) -> InstrumentQuerySet:
@@ -193,38 +205,38 @@ class InstrumentType(models.Model):
 
     @classmethod
     @property
-    def PRODUCT(cls):
+    def PRODUCT(cls):  # noqa
         return InstrumentType.objects.get_or_create(
             key="product", defaults={"name": "Product", "short_name": "Product"}
         )[0]
 
     @classmethod
     @property
-    def EQUITY(cls):
+    def EQUITY(cls):  # noqa
         return InstrumentType.objects.get_or_create(key="equity", defaults={"name": "equity", "short_name": "equity"})[
             0
         ]
 
     @classmethod
     @property
-    def INDEX(cls):
+    def INDEX(cls):  # noqa
         return InstrumentType.objects.get_or_create(key="index", defaults={"name": "Index", "short_name": "Index"})[0]
 
     @classmethod
     @property
-    def CASH(cls):
+    def CASH(cls):  # noqa
         return InstrumentType.objects.get_or_create(key="cash", defaults={"name": "Cash", "short_name": "Cash"})[0]
 
     @classmethod
     @property
-    def CASHEQUIVALENT(cls):
+    def CASHEQUIVALENT(cls):  # noqa
         return InstrumentType.objects.get_or_create(
             key="cash_equivalent", defaults={"name": "Cash Equivalents", "short_name": "Cash Equivalents"}
         )[0]
 
     @classmethod
     @property
-    def PRODUCT_GROUP(cls):
+    def PRODUCT_GROUP(cls):  # noqa
         return InstrumentType.objects.get_or_create(
             key="product_group", defaults={"name": "Product Group", "short_name": "Product Group"}
         )[0]
@@ -306,6 +318,11 @@ class Instrument(ComplexToStringMixin, TagModelMixin, ImportMixin, InstrumentPMS
     exchange = models.ForeignKey(
         to="wbfdm.Exchange", null=True, blank=True, on_delete=models.PROTECT, related_name="instruments"
     )
+    round_lot_size = models.IntegerField(
+        default=1,
+        verbose_name="Round Lot Size",
+        help_text="A round lot (or board lot) is the normal unit of trading of a security.",
+    )
 
     source_id = models.CharField(max_length=64, null=True, blank=True)
     source = models.CharField(max_length=64, null=True, blank=True)
@@ -594,17 +611,24 @@ class Instrument(ComplexToStringMixin, TagModelMixin, ImportMixin, InstrumentPMS
 
     @property
     def identifier_repr(self) -> str:
-        identifier_repr = ""
-        if self.instrument_type and self.instrument_type.key == "product":
-            # Then we prioritize ISIN over ticker
-            identifiers = ["isin", "ticker", "refinitiv_identifier_code", "refinitiv_mnemonic_code"]
+        identifiers = []
+        if self.is_security:
+            identifier_labels = ["ticker", "refinitiv_identifier_code", "refinitiv_mnemonic_code"]
         else:
-            identifiers = ["ticker", "isin", "refinitiv_identifier_code", "refinitiv_mnemonic_code"]
-        for identifier in identifiers:
-            if v := getattr(self, identifier, None):
-                identifier_repr = v
+            identifier_labels = ["refinitiv_mnemonic_code"]
+        for label in identifier_labels:
+            if v := getattr(self, label, None):
+                identifiers.append(v)
                 break
-        return identifier_repr.replace(":", "-")
+        if self.isin and self.is_security:
+            identifiers.append(self.isin)
+
+        return " - ".join(identifiers).replace(":", "-")
+
+    @property
+    def bloomberg_ticker(self):
+        if self.exchange and (bbg_composite := self.exchange.bbg_composite):
+            return self.ticker + " " + bbg_composite
 
     @property
     def valuations(self):
@@ -652,11 +676,11 @@ class Instrument(ComplexToStringMixin, TagModelMixin, ImportMixin, InstrumentPMS
             raise ValidationError("An instrument in the investable universe cannot have children")
         return self
 
-    def pre_save(self):
+    def pre_save(self):  # noqa: C901
         if self.instrument_type:
             self.is_security = self.instrument_type.is_security
-        if self.delisted_date:
-            self.is_security = False
+        # if self.delisted_date:
+        #     self.is_security = False
         if not self.name_repr:
             self.name_repr = self.name
         if not self.founded_year and self.inception_date:
@@ -693,6 +717,12 @@ class Instrument(ComplexToStringMixin, TagModelMixin, ImportMixin, InstrumentPMS
                 self.instrument_type = child.instrument_type
             if not self.currency:
                 self.currency = child.currency
+        if (
+            self.refinitiv_identifier_code
+            and self.exchange
+            and (exchange_ric := self.exchange.refinitiv_identifier_code)
+        ):
+            self.refinitiv_identifier_code = clean_ric(self.refinitiv_identifier_code, exchange_ric)
 
     def save(self, *args, **kwargs):
         self.pre_save()
@@ -700,16 +730,22 @@ class Instrument(ComplexToStringMixin, TagModelMixin, ImportMixin, InstrumentPMS
             self.is_primary = not self.parent or (self.exchange is not None and self.parent.exchange == self.exchange)
         super().save(*args, **kwargs)
 
-    def compute_str(self):
-        repr = self.name_repr  # we follow bloomberg instrument representation format
-        if self.instrument_type:
+    def get_compute_str(self):
+        repr = self.name_repr or self.name or ""
+        repr = repr.title()  # we follow bloomberg instrument representation format
+        if self.instrument_type and self.is_security:
             repr += f" {self.instrument_type.short_name}"
-        if self.identifier_repr:
-            repr += f" - {self.identifier_repr}"
-        if self.exchange:
-            repr += f" ({str(self.exchange)})"
+        if self.is_security or not self.level == 0:
+            if self.identifier_repr:
+                repr += f" - {self.identifier_repr}"
+            # if the object has an exchange and is not a security nor a company (a quote then), we append the exchange representation
+            if self.exchange and self.parent is not None and not self.is_security:
+                repr += f" ({str(self.exchange)})"
         return repr
 
+    def compute_str(self):
+        return self.get_compute_str()
+
     def is_active_at_date(self, today: date) -> bool:
         return (
             self.inception_date is not None
@@ -929,6 +965,9 @@ def pre_save_instrument(sender, instance, raw, **kwargs):
                 and pre_instance.isin not in instance.old_isins
             ):
                 instance.old_isins = [*instance.old_isins, pre_instance.isin]
+                for children in instance.children.all():
+                    children.isin = instance.isin
+                    children.save()
             if pre_instance.name_repr != instance.name_repr:
                 # if a family member get is name representation updated, we update it for the whole family
                 pre_instance.get_family().update(name_repr=instance.name_repr)
@@ -970,6 +1009,10 @@ class Cash(Instrument):
 
     def save(self, *args, **kwargs):
         self.is_cash = True
+        self.dl_parameters["market_data"] = {
+            "path": "wbfdm.contrib.internal.dataloaders.market_data.MarketDataDataloader"
+        }
+
         super().save(*args, **kwargs)
 
     class Meta:
@@ -999,3 +1042,73 @@ class Equity(Instrument):
 @receiver(create_news_relationships, sender="wbnews.News")
 def get_news_relationships_for_instruments_task(sender: type, instance: "News", **kwargs) -> shared_task:
     return run_company_extraction_llm.s(instance.title, instance.description, instance.summary)
+
+
+@shared_task(queue="pms")
+def detect_and_correct_financial_timeseries(
+    max_days_interval: int | None = None,
+    check_date: date | None = None,
+    with_pelt: bool = False,
+    detect_only: bool = False,
+    full_reimport: bool = False,
+    debug: bool = False,
+):
+    """Detects and corrects anomalies in financial time series data for instruments.
+
+    Analyzes price data using statistical methods to identify outliers and change points,
+    then triggers price reimport for affected date ranges when corrections are needed.
+
+    Args:
+        max_days_interval: Maximum lookback window in days for analysis (None = all history)
+        check_date: Reference date for analysis (defaults to current date)
+        with_pelt: Enable Pelt's change point detection alongside basic z-score outlier detection
+        detect_only: Run detection without performing data correction/reimport
+        full_reimport: Reimport entire price history when corruption detected (requires max_days_interval=None)
+        debug: Show progress bar during instrument processing
+
+    """
+    if not check_date:
+        check_date = date.today()
+    gen = (
+        Instrument.investable_universe.filter(is_managed=False)
+        .filter_active_at_date(check_date)
+        .exclude(source="dsws")
+    )
+    if debug:
+        gen = tqdm(gen, total=gen.count())
+    for instrument in gen:
+        prices = instrument.valuations.all()
+        if max_days_interval:
+            prices = prices.filter(date__gte=check_date - timedelta(days=max_days_interval))
+        # construct the price timeseries
+        prices_series = (
+            pd.DataFrame(
+                prices.filter_only_valid_prices().values_list("date", "net_value"), columns=["date", "net_value"]
+            )
+            .set_index("date")["net_value"]
+            .astype(float)
+            .sort_index()
+        )
+        if not prices_series.empty:
+            outliers = outlier_detection(prices_series).index.tolist()
+            # if pelt enable, add the outliers found by the PELT model
+            if with_pelt:
+                outliers.extend(statistical_change_point_detection(prices_series).index.tolist())
+            if outliers:
+                logger.info(f"Abnormal change point detected for {instrument} at {outliers}.")
+                if not detect_only:
+                    # for a full reimport, we delete the whole existing price series and reimport since inception
+                    if full_reimport and not max_days_interval:
+                        start_import_date = instrument.inception_date
+                        end_import_date = check_date
+                        instrument.prices.filter(assets__isnull=True).delete()
+                    else:
+                        start_import_date = min(outliers) - timedelta(days=7)
+                        end_import_date = max(outliers) + timedelta(days=7)
+                    logger.info(f"Reimporting price from {start_import_date} to {end_import_date}...")
+                    instrument.import_prices(start=start_import_date, end=end_import_date)
+
+
+@receiver(investable_universe_updated, sender="wbfdm.Instrument")
+def investable_universe_change_point_detection(*args, end_date: date | None = None, **kwargs):
+    detect_and_correct_financial_timeseries.delay(check_date=end_date, max_days_interval=365)

wbfdm/models/instruments/llm/create_instrument_news_relationships.py

@@ -4,8 +4,8 @@ from typing import Any
 from celery import shared_task
 from django.contrib.contenttypes.models import ContentType
 from langchain_core.messages import HumanMessage, SystemMessage
-from pydantic import BaseModel, Field
-from wbcore.contrib.ai.exceptions import APIStatusErrors
+from pydantic import BaseModel, Field, ValidationError
+from wbcore.contrib.ai.exceptions import APIStatusErrors, BadRequestErrors
 from wbcore.contrib.ai.llm.utils import run_llm
 
 logger = logging.getLogger("llm")
@@ -57,29 +57,36 @@ def run_company_extraction_llm(title: str, description: str, *args) -> list[dict
                 SystemMessage(
                     content="You will be parsed a news article, please provide the name of the publicly listed companies mentioned in the article, along with their ISIN, ticker, RIC, sentiment, and analysis."
                 ),
-                HumanMessage(content=f"Title: {title}, Description: {description}"),
+                HumanMessage(content="Title: {title}, Description: {description}"),
             ],
             output_model=CompaniesModel,
-        )
-        instrument_ct = ContentType.objects.get_for_model(Instrument)
-        for company in res.get("companies", []):
-            instrument = InstrumentLookup(Instrument).lookup(
-                only_security=True,
-                name=company.name,
-                isin=company.isin,
-                ticker=company.ticker,
-                refinitiv_identifier_code=company.refinitiv_identifier_code,
-            )
-            if instrument is not None:
-                relationships.append(
-                    {
-                        "content_type_id": instrument_ct.id,
-                        "object_id": instrument.get_root().id,
-                        "sentiment": company.sentiment,
-                        "analysis": company.analysis,
-                        "content_object_repr": str(instrument),
-                    }
+            query={"title": title, "description": description},
+        )[0]
+        if isinstance(res, CompaniesModel):
+            instrument_ct = ContentType.objects.get_for_model(Instrument)
+            for company in res.companies:
+                instrument = InstrumentLookup(Instrument).lookup(
+                    only_security=True,
+                    name=company.name,
+                    isin=company.isin,
+                    ticker=company.ticker,
+                    refinitiv_identifier_code=company.refinitiv_identifier_code,
                 )
+                if instrument is not None:
+                    relationships.append(
+                        {
+                            "content_type_id": instrument_ct.id,
+                            "object_id": instrument.get_root().id,
+                            "sentiment": company.sentiment,
+                            "analysis": company.analysis,
+                            "content_object_repr": str(instrument),
+                        }
+                    )
+    except (
+        ValidationError,
+        *BadRequestErrors,
+    ):  # we silent bad request error because there is nothing we can do about it
+        pass
     except tuple(APIStatusErrors) as e:  # for APIStatusError, we let celery retry it
         raise e
     except Exception as e:  # otherwise we log the error and silently fail

wbfdm/models/instruments/mixin/financials_computed.py

@@ -697,10 +697,6 @@ import pandas as pd
 #
 #
 class InstrumentPriceComputedMixin:
-    def _compute_outstanding_shares(self):
-        if self.outstanding_shares is None and (previous_price := self.previous_price):
-            return previous_price.outstanding_shares
-
     def _compute_outstanding_shares_consolidated(self):
         if self.outstanding_shares_consolidated is None and self.outstanding_shares is not None:
             return self.outstanding_shares