wbfdm 1.51.2rc1__tar.gz → 1.51.5__tar.gz

{wbfdm-1.51.2rc1 → wbfdm-1.51.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wbfdm
-Version: 1.51.2rc1
+Version: 1.51.5
 Summary: The workbench module ensures rapid access to diverse financial data (market, fundamental, forecasts, ESG), with features for storing instruments, classifying them, and conducting financial analysis.
 Author-email: Christopher Wittlinger <c.wittlinger@stainly.com>
 Requires-Dist: roman==4.*

{wbfdm-1.51.2rc1 → wbfdm-1.51.5}/wbfdm/analysis/financial_analysis/statement_with_estimates.py

@@ -404,7 +404,6 @@ class StatementWithEstimates:
                 "stock_compensation_employee_ratio",
                 Financial.CAPEX.value,
                 Financial.SHARES_OUTSTANDING.value,
-                MarketData.MARKET_CAPITALIZATION.value,
                 MarketData.CLOSE.value,
                 MarketData.MARKET_CAPITALIZATION.value,
                 "net_cash",

{wbfdm-1.51.2rc1 → wbfdm-1.51.5}/wbfdm/analysis/financial_analysis/utils.py

@@ -133,6 +133,12 @@ class Loader:
                 self.errors["missing_data"].append(
                     "We could not find any market data covering the financial statement period"
                 )
+            ## TODO We might want to still exclude them from the final df but keep them for the estimate that used these
+            ## we actually want to keep the market data in the forecast column, because they are used for other statistic computation
+            # df.loc[df.index.get_level_values("estimate"), market_data_df.columns.difference(["period_end_date"])] = (
+            #     None
+            # )
+
         return df.rename_axis("financial", axis="columns")
 
     def _annotate_statement_data(self, df: pd.DataFrame, statement_values: list[Financial]) -> pd.DataFrame:

{wbfdm-1.51.2rc1 → wbfdm-1.51.5}/wbfdm/contrib/metric/backends/statistics.py

@@ -171,7 +171,7 @@ class Dataloader(BaseDataloader):
             )
             fx_rate = df_price["fx_rate"]
             df = pd.concat([df_revenue, df_price.drop("fx_rate", axis=1)], axis=1)
-            for key in ["revenue_y_1", "revenue_y0", "revenue_y1", "market_capitalization", "price"]:
+            for key in ["revenue_y_1", "revenue_y0", "revenue_y1", "market_capitalization"]:
                 if key in df.columns:
                     df[key] = df[key] / fx_rate
 

{wbfdm-1.51.2rc1 → wbfdm-1.51.5}/wbfdm/contrib/qa/sync/utils.py

@@ -1,12 +1,15 @@
+import logging
 from contextlib import suppress
+from datetime import date, timedelta
 from typing import Callable
 
 import pytz
 from django.conf import settings
-from django.db import connections
+from django.db import IntegrityError, connections
 from django.db.models import Max, QuerySet
 from django.template.loader import get_template
 from jinjasql import JinjaSql  # type: ignore
+from psycopg.errors import UniqueViolation
 from tqdm import tqdm
 from wbcore.contrib.currency.models import Currency
 from wbcore.contrib.dataloader.utils import dictfetchall, dictfetchone
@@ -15,6 +18,8 @@ from wbcore.utils.cache import mapping
 from wbfdm.models.exchanges.exchanges import Exchange
 from wbfdm.models.instruments.instruments import Instrument, InstrumentType
 
+logger = logging.getLogger("pms")
+
 BATCH_SIZE: int = 10000
 instrument_type_map = {
     "ADR": "american_depository_receipt",
@@ -135,6 +140,36 @@ def get_instrument_from_data(data, parent_source: str | None = None) -> Instrume
         return instrument
 
 
+def _delist_existing_duplicates(instrument: Instrument) -> None:
+    """Handle duplicate instruments by delisting existing entries"""
+    unique_identifiers = ["refinitiv_identifier_code", "refinitiv_mnemonic_code", "isin", "sedol", "valoren", "cusip"]
+
+    for identifier_field in unique_identifiers:
+        if identifier := getattr(instrument, identifier_field):
+            with suppress(Instrument.DoesNotExist):
+                duplicate = Instrument.objects.get(
+                    is_security=True, delisted_date__isnull=True, **{identifier_field: identifier}
+                )
+                duplicate.delisted_date = date.today() - timedelta(days=1)
+                duplicate.save()
+
+
+def _save_single_instrument(instrument: Instrument) -> None:
+    """Attempt to save an instrument with duplicate handling"""
+    try:
+        instrument.save()
+    except (UniqueViolation, IntegrityError) as e:
+        if instrument.is_security:
+            _delist_existing_duplicates(instrument)
+            try:
+                instrument.save()
+                logger.info(f"{instrument} successfully saved after automatic delisting")
+            except (UniqueViolation, IntegrityError) as e:
+                logger.error(f"Persistent integrity error: {e}")
+        else:
+            logger.error(f"Non-security instrument error: {e}")
+
+
 def _bulk_create_instruments_chunk(instruments: list[Instrument], update_unique_identifiers: bool = False):
     update_fields = [
         "name",
@@ -168,9 +203,17 @@ def _bulk_create_instruments_chunk(instruments: list[Instrument], update_unique_
             ]
         )
         bulk_update_kwargs = {"ignore_conflicts": True}
-    Instrument.objects.bulk_create(
-        instruments, update_fields=update_fields, unique_fields=["source", "source_id"], **bulk_update_kwargs
-    )
+    try:
+        Instrument.objects.bulk_create(
+            instruments, update_fields=update_fields, unique_fields=["source", "source_id"], **bulk_update_kwargs
+        )
+    except IntegrityError:
+        # we caught an integrity error on the bulk save, so we try to save one by one
+        logger.info(
+            "we detected an integrity error while bulk saving instruments. We save them one by one and delist the already existing instrument from the db if we can. "
+        )
+        for instrument in instruments:
+            _save_single_instrument(instrument)
 
 
 def update_instruments(sql_name: str, parent_source: str | None = None, context=None, debug: bool = False, **kwargs):
@@ -211,7 +254,7 @@ def update_or_create_item(
         defaults=defaults,
     )
     instrument.dl_parameters.update(dl_parameters)
-    instrument.save()
+    _save_single_instrument(instrument)
 
     return instrument
 

{wbfdm-1.51.2rc1 → wbfdm-1.51.5}/wbfdm/tasks.py

@@ -84,6 +84,12 @@ def synchronize_exchanges_as_task():
 
 @shared_task(queue="portfolio")
 def full_synchronization_as_task():
+    # we get all instrument without name or where we would expect a parent and consider them for clean up.
+    qs = Instrument.objects.filter(prices__isnull=True).filter(
+        (Q(name="") & Q(name_repr="")) | (Q(source__in=["qa-ds2-security", "qa-ds2-quote"]) & Q(parent__isnull=True))
+    )
+    for instrument in qs:
+        instrument.delete()
     initialize_exchanges()
     initialize_instruments()
     with transaction.atomic():

{wbfdm-1.51.2rc1 → wbfdm-1.51.5}/wbfdm/viewsets/statements/statements.py

@@ -54,7 +54,11 @@ class StatementPandasViewSet(InstrumentMixin, ExportPandasAPIViewSet):
                 pf.CharField(key="external_code", label="Code"),
                 pf.CharField(key="external_description", label="Description"),
                 pf.SparklineField(key="progress", label="Yearly Trend", dimension="double"),
-                *[pf.FloatField(key=field, label=field, display_mode=DisplayMode.SHORTENED) for field in self.columns],
+                *[
+                    pf.FloatField(key=field, label=field, display_mode=DisplayMode.SHORTENED)
+                    for field in self.columns
+                    if field not in ["external_ordering", "external_code", "external_description", "progress"]
+                ],
             ]
         )