wbfdm 1.51.2rc1__py2.py3-none-any.whl → 1.51.5__py2.py3-none-any.whl

wbfdm/analysis/financial_analysis/statement_with_estimates.py

@@ -404,7 +404,6 @@ class StatementWithEstimates:
                 "stock_compensation_employee_ratio",
                 Financial.CAPEX.value,
                 Financial.SHARES_OUTSTANDING.value,
-                MarketData.MARKET_CAPITALIZATION.value,
                 MarketData.CLOSE.value,
                 MarketData.MARKET_CAPITALIZATION.value,
                 "net_cash",

wbfdm/analysis/financial_analysis/utils.py

@@ -133,6 +133,12 @@ class Loader:
                 self.errors["missing_data"].append(
                     "We could not find any market data covering the financial statement period"
                 )
+            ## TODO We might want to still exclude them from the final df but keep them for the estimate that used these
+            ## we actually want to keep the market data in the forecast column, because they are used for other statistic computation
+            # df.loc[df.index.get_level_values("estimate"), market_data_df.columns.difference(["period_end_date"])] = (
+            #     None
+            # )
+
         return df.rename_axis("financial", axis="columns")
 
     def _annotate_statement_data(self, df: pd.DataFrame, statement_values: list[Financial]) -> pd.DataFrame:

wbfdm/contrib/metric/backends/statistics.py

@@ -171,7 +171,7 @@ class Dataloader(BaseDataloader):
             )
             fx_rate = df_price["fx_rate"]
             df = pd.concat([df_revenue, df_price.drop("fx_rate", axis=1)], axis=1)
-            for key in ["revenue_y_1", "revenue_y0", "revenue_y1", "market_capitalization", "price"]:
+            for key in ["revenue_y_1", "revenue_y0", "revenue_y1", "market_capitalization"]:
                 if key in df.columns:
                     df[key] = df[key] / fx_rate
 

wbfdm/contrib/qa/sync/utils.py

@@ -1,12 +1,15 @@
+import logging
 from contextlib import suppress
+from datetime import date, timedelta
 from typing import Callable
 
 import pytz
 from django.conf import settings
-from django.db import connections
+from django.db import IntegrityError, connections
 from django.db.models import Max, QuerySet
 from django.template.loader import get_template
 from jinjasql import JinjaSql  # type: ignore
+from psycopg.errors import UniqueViolation
 from tqdm import tqdm
 from wbcore.contrib.currency.models import Currency
 from wbcore.contrib.dataloader.utils import dictfetchall, dictfetchone
@@ -15,6 +18,8 @@ from wbcore.utils.cache import mapping
 from wbfdm.models.exchanges.exchanges import Exchange
 from wbfdm.models.instruments.instruments import Instrument, InstrumentType
 
+logger = logging.getLogger("pms")
+
 BATCH_SIZE: int = 10000
 instrument_type_map = {
     "ADR": "american_depository_receipt",
@@ -135,6 +140,36 @@ def get_instrument_from_data(data, parent_source: str | None = None) -> Instrume
         return instrument
 
 
+def _delist_existing_duplicates(instrument: Instrument) -> None:
+    """Handle duplicate instruments by delisting existing entries"""
+    unique_identifiers = ["refinitiv_identifier_code", "refinitiv_mnemonic_code", "isin", "sedol", "valoren", "cusip"]
+
+    for identifier_field in unique_identifiers:
+        if identifier := getattr(instrument, identifier_field):
+            with suppress(Instrument.DoesNotExist):
+                duplicate = Instrument.objects.get(
+                    is_security=True, delisted_date__isnull=True, **{identifier_field: identifier}
+                )
+                duplicate.delisted_date = date.today() - timedelta(days=1)
+                duplicate.save()
+
+
+def _save_single_instrument(instrument: Instrument) -> None:
+    """Attempt to save an instrument with duplicate handling"""
+    try:
+        instrument.save()
+    except (UniqueViolation, IntegrityError) as e:
+        if instrument.is_security:
+            _delist_existing_duplicates(instrument)
+            try:
+                instrument.save()
+                logger.info(f"{instrument} successfully saved after automatic delisting")
+            except (UniqueViolation, IntegrityError) as e:
+                logger.error(f"Persistent integrity error: {e}")
+        else:
+            logger.error(f"Non-security instrument error: {e}")
+
+
 def _bulk_create_instruments_chunk(instruments: list[Instrument], update_unique_identifiers: bool = False):
     update_fields = [
         "name",
@@ -168,9 +203,17 @@ def _bulk_create_instruments_chunk(instruments: list[Instrument], update_unique_
             ]
         )
         bulk_update_kwargs = {"ignore_conflicts": True}
-    Instrument.objects.bulk_create(
-        instruments, update_fields=update_fields, unique_fields=["source", "source_id"], **bulk_update_kwargs
-    )
+    try:
+        Instrument.objects.bulk_create(
+            instruments, update_fields=update_fields, unique_fields=["source", "source_id"], **bulk_update_kwargs
+        )
+    except IntegrityError:
+        # we caught an integrity error on the bulk save, so we try to save one by one
+        logger.info(
+            "we detected an integrity error while bulk saving instruments. We save them one by one and delist the already existing instrument from the db if we can. "
+        )
+        for instrument in instruments:
+            _save_single_instrument(instrument)
 
 
 def update_instruments(sql_name: str, parent_source: str | None = None, context=None, debug: bool = False, **kwargs):
@@ -211,7 +254,7 @@ def update_or_create_item(
         defaults=defaults,
     )
     instrument.dl_parameters.update(dl_parameters)
-    instrument.save()
+    _save_single_instrument(instrument)
 
     return instrument
 

wbfdm/tasks.py

@@ -84,6 +84,12 @@ def synchronize_exchanges_as_task():
 
 @shared_task(queue="portfolio")
 def full_synchronization_as_task():
+    # we get all instrument without name or where we would expect a parent and consider them for clean up.
+    qs = Instrument.objects.filter(prices__isnull=True).filter(
+        (Q(name="") & Q(name_repr="")) | (Q(source__in=["qa-ds2-security", "qa-ds2-quote"]) & Q(parent__isnull=True))
+    )
+    for instrument in qs:
+        instrument.delete()
     initialize_exchanges()
     initialize_instruments()
     with transaction.atomic():

wbfdm/viewsets/statements/statements.py

@@ -54,7 +54,11 @@ class StatementPandasViewSet(InstrumentMixin, ExportPandasAPIViewSet):
                 pf.CharField(key="external_code", label="Code"),
                 pf.CharField(key="external_description", label="Description"),
                 pf.SparklineField(key="progress", label="Yearly Trend", dimension="double"),
-                *[pf.FloatField(key=field, label=field, display_mode=DisplayMode.SHORTENED) for field in self.columns],
+                *[
+                    pf.FloatField(key=field, label=field, display_mode=DisplayMode.SHORTENED)
+                    for field in self.columns
+                    if field not in ["external_ordering", "external_code", "external_description", "progress"]
+                ],
             ]
         )
 

{wbfdm-1.51.2rc1.dist-info → wbfdm-1.51.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wbfdm
-Version: 1.51.2rc1
+Version: 1.51.5
 Summary: The workbench module ensures rapid access to diverse financial data (market, fundamental, forecasts, ESG), with features for storing instruments, classifying them, and conducting financial analysis.
 Author-email: Christopher Wittlinger <c.wittlinger@stainly.com>
 Requires-Dist: roman==4.*

{wbfdm-1.51.2rc1.dist-info → wbfdm-1.51.5.dist-info}/RECORD

@@ -5,7 +5,7 @@ wbfdm/enums.py,sha256=5AuUouk5uuSNmRc6e-SiBu4FPmHVTN60ol9ftiuVrAc,33041
 wbfdm/jinja2.py,sha256=pkIC1U-0rf6vn0DDEUzZ8dPYiTGEPY8LBTRMi9wYiuc,199
 wbfdm/preferences.py,sha256=8ghDcaapOMso1kjtNfKbSFykPUTxzqI5R77gM3BgiMs,927
 wbfdm/signals.py,sha256=PhAsFpQZF1YVe5UpedaRelUD_TVjemqRYm1HzV-bhmY,597
-wbfdm/tasks.py,sha256=rjbbDewuiDmfO54tBUwPiNV9AoDKnIx382riyExWwnI,4362
+wbfdm/tasks.py,sha256=6rI2SH94ZGL4ycjNX8cntoYzUcb_wJciNmAomGWBILs,4711
 wbfdm/urls.py,sha256=pDp9I0kktxicad8sXXEUT7402jZPMJNcE5R1doTlcMw,8887
 wbfdm/utils.py,sha256=4cWrCpqXxHIjtSlt4DDPFvmtaqXw_H0nqhM6sGuXx0o,1938
 wbfdm/admin/__init__.py,sha256=Z1VtH_gjD71K79KcD-2Q2Lu_p_7j0akMZj7gNxdz1CQ,1398
@@ -28,8 +28,8 @@ wbfdm/analysis/financial_analysis/__init__.py,sha256=l0hGfgYIO4VAkSCVrMyzjvJ81fC
 wbfdm/analysis/financial_analysis/financial_metric_analysis.py,sha256=bZnt2zANYBpBiY_ZlDokzTx0iJESvXiNcbnX7lVRs9g,3491
 wbfdm/analysis/financial_analysis/financial_ratio_analysis.py,sha256=wx2ETs7XvNqhX-cUeIbhVKDiPDsnO39cQlT5qntv0GY,4623
 wbfdm/analysis/financial_analysis/financial_statistics_analysis.py,sha256=iABYkmWxVlF1H1zZiohbybxSlQXvpVlMkjU7336Zqww,13186
-wbfdm/analysis/financial_analysis/statement_with_estimates.py,sha256=Kv8phclh8XRVSwEktL3dIydrhqR_pQzz31v7iCK3hpI,24045
-wbfdm/analysis/financial_analysis/utils.py,sha256=Ve6WilLVTTTclL0KxNpdCcbdZJMbtpB9_5lZCHNXrIQ,13577
+wbfdm/analysis/financial_analysis/statement_with_estimates.py,sha256=8BFhqHnvaqy4VwM0fZ6tljhQIuDbNHvl8B250LqXQR8,23989
+wbfdm/analysis/financial_analysis/utils.py,sha256=tV7w0k7EyJ_3eaHwn2u90imFTUc-Tsyqukf11cGFVdI,13992
 wbfdm/analysis/technical_analysis/__init__.py,sha256=nQdpDl2okzWID6UT7fyfl36RSLNO1ZZzxRt_mkGVVkY,50
 wbfdm/analysis/technical_analysis/technical_analysis.py,sha256=3ZQOMUWovP6Fq0jOapHGcvBJmisWjEI4TcU5Qb0x8J0,4962
 wbfdm/analysis/technical_analysis/traces.py,sha256=GhyvVzdg1fmG5i1fai2zz9BnJ__5NH1-eKml2owXIO4,6484
@@ -61,7 +61,7 @@ wbfdm/contrib/metric/admin/metrics.py,sha256=SJU-5ona3npgRjTgDoHvo9aCztq-VMnIVvv
 wbfdm/contrib/metric/backends/__init__.py,sha256=Bl7bGEYVx38uiN2BS-0pr2WpIkbhUl7yPCEwt1W4tUg,128
 wbfdm/contrib/metric/backends/base.py,sha256=TTTcNHjuzGCnzoDBqYZOb-nT7fnuYjjp0D6ACqR5yXk,6765
 wbfdm/contrib/metric/backends/performances.py,sha256=yt9E7A_OEWQBQabf9bqt4Rc3S6oUXbSeM6NR1nYidtg,11642
-wbfdm/contrib/metric/backends/statistics.py,sha256=GeU4QOVGcz_Iri_18ku7PqhDDfPEpZpQJssBtCL6RsE,7903
+wbfdm/contrib/metric/backends/statistics.py,sha256=EFbTuJpKxpntkSk4Qo0BSCFFjxBqaQ7ODUkX2pbjDyY,7894
 wbfdm/contrib/metric/backends/utils.py,sha256=xpsDmoL2Jv9-KTuo-ay65D4m9Kf35jGc9cbeejhFLQU,131
 wbfdm/contrib/metric/migrations/0001_initial.py,sha256=EwXRJrG7zQYT1bD6jjaf1gZNfpd8f2gzta09GYqukiM,3403
 wbfdm/contrib/metric/migrations/0002_remove_instrumentmetric_unique_instrument_metric_and_more.py,sha256=xp6MACDohMSs4YfFbprCLa5G7zEuBhfZgpVXyMsZPWI,808
@@ -114,7 +114,7 @@ wbfdm/contrib/qa/jinja2/qa/sql/ibes/estimates.sql,sha256=OWgeogSFj7-OdXvJTvNsThN
 wbfdm/contrib/qa/jinja2/qa/sql/ibes/financials.sql,sha256=0jXNMdX8ixVRJ8YyvtoJRQ510pHFYEioMy7NTp5ecck,2582
 wbfdm/contrib/qa/sync/exchanges.py,sha256=XU7dj-rQzMlDku9lnmAACaTRoxx8pFSyr5kCK79cYAc,3124
 wbfdm/contrib/qa/sync/instruments.py,sha256=8aTQVJ_cw1phe4FWikn79pjCfUijaTcwkdhQCtSXKH0,3156
-wbfdm/contrib/qa/sync/utils.py,sha256=1diZFEMQwjdGhfL6jv2xX7N_-TyO0Fwtnqw_3K5ykYI,9326
+wbfdm/contrib/qa/sync/utils.py,sha256=582wwGFRRqBxq_H45sTPe0GqnWemy9uU1cYDqQ3BtKs,11249
 wbfdm/dataloaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wbfdm/dataloaders/cache.py,sha256=K9BeVxT7p-BMvjurINt18bfrUDccp840uIjfDBLJRNk,4841
 wbfdm/dataloaders/protocols.py,sha256=xFbexmhh38MWmkWsOSsfcNxdtrrWfO02qNihsqwA_BQ,2987
@@ -354,9 +354,9 @@ wbfdm/viewsets/instruments/instruments.py,sha256=-UG8fceQumx06xmZ6tRnjFvFu_Jrjz5
 wbfdm/viewsets/instruments/instruments_relationships.py,sha256=D2Ym84zXBFegQC9nMThKSWqSAcWa0WvcS_GXxM2po68,11076
 wbfdm/viewsets/instruments/utils.py,sha256=kr5HwkSaWp-6GHVEQIb3CnIspDGSxfqghyOqg0oHO-A,1166
 wbfdm/viewsets/statements/__init__.py,sha256=odxtFYUDICPmz8WCE3nx93EvKZLSPBEI4d7pozcQz2A,47
-wbfdm/viewsets/statements/statements.py,sha256=bmS89l60gnSb2NcoadkUt3GyrAR2JdQINH38-_Eg5Xs,4135
+wbfdm/viewsets/statements/statements.py,sha256=gA6RCI8-B__JwjEb6OZxpn8Y-9aF-YQ3HIQ7e1vfJMw,4304
 wbfdm/viewsets/technical_analysis/__init__.py,sha256=qtCIBg0uSiZeJq_1tEQFilnorMBkMe6uCMfqar6-cLE,77
 wbfdm/viewsets/technical_analysis/monthly_performances.py,sha256=O1j8CGfOranL74LqVvcf7jERaDIboEJZiBf_AbbVDQ8,3974
-wbfdm-1.51.2rc1.dist-info/METADATA,sha256=pNgk8s1BUhjPIkNxbvJ8cOk7iSZenAPhO8xNafU4Vvk,740
-wbfdm-1.51.2rc1.dist-info/WHEEL,sha256=tkmg4JIqwd9H8mL30xA7crRmoStyCtGp0VWshokd1Jc,105
-wbfdm-1.51.2rc1.dist-info/RECORD,,
+wbfdm-1.51.5.dist-info/METADATA,sha256=TaKLWPf99XxSz7Uu_EMluMgcQOHChYLVun8pTrb9uzc,737
+wbfdm-1.51.5.dist-info/WHEEL,sha256=tkmg4JIqwd9H8mL30xA7crRmoStyCtGp0VWshokd1Jc,105
+wbfdm-1.51.5.dist-info/RECORD,,