wbfdm 1.49.5__py2.py3-none-any.whl → 1.59.4__py2.py3-none-any.whl

wbfdm/contrib/metric/viewsets/configs/display.py

@@ -17,21 +17,21 @@ class InstrumentMetricDisplayConfig(DisplayViewConfig):
     def get_list_display(self) -> Optional[dp.ListDisplay]:
         return dp.ListDisplay(
             fields=[
+                dp.Field(key="basket_repr", label="Basket", pinned="left"),
                 dp.Field(key="key", label="Key"),
                 dp.Field(key="date", label="Date"),
                 dp.Field(key="instrument", label="Instrument"),
                 dp.Field(key="parent_metric", label="Parent Metric"),
             ],
             tree=True,
-            tree_group_pinned="left",
             tree_group_field="basket_repr",
-            tree_group_label="Basket",
             tree_group_level_options=[
                 dp.TreeGroupLevelOption(
                     filter_key="parent_metric",
                     filter_depth=1,
                     # lookup="id_repr",
                     clear_filter=True,
+                    filter_blacklist=["parent__isnull"],
                     list_endpoint=reverse(
                         "metric:instrumentmetric-list",
                         args=[],

wbfdm/contrib/metric/viewsets/mixins.py

@@ -185,19 +185,19 @@ class InstrumentMetricMixin(_Base):
         serializer_class = self.get_serializer_class()
         kwargs.setdefault("context", self.get_serializer_context())
 
-        BaseMeta = serializer_class.Meta
-        fields = list(getattr(BaseMeta, "fields", ()))
-        read_only_fields = list(getattr(BaseMeta, "read_only_fields", ()))
+        base_meta = serializer_class.Meta
+        fields = list(getattr(base_meta, "fields", ()))
+        read_only_fields = list(getattr(base_meta, "read_only_fields", ()))
         for extra_field in self._metric_serializer_fields.keys():
             fields.append(extra_field)
             read_only_fields.append(extra_field)
 
-        Meta = type(str("Meta"), (BaseMeta,), {"fields": fields, "read_only_fields": read_only_fields})
+        meta = type(str("Meta"), (base_meta,), {"fields": fields, "read_only_fields": read_only_fields})
         new_class = type(
             serializer_class.__name__,
             (serializer_class,),
             {
-                "Meta": Meta,
+                "Meta": meta,
                 **self._metric_serializer_fields,
                 "SERIALIZER_CLASS_FOR_REMOTE_ADDITIONAL_RESOURCES": serializer_class,
             },
@@ -233,7 +233,7 @@ class InstrumentMetricMixin(_Base):
             # for all the missings keys (not present in the aggregates already), we compute the aggregatation based on the aggregate function given by the MetricField class
             missing_aggregate_map = {}
             for metric_key in self.metric_keys:
-                for field_key, field_filter in metric_key.subfields_filter_map.items():
+                for field_key in metric_key.subfields_filter_map.keys():
                     if field_key in self._metric_serializer_fields.keys() and field_key not in aggregates:
                         missing_aggregate_map[field_key] = metric_key.subfields_map[field_key]
             missing_aggregate = queryset.aggregate(

wbfdm/contrib/msci/client.py

@@ -38,6 +38,7 @@ class MSCIClient:
                     "grant_type": "client_credentials",
                     "audience": "https://esg/data",
                 },
+                timeout=10,
             )
             if resp.status_code == requests.codes.ok:
                 with suppress(KeyError, requests.exceptions.JSONDecodeError):
@@ -58,6 +59,7 @@ class MSCIClient:
                     "factor_name_list": factors,
                 },
                 headers={"AUTHORIZATION": f"Bearer {self.oauth_token}"},
+                timeout=10,
             )
             if response.ok:
                 for row in response.json().get("result", {}).get("issuers", []):
@@ -66,6 +68,7 @@ class MSCIClient:
     def controversies(self, identifiers: list[str], factors: list[str]) -> Generator[dict[str, str], None, None]:
         next_url = "https://api2.msci.com/esg/data/v2.0/issuers"
         offset = 0
+        limit = 100
         while next_url:
             with suppress(ConnectionError):
                 response = requests.post(
@@ -73,10 +76,11 @@ class MSCIClient:
                     json={
                         "issuer_identifier_list": identifiers,
                         "factor_name_list": factors,
-                        "limit": 100,
+                        "limit": limit,
                         "offset": offset,
                     },
                     headers={"AUTHORIZATION": f"Bearer {self.oauth_token}"},
+                    timeout=10,
                 )
 
                 if not response.ok:
@@ -87,6 +91,6 @@ class MSCIClient:
                         next_url = json_res["paging"]["links"]["next"]
                     except KeyError:
                         next_url = None
-                    offset += 100
+                    offset += limit
                     for row in json_res.get("result", {}).get("issuers", []):
                         yield row

wbfdm/contrib/qa/database_routers.py

@@ -20,6 +20,6 @@ class QARouter:
         return None
 
     def allow_migrate(self, db, app_label, model_name=None, **hints):
-        if app_label == "qa":
+        if db == "qa":
             return False
         return None

wbfdm/contrib/qa/dataloaders/adjustments.py

@@ -57,7 +57,8 @@ class DatastreamAdjustmentsDataloader(AdjustmentsProtocol, Dataloader):
                     pk.MSSQLQuery.create_table(infocode).columns(pk.Column("infocode", SqlTypes.INTEGER)).get_sql()
                 )
                 for batch in batched(lookup.keys(), 1000):
-                    cursor.execute(f"insert into #ds2infocode values {",".join(map(lambda x: f"({x})", batch))};")
+                    placeholders = ",".join(map(lambda x: f"({x})", batch))
+                    cursor.execute("insert into #ds2infocode values %s;", (placeholders,))
 
             cursor.execute(query.get_sql())
 

wbfdm/contrib/qa/dataloaders/corporate_actions.py

@@ -58,7 +58,8 @@ class DatastreamCorporateActionsDataloader(CorporateActionsProtocol, Dataloader)
                     pk.MSSQLQuery.create_table(infocode).columns(pk.Column("infocode", SqlTypes.INTEGER)).get_sql()
                 )
                 for batch in batched(lookup.keys(), 1000):
-                    cursor.execute(f"insert into #ds2infocode values {','.join(map(lambda x: f'({x})', batch))};")
+                    placeholders = ",".join(map(lambda x: f"({x})", batch))
+                    cursor.execute("insert into #ds2infocode values %s;", (placeholders,))
 
             cursor.execute(query.get_sql())
             for row in dictfetchall(cursor, CorporateActionDataDict):

wbfdm/contrib/qa/dataloaders/financials.py

@@ -8,6 +8,7 @@ from jinjasql import JinjaSql  # type: ignore
 from wbcore.contrib.dataloader.dataloaders import Dataloader
 from wbcore.contrib.dataloader.utils import dictfetchall
 
+from wbfdm.contrib.qa.dataloaders.fx_rates import FXRateConverter
 from wbfdm.dataloaders.protocols import FinancialsProtocol
 from wbfdm.dataloaders.types import FinancialDataDict
 from wbfdm.enums import (
@@ -20,7 +21,7 @@ from wbfdm.enums import (
 )
 
 
-class IBESFinancialsDataloader(FinancialsProtocol, Dataloader):
+class IBESFinancialsDataloader(FXRateConverter, FinancialsProtocol, Dataloader):
     def financials(
         self,
         values: list[Financial],
@@ -40,6 +41,16 @@ class IBESFinancialsDataloader(FinancialsProtocol, Dataloader):
         target_currency: str | None = None,
     ) -> Iterator[FinancialDataDict]:
         lookup = {k: v for k, v in self.entities.values_list("dl_parameters__financials__parameters", "id")}
+        if from_year and not from_date:
+            from_date = date(year=from_year, month=1, day=1)
+        if to_date and not to_date:
+            to_date = date(year=to_year + 1, month=1, day=1)
+
+        if target_currency:
+            if not from_date or not to_date:
+                raise ValueError("From date and to date needs to be properly specified to allow fx rate convertion")
+            self.load_fx_rates(self.entities, target_currency, from_date, to_date)
+
         for batch in batched(lookup.keys(), 1000):
             sql = ""
             if series_type == SeriesType.COMPLETE:
@@ -81,4 +92,11 @@ class IBESFinancialsDataloader(FinancialsProtocol, Dataloader):
                     row["value_low"] = row.get("value_low", row["value"])
                     row["value_amount"] = row.get("value_amount", row["value"])
                     row["value_stdev"] = row.get("value_stdev", row["value"])
+                    if target_currency:
+                        row = self.apply_fx_rate(
+                            row,
+                            ["value", "value_high", "value_low", "value_amount"],
+                            apply=True,
+                            date_label="period_end_date",
+                        )
                     yield row

wbfdm/contrib/qa/dataloaders/fx_rates.py

@@ -0,0 +1,86 @@
+from collections import defaultdict
+from datetime import date, timedelta
+from typing import Iterator
+
+import pypika as pk
+from django.db import connections
+from pypika import Case
+from pypika import functions as fn
+from pypika.enums import Order, SqlTypes
+from wbcore.contrib.dataloader.dataloaders import Dataloader
+from wbcore.contrib.dataloader.utils import dictfetchall
+
+from wbfdm.dataloaders.protocols import FXRateProtocol
+from wbfdm.dataloaders.types import FXRateDict
+
+
+class FXRateConverter:
+    fx_rates: dict[str, dict[date, float]]
+    target_currency: str
+
+    def load_fx_rates(self, entities, target_currency: str, from_date: date, to_date: date):
+        fx_rates = defaultdict(dict)
+        self.target_currency = target_currency
+        if from_date:
+            for fx_rate in DatastreamFXRatesDataloader(entities).fx_rates(
+                from_date, to_date if to_date else date.today(), target_currency
+            ):
+                fx_rates[fx_rate["currency_pair"]][fx_rate["fx_date"]] = fx_rate["fx_rate"]
+        self.fx_rates = fx_rates
+
+    def apply_fx_rate(self, row, currency_columns: list[str], apply: bool = True, date_label: str = "valuation_date"):
+        if row["currency"]:
+            fx_rate = None
+            if self.target_currency == row["currency"]:
+                fx_rate = 1.0
+            else:
+                currency_fx_rates = self.fx_rates[f'{row["currency"]}{self.target_currency}']
+                if currency_fx_rates:
+                    try:
+                        fx_rate = currency_fx_rates[row[date_label]] or 1.0
+                    except KeyError:
+                        max_idx = max(currency_fx_rates.keys())
+                        fx_rate = currency_fx_rates[max_idx]
+            if apply and fx_rate is not None:
+                for col in currency_columns:
+                    if v := row.get(col):
+                        row[col] = v * fx_rate
+                row["currency"] = self.target_currency
+            row["fx_rate"] = fx_rate
+        return row
+
+
+class DatastreamFXRatesDataloader(FXRateProtocol, Dataloader):
+    def fx_rates(
+        self,
+        from_date: date,
+        to_date: date,
+        target_currency: str,
+    ) -> Iterator[FXRateDict]:
+        currencies = list(self.entities.values_list("currency__key", flat=True))
+        # Define tables
+        fx_rate = pk.Table("DS2FxRate")
+        fx_code = pk.Table("DS2FxCode")
+
+        # Base query to get data we always need unconditionally
+        query = (
+            pk.MSSQLQuery.from_(fx_rate)
+            # We join on _codes, which removes all instruments not in _codes - implicit where
+            .join(fx_code)
+            .on(fx_rate.ExRateIntCode == fx_code.ExRateIntCode)
+            .where((fx_rate.ExRateDate >= from_date) & (fx_rate.ExRateDate <= to_date + timedelta(days=1)))
+            .where(
+                (fx_code.ToCurrCode == target_currency)
+                & (fx_code.FromCurrCode.isin(currencies))
+                & (fx_code.RateTypeCode == "SPOT")
+            )
+            .orderby(fx_rate.ExRateDate, order=Order.desc)
+            .select(
+                fn.Cast(fx_rate.ExRateDate, SqlTypes.DATE).as_("fx_date"),
+                fn.Concat(fx_code.FromCurrCode, fx_code.ToCurrCode).as_("currency_pair"),
+                (Case().when(fx_code.FromCurrCode == target_currency, 1).else_(1 / fx_rate.midrate)).as_("fx_rate"),
+            )
+        )
+        with connections["qa"].cursor() as cursor:
+            cursor.execute(query.get_sql())
+            yield from dictfetchall(cursor, FXRateDict)

wbfdm/contrib/qa/dataloaders/market_data.py

@@ -7,13 +7,14 @@ from typing import TYPE_CHECKING, Iterator
 
 import pypika as pk
 from django.db import ProgrammingError, connections
-from pypika import Case, Column, MSSQLQuery
+from pypika import Column, MSSQLQuery
 from pypika import functions as fn
 from pypika.enums import Order, SqlTypes
 from pypika.terms import ValueWrapper
 from wbcore.contrib.dataloader.dataloaders import Dataloader
 from wbcore.contrib.dataloader.utils import dictfetchall
 
+from wbfdm.contrib.qa.dataloaders.fx_rates import FXRateConverter
 from wbfdm.contrib.qa.dataloaders.utils import create_table
 from wbfdm.dataloaders.protocols import MarketDataProtocol
 from wbfdm.dataloaders.types import MarketDataDict
@@ -36,15 +37,16 @@ class DS2MarketData(Enum):
     SHARES_OUTSTANDING = "NumShrs"
 
 
-class DatastreamMarketDataDataloader(MarketDataProtocol, Dataloader):
-    def market_data(
+class DatastreamMarketDataDataloader(FXRateConverter, MarketDataProtocol, Dataloader):
+    def market_data(  # noqa: C901
         self,
-        values: list[MarketData] = [MarketData.CLOSE],
+        values: list[MarketData] | None = None,
         from_date: date | None = None,
         to_date: date | None = None,
         exact_date: date | None = None,
         frequency: Frequency = Frequency.DAILY,
         target_currency: str | None = None,
+        apply_fx_rate: bool = True,
         **kwargs,
     ) -> Iterator[MarketDataDict]:
         """Get market data for instruments.
@@ -59,18 +61,21 @@ class DatastreamMarketDataDataloader(MarketDataProtocol, Dataloader):
         Returns:
             Iterator[MarketDataDict]: An iterator of dictionaries conforming to the DailyValuationDict.
         """
-
         lookup = {
             f"{k[0]},{k[1]}": v for k, v in self.entities.values_list("dl_parameters__market_data__parameters", "id")
         }
-
+        if exact_date:
+            from_date = exact_date
+            to_date = exact_date
+        if target_currency:
+            self.load_fx_rates(self.entities, target_currency, from_date, to_date)
         # Define tables
         pricing = pk.Table("vw_DS2Pricing")
 
         mapping, create_mapping_table = create_table(
             "#ds2infoexchcode", Column("InfoCode", SqlTypes.INTEGER), Column("ExchIntCode", SqlTypes.INTEGER)
         )
-
+        currency_columns = [e.value for e in MarketData if e.value != MarketData.SHARES_OUTSTANDING]
         # Base query to get data we always need unconditionally
         query = (
             pk.MSSQLQuery.from_(pricing)
@@ -89,31 +94,7 @@ class DatastreamMarketDataDataloader(MarketDataProtocol, Dataloader):
             .orderby(pricing.MarketDate, order=Order.desc)
         )
 
-        # if a target currency is required, we join on the fx tables and set the currency to the desired one
-        # otherwise we just set the currency to whatever the currency is from the instrument
-        fx_rate = None
-        if target_currency:
-            query = query.select(ValueWrapper(target_currency).as_("currency"))
-            fx_code = pk.Table("DS2FxCode")
-            fx_rate = pk.Table("DS2FxRate")
-            query = (
-                query
-                # Join FX code table matching currencies and ensuring SPOT rate type
-                .left_join(fx_code)
-                .on(
-                    (fx_code.FromCurrCode == pricing.Currency)
-                    & (fx_code.ToCurrCode == target_currency)
-                    & (fx_code.RateTypeCode == "SPOT")
-                )
-                # Join FX rate table matching internal code and date
-                .left_join(fx_rate)
-                .on((fx_rate.ExRateIntCode == fx_code.ExRateIntCode) & (fx_rate.ExRateDate == pricing.MarketDate))
-                # We filter out rows which do not have a proper fx rate (we exclude same currency conversions)
-                .where((Case().when(pricing.Currency == target_currency, 1).else_(fx_rate.midrate).isnotnull()))
-            )
-
-        else:
-            query = query.select(pricing.Currency.as_("currency"))
+        query = query.select(pricing.Currency.as_("currency"))
 
         # if market cap or shares outstanding are required we need to join with an additional table
         if MarketData.MARKET_CAPITALIZATION in values or MarketData.SHARES_OUTSTANDING in values:
@@ -127,9 +108,6 @@ class DatastreamMarketDataDataloader(MarketDataProtocol, Dataloader):
             )
 
             value = pricing_2.Close_
-            if fx_rate:
-                value /= Case().when(pricing_2.Currency == target_currency, 1).else_(fx_rate.midrate)
-
             query = query.select(value.as_("undadjusted_close"))
             query = query.select(
                 MSSQLQuery.from_(num_shares)
@@ -140,14 +118,24 @@ class DatastreamMarketDataDataloader(MarketDataProtocol, Dataloader):
                 .as_("unadjusted_outstanding_shares")
             )
 
+        if MarketData.MARKET_CAPITALIZATION_CONSOLIDATED in values:
+            mkt_val = pk.Table("DS2MktVal")
+            query = query.left_join(mkt_val).on(
+                (mkt_val.InfoCode == pricing.InfoCode) & (mkt_val.ValDate == pricing.MarketDate)
+            )
+            query = query.select((mkt_val.ConsolMktVal * 1_000_000).as_("market_capitalization_consolidated"))
+
         for market_data in filter(
-            lambda x: x not in (MarketData.SHARES_OUTSTANDING, MarketData.MARKET_CAPITALIZATION), values
+            lambda x: x
+            not in (
+                MarketData.SHARES_OUTSTANDING,
+                MarketData.MARKET_CAPITALIZATION,
+                MarketData.MARKET_CAPITALIZATION_CONSOLIDATED,
+            ),
+            values,
         ):
             ds2_value = DS2MarketData[market_data.name].value
             value = getattr(pricing, ds2_value)
-            if fx_rate and market_data is not MarketData.SHARES_OUTSTANDING:
-                value /= Case().when(pricing.Currency == target_currency, 1).else_(fx_rate.midrate)
-
             query = query.select(value.as_(market_data.value))
 
         # Add conditional where clauses
@@ -186,7 +174,8 @@ class DatastreamMarketDataDataloader(MarketDataProtocol, Dataloader):
 
                 if MarketData.SHARES_OUTSTANDING in values:
                     row["outstanding_shares"] = (row["market_capitalization"] / row["close"]) if row["close"] else None
-
+                if target_currency:
+                    row = self.apply_fx_rate(row, currency_columns, apply=apply_fx_rate)
                 yield row
 
             cursor.execute(MSSQLQuery.drop_table(mapping).get_sql())

wbfdm/contrib/qa/dataloaders/officers.py

@@ -60,7 +60,7 @@ class RKDOfficersDataloader(OfficersProtocol, Dataloader):
                 )
                 for batch in batched(lookup.keys(), 1000):
                     placeholders = ",".join(map(lambda x: f"('{x}')", batch))
-                    cursor.execute(f"insert into #rkd_codes values {placeholders};")
+                    cursor.execute("insert into #rkd_codes values %s;", (placeholders,))
 
             cursor.execute(query.get_sql())
 

wbfdm/contrib/qa/dataloaders/statements.py

@@ -7,6 +7,7 @@ from jinjasql import JinjaSql  # type: ignore
 from wbcore.contrib.dataloader.dataloaders import Dataloader
 from wbcore.contrib.dataloader.utils import dictfetchall
 
+from wbfdm.contrib.qa.dataloaders.fx_rates import FXRateConverter
 from wbfdm.dataloaders.protocols import StatementsProtocol
 from wbfdm.dataloaders.types import StatementDataDict
 from wbfdm.enums import DataType, Financial, PeriodType, StatementType
@@ -224,7 +225,7 @@ standardized_sql = """
 """
 
 
-class RKDStatementsDataloader(StatementsProtocol, Dataloader):
+class RKDStatementsDataloader(FXRateConverter, StatementsProtocol, Dataloader):
     def statements(
         self,
         statement_type: StatementType | None = None,
@@ -239,6 +240,15 @@ class RKDStatementsDataloader(StatementsProtocol, Dataloader):
     ) -> Iterator[StatementDataDict]:
         lookup = {k: v for k, v in self.entities.values_list("dl_parameters__statements__parameters", "id")}
         sql = reported_sql if data_type is DataType.REPORTED else standardized_sql
+        if not financials:
+            financials = []
+        external_codes = [RKDFinancial[fin.name].value for fin in financials if fin.name in RKDFinancial.__members__]
+        if from_year and not from_date:
+            from_date = date(year=from_year, month=1, day=1)
+        if to_date and not to_date:
+            to_date = date(year=to_year + 1, month=1, day=1)
+        if target_currency:
+            self.load_fx_rates(self.entities, target_currency, from_date, to_date)
         query, bind_params = JinjaSql(param_style="format").prepare_query(
             sql,
             {
@@ -249,7 +259,7 @@ class RKDStatementsDataloader(StatementsProtocol, Dataloader):
                 "from_date": from_date,
                 "to_date": to_date,
                 "period_type": period_type.value,
-                "external_codes": [RKDFinancial[fin.name].value for fin in financials or []],
+                "external_codes": external_codes,
             },
         )
         with connections["qa"].cursor() as cursor:
@@ -264,5 +274,10 @@ class RKDStatementsDataloader(StatementsProtocol, Dataloader):
                 row["year"] = int(row["year"] or row["period_end_date"].year)
                 row["instrument_id"] = lookup[row["external_identifier"]]
                 if financials:
-                    row["financial"] = Financial[RKDFinancial(row["external_code"]).name].value
+                    try:
+                        row["financial"] = Financial[RKDFinancial(row["external_code"]).name].value
+                    except (ValueError, KeyError):
+                        continue
+                    if target_currency:
+                        row = self.apply_fx_rate(row, ["value"], apply=True, date_label="period_end_date")
                 yield row

wbfdm/contrib/qa/jinja2/qa/sql/ibes/financials.sql

@@ -52,7 +52,7 @@ where
                 or (fin.ExpireDate is null and ExpireDate is null)
             )
     )
-
+    and fin.isParent = 'false'
     and fin.EstPermID in (
         {% for instrument in instruments %}
             {{ instrument }}{% if not loop.last %},{% endif %}

wbfdm/contrib/qa/sync/exchanges.py

@@ -32,7 +32,8 @@ class QAExchangeSync(Sync[Exchange]):
 
     def update_or_create_item(self, external_id: int) -> Exchange:
         defaults = self.get_item(external_id)
-        defaults["country_id"] = mapping(Geography.countries, "code_2").get(defaults["country_id"])
+        if country_id := defaults.get("country_id"):
+            defaults["country_id"] = mapping(Geography.countries, "code_2").get(country_id)
         exchange, _ = Exchange.objects.update_or_create(
             source=self.SOURCE,
             source_id=external_id,

wbfdm/contrib/qa/sync/utils.py

@@ -1,12 +1,17 @@
+import logging
 from contextlib import suppress
+from datetime import date, timedelta
 from typing import Callable
 
 import pytz
 from django.conf import settings
-from django.db import connections
+from django.core.exceptions import ObjectDoesNotExist
+from django.db import IntegrityError, connections
 from django.db.models import Max, QuerySet
 from django.template.loader import get_template
 from jinjasql import JinjaSql  # type: ignore
+from mptt.exceptions import InvalidMove
+from psycopg.errors import UniqueViolation
 from tqdm import tqdm
 from wbcore.contrib.currency.models import Currency
 from wbcore.contrib.dataloader.utils import dictfetchall, dictfetchone
@@ -15,6 +20,8 @@ from wbcore.utils.cache import mapping
 from wbfdm.models.exchanges.exchanges import Exchange
 from wbfdm.models.instruments.instruments import Instrument, InstrumentType
 
+logger = logging.getLogger("pms")
+
 BATCH_SIZE: int = 10000
 instrument_type_map = {
     "ADR": "american_depository_receipt",
@@ -135,6 +142,39 @@ def get_instrument_from_data(data, parent_source: str | None = None) -> Instrume
         return instrument
 
 
+def _delist_existing_duplicates(instrument: Instrument) -> None:
+    """Handle duplicate instruments by delisting existing entries"""
+    unique_identifiers = ["refinitiv_identifier_code", "refinitiv_mnemonic_code", "isin", "sedol", "valoren", "cusip"]
+
+    for identifier_field in unique_identifiers:
+        if identifier := getattr(instrument, identifier_field):
+            if instrument.delisted_date:  # if delisted, we unset the identifier that can lead to constraint error
+                setattr(instrument, identifier_field, None)
+            else:
+                with suppress(Instrument.DoesNotExist):
+                    duplicate = Instrument.objects.get(
+                        is_security=True, delisted_date__isnull=True, **{identifier_field: identifier}
+                    )
+                    duplicate.delisted_date = date.today() - timedelta(days=1)
+                    duplicate.save()
+
+
+def _save_single_instrument(instrument: Instrument) -> None:
+    """Attempt to save an instrument with duplicate handling"""
+    try:
+        instrument.save()
+    except (UniqueViolation, IntegrityError) as e:
+        if instrument.is_security:
+            _delist_existing_duplicates(instrument)
+            try:
+                instrument.save()
+                logger.info(f"{instrument} successfully saved after automatic delisting")
+            except (UniqueViolation, IntegrityError) as e:
+                logger.error(f"Persistent integrity error: {e}")
+        else:
+            logger.error(f"Non-security instrument error: {e}")
+
+
 def _bulk_create_instruments_chunk(instruments: list[Instrument], update_unique_identifiers: bool = False):
     update_fields = [
         "name",
@@ -168,9 +208,17 @@ def _bulk_create_instruments_chunk(instruments: list[Instrument], update_unique_
             ]
         )
         bulk_update_kwargs = {"ignore_conflicts": True}
-    Instrument.objects.bulk_create(
-        instruments, update_fields=update_fields, unique_fields=["source", "source_id"], **bulk_update_kwargs
-    )
+    try:
+        Instrument.objects.bulk_create(
+            instruments, update_fields=update_fields, unique_fields=["source", "source_id"], **bulk_update_kwargs
+        )
+    except IntegrityError:
+        # we caught an integrity error on the bulk save, so we try to save one by one
+        logger.error(
+            "we detected an integrity error while bulk saving instruments. We save them one by one and delist the already existing instrument from the db if we can. "
+        )
+        for instrument in instruments:
+            _save_single_instrument(instrument)
 
 
 def update_instruments(sql_name: str, parent_source: str | None = None, context=None, debug: bool = False, **kwargs):
@@ -198,22 +246,29 @@ def update_instruments(sql_name: str, parent_source: str | None = None, context=
 
 def update_or_create_item(
     external_id: int, get_item: Callable, source: str, parent_source: str | None = None
-) -> Instrument:
+) -> Instrument | None:
     defaults = convert_data(get_item(external_id), parent_source=parent_source)
 
     dl_parameters = defaults.pop("dl_parameters", {})
     defaults.pop("source", None)
     defaults.pop("source_id", None)
+    try:
+        instrument, _ = Instrument.objects.update_or_create(
+            source=source,
+            source_id=external_id,
+            defaults=defaults,
+        )
+        instrument.dl_parameters.update(dl_parameters)
+        _save_single_instrument(instrument)
+        return instrument
 
-    instrument, _ = Instrument.objects.update_or_create(
-        source=source,
-        source_id=external_id,
-        defaults=defaults,
-    )
-    instrument.dl_parameters.update(dl_parameters)
-    instrument.save()
-
-    return instrument
+    except (
+        InvalidMove,
+        ObjectDoesNotExist,
+    ):  # we might encounter a object does not exist error in case the inserted parent does not exist yet in our db which might happen if it was just deleted because invalid
+        logger.warning(
+            f"We encountered a MPTT ill-formed node for source ID {external_id}. Rebuilding the tree might be necessary."
+        )
 
 
 def get_item(external_id: int, template_name: str) -> dict:
@@ -238,13 +293,17 @@ def trigger_partial_update(
     else:
         with connections["qa"].cursor() as cursor:
             cursor.execute(
-                f"SELECT MAX(last_user_update) FROM sys.dm_db_index_usage_stats WHERE OBJECT_NAME(object_id) = '{table_change_name}'"
+                "SELECT MAX(last_user_update) FROM sys.dm_db_index_usage_stats WHERE OBJECT_NAME(object_id) = %s",
+                (table_change_name,),
             )
             max_last_updated_qa = (
                 pytz.timezone(settings.TIME_ZONE).localize(result[0]) if (result := cursor.fetchone()) else None
             )
             if max_last_updated_qa and max_last_updated_qa > max_last_updated:
                 for _, security_id in cursor.execute(
-                    f"SELECT UpdateFlag_, {id_field} FROM {table_change_name}"
+                    f"SELECT UpdateFlag_, {id_field} FROM {table_change_name}"  # noqa: S608
                 ).fetchall():
-                    update_or_create_item(security_id)
+                    try:
+                        update_or_create_item(security_id)
+                    except Exception as e:
+                        logger.error(f"Error updating instrument {security_id}: {e}")