wbfdm 1.49.5__py2.py3-none-any.whl → 1.59.4__py2.py3-none-any.whl

wbfdm/import_export/handlers/instrument.py

@@ -7,7 +7,8 @@ from typing import Any, Dict, Optional
 from django.contrib.postgres.search import TrigramSimilarity
 from django.core.exceptions import MultipleObjectsReturned
 from django.db import IntegrityError, models
-from django.db.models import Q
+from django.db.models import Exists, OuterRef, Q
+from slugify import slugify
 from wbcore.contrib.currency.import_export.handlers import CurrencyImportHandler
 from wbcore.contrib.geography.models import Geography
 from wbcore.contrib.io.exceptions import DeserializationError
@@ -24,6 +25,7 @@ class InstrumentLookup:
         "refinitiv_mnemonic_code",
         "identifier",
         "ticker",
+        "currency",
     ]
 
     def __init__(self, model, trigram_similarity_min_score: float = 0.8):
@@ -33,34 +35,20 @@ class InstrumentLookup:
 
     @classmethod
     def _get_cache_key(cls, **data):
-        if data:
-            return "-".join([f"{k}:{data.get(k, None)}" for k in cls.ORDERED_KEYS if data.get(k, None) is not None])
+        return "-".join([f"{k}:{slugify(str(data[k]))}" for k in cls.ORDERED_KEYS if data.get(k, None) is not None])
 
-    def _lookup_instrument(
-        self,
-        instrument_type=None,
-        currency=None,
-        exchange=None,
-        name=None,
-        only_investable_universe: bool = False,
-        exact_lookup: bool = False,
-        **identifiers,
-    ):
-        identifiers = {k: v for k, v in identifiers.items() if v is not None}
-        # General lookup, we try to gracefully find the instrument based on all available identifier fields
-        cache_key = self._get_cache_key(**identifiers)
+    def _get_cache(self, **kwargs):
+        cache_key = self._get_cache_key(**kwargs)
         if cache_key and cache_key in self.cache:
             return self.cache[cache_key]
 
-        instrument = None
-
-        # We need to lookup ticker because some provider gives us ticker with or without space in it
-        if only_investable_universe:
-            instruments = self.model.objects.filter(is_investable_universe=True)
-        else:
-            instruments = self.model.objects.filter(is_security=True)
+    def _set_cache(self, instrument, **kwargs):
+        cache_key = self._get_cache_key(**kwargs)
+        self.cache[cache_key] = instrument
 
+    def _get_instruments_from_identifiers(self, queryset, **identifiers):
         # Try exact lookup on the filtered out universe
+        identifiers = {k: v for k, v in identifiers.items() if v is not None}
         for identifier_key in [
             "isin",
             "refinitiv_identifier_code",
@@ -76,87 +64,153 @@ class InstrumentLookup:
                         identifier_key != "refinitiv_identifier_code"
                     ):  # RIC cannot be uppercased because its symbology implies meaning for lowercase characters
                         identifier = identifier.upper()
-                    instrument = instruments.get(**{identifier_key: identifier})
-                    break
-        if not instrument and not exact_lookup:
-            if instrument_type:
-                if isinstance(instrument_type, str):  # in case we receive a key as instrument type
-                    instruments = instruments.filter(instrument_type__key=instrument_type)
-                else:  # in case we receive a primary key as instrument type
-                    instruments = instruments.filter(instrument_type=instrument_type)
-            lookup_fields = [
-                "isin",
-                "refinitiv_identifier_code",
-                "refinitiv_mnemonic_code",
-                "refinitiv_ticker",
-                "identifier",
-                "ticker",
-            ]
-
-            conditions = []
-            for field in lookup_fields:
-                if field_value := identifiers.get(field, None):
-                    conditions.append(Q(**{f"{field}": field_value}))
-                    if field == "isin":
-                        conditions.append(Q(old_isins__contains=[field_value]))
-            if conditions:
-                instruments = instruments.filter(reduce(operator.or_, conditions))
-
-            if currency:  # if currency is provides, we use it as validator
-                instruments = instruments.filter(currency=currency)
-
-            if exchange:
-                instruments_tmp = instruments.filter(exchange=exchange)
-                if instruments_tmp.exists():
-                    instruments = instruments_tmp
-
-            # last chance
-            if name and instruments.count() > 1:
-                instruments = instruments.annotate(similarity_score=TrigramSimilarity("name", name))
-                if instruments.filter(similarity_score__gt=self.trigram_similarity_min_score).count() == 1:
-                    instruments = instruments.filter(similarity_score__gt=self.trigram_similarity_min_score)
-            if instruments.count() == 1:
-                instrument = instruments.first()
-            elif instrument_type and identifiers:
-                # if instrument type was provided but we still didn't find the security, we try without the instrument type in case it was mislabeled
-                instrument = self._lookup_instrument(
-                    only_investable_universe=only_investable_universe,
-                    exact_lookup=exact_lookup,
-                    currency=currency,
-                    exchange=exchange,
-                    **identifiers,
-                )
-            if not instrument and name and identifiers:
-                # Sometime, identifier provided emptied the queryset of possible instruments. In a last chance approach, we try to only look for security with the given name
-                instrument = self._lookup_instrument(
-                    only_investable_universe=only_investable_universe,
-                    exact_lookup=exact_lookup,
-                    instrument_type=instrument_type,
-                    currency=currency,
-                    exchange=exchange,
-                    name=name,
+                    # we try to lookup the instrument with the unique identifier
+                    instrument = queryset.get(**{identifier_key: identifier})
+                    return self.model.objects.filter(id=instrument.id)
+        return queryset
+
+    def _filter_identifiers(self, queryset, **identifiers):
+        lookup_fields = [
+            "isin",
+            "refinitiv_identifier_code",
+            "refinitiv_mnemonic_code",
+            "refinitiv_ticker",
+            "identifier",
+            "ticker",
+        ]
+        conditions = []
+        for field in lookup_fields:
+            if field_value := identifiers.get(field, None):
+                conditions.append(Q(**{f"{field}": field_value}))
+                if field == "isin":
+                    conditions.append(Q(old_isins__contains=[field_value]))
+                if field == "ticker":
+                    conditions.append(
+                        Q(ticker__regex=rf"^{field_value}(\.[A-Za-z])$")
+                    )  # initial regex: ([A-Za-z]?|\.?[A-Za-z])$ This led to a false positive, I don't remember why we allow an extra random character to be consider for eval
+                    conditions.append(Q(refinitiv_mnemonic_code=f"@{field_value}"))
+
+        if conditions:
+            return queryset.filter(reduce(operator.or_, conditions))
+        return queryset
+
+    def _filter_name(self, queryset, name):
+        instruments = queryset.annotate(similarity_score=TrigramSimilarity("name", name))
+        if (
+            queryset.count() > 1
+            and instruments.filter(similarity_score__gt=self.trigram_similarity_min_score).count() == 1
+        ):
+            return instruments.filter(similarity_score__gt=self.trigram_similarity_min_score)
+        return queryset
+
+    def _filter_currency(self, queryset, currency):
+        qs = queryset.filter(currency=currency)
+        if qs.exists():
+            return qs
+        return queryset
+
+    def _filter_exchange(self, queryset, exchange):
+        return queryset.filter(exchange=exchange)
+
+    def _filter_instrument_type(self, queryset, instrument_type):
+        if not isinstance(instrument_type, str):
+            instrument_type = instrument_type.key
+        if instrument_type == "equity":
+            qs = queryset.filter(instrument_type__key__in=["equity", "american_depository_receipt"])
+        else:
+            qs = queryset.filter(instrument_type__key=instrument_type)
+        if qs.exists():
+            return qs
+        return queryset
+
+    def _get_queryset(self, only_investable_universe: bool = False):
+        instruments = self.model.objects.filter(is_security=True)
+        if only_investable_universe:
+            instruments = instruments.annotate(
+                has_children_in_investable_universe=Exists(
+                    self.model.objects.filter(is_investable_universe=True, parent=OuterRef("id"))
                 )
-        if instrument:
-            self.cache[cache_key] = instrument
-        return instrument
+            ).filter(Q(is_investable_universe=True) | Q(has_children_in_investable_universe=True))
+        return instruments
 
-    def lookup(self, only_security: bool = False, exact_lookup: bool = False, **lookup_kwargs):
-        # To speed up lookup process, we try to get the quote from the investable universe first
-        security = self._lookup_instrument(only_investable_universe=True, exact_lookup=exact_lookup, **lookup_kwargs)
-        if not security:
-            security = self._lookup_instrument(
-                only_investable_universe=False, exact_lookup=exact_lookup, **lookup_kwargs
+    def _get_security_candidates(  # noqa: C901
+        self,
+        queryset,
+        instrument_type=None,
+        currency=None,
+        name=None,
+        **identifiers,
+    ):
+        # We need to lookup ticker because some provider gives us ticker with or without space in it
+        instruments = self._get_instruments_from_identifiers(queryset, **identifiers)
+        if instrument_type:
+            instruments = self._filter_instrument_type(instruments, instrument_type)
+
+        if instruments.count() == 1:
+            return instruments
+
+        instruments = self._filter_identifiers(instruments, **identifiers)
+
+        if currency:
+            instruments = self._filter_currency(instruments, currency)
+        if name:
+            instruments = self._filter_name(instruments, name)
+        return instruments
+
+    def _lookup_quote(
+        self,
+        security,
+        currency=None,
+        exchange=None,
+        refinitiv_identifier_code=None,
+        refinitiv_mnemonic_code=None,
+        **kwargs,
+    ):
+        quotes = security.children.all()
+        if not quotes.exists():
+            return security
+        if quotes.count() == 1:
+            return quotes.first()
+        if exchange:
+            quotes = self._filter_exchange(quotes, exchange)
+        if currency:
+            quotes = self._filter_currency(quotes, currency)
+        if refinitiv_mnemonic_code or refinitiv_identifier_code:
+            quotes = self._filter_identifiers(
+                quotes,
+                refinitiv_identifier_code=refinitiv_identifier_code,
+                refinitiv_mnemonic_code=refinitiv_mnemonic_code,
             )
-        if not only_security and security:
-            quotes = security.children.all()
-            if quotes.exists():
-                # We try to find the quote for that security based on the given exchange (if provided). Otherwise, we default to the security primary exchange
-                exchange = lookup_kwargs.get("exchange")
-                if exchange and quotes.filter(exchange=exchange).exists():
-                    return quotes.filter(exchange=exchange).first()
-                else:
-                    return quotes.filter(is_primary=True).first()
-        return security
+        if quotes.filter(is_investable_universe=True).count() == 1:
+            quotes = quotes.filter(is_investable_universe=True)
+        if quotes.count() == 1:
+            return quotes.first()
+        return security.children.filter(is_primary=True).first()
+
+    def lookup_security(self, **kwargs):
+        queryset = self._get_queryset(only_investable_universe=True)
+        security_candidates_in_investable_universe = self._get_security_candidates(queryset, **kwargs)
+
+        if security_candidates_in_investable_universe.count() == 1:
+            return security_candidates_in_investable_universe.first()
+        queryset = self._get_queryset(only_investable_universe=False)
+        security_candidates = self._get_security_candidates(queryset, **kwargs)
+
+        if security_candidates.exists() > 1:
+            security_candidates = security_candidates.filter(delisted_date__isnull=True)
+        if security_candidates.count() == 1:
+            return security_candidates.first()
+
+    def lookup(self, only_security: bool = False, **lookup_kwargs):
+        # To speed up lookup process, we try to get the quote from the investable universe first
+        if instrument := self._get_cache(**lookup_kwargs):
+            return instrument
+        instrument = self.lookup_security(**lookup_kwargs)
+        if not only_security and instrument:
+            instrument = self._lookup_quote(instrument, **lookup_kwargs)
+        if instrument:
+            self._set_cache(instrument, **lookup_kwargs)
+        return instrument
 
 
 class InstrumentImportHandler(ImportExportHandler):
@@ -175,7 +229,9 @@ class InstrumentImportHandler(ImportExportHandler):
         if isinstance(data, int):
             data = dict(id=data)
         if data.get("currency", None):
-            data["currency"] = self.currency_handler.process_object(data["currency"], read_only=True)[0]
+            data["currency"] = self.currency_handler.process_object(
+                data["currency"], read_only=True, raise_exception=False
+            )[0]
         if instrument_type := data.get("instrument_type", None):
             if isinstance(instrument_type, str):
                 data["instrument_type"] = InstrumentType.objects.get_or_create(
@@ -209,8 +265,8 @@ class InstrumentImportHandler(ImportExportHandler):
         if instrument_id := data.pop("id", None):
             try:
                 return self.model.objects.get(id=instrument_id)
-            except self.model.DoesNotExist:
-                raise DeserializationError("Instrument id does not match an existing instrument")
+            except self.model.DoesNotExist as e:
+                raise DeserializationError("Instrument id does not match an existing instrument") from e
         else:
             return self.instrument_lookup.lookup(only_security=only_security, **data)
 

wbfdm/import_export/handlers/option.py

@@ -26,7 +26,7 @@ class OptionAggregateImportHandler(ImportExportHandler):
 
     def _get_instance(self, data: Dict[str, Any], history: Optional[models.QuerySet] = None, **kwargs) -> models.Model:
         with suppress(ObjectDoesNotExist):
-            return self.model.objects.filter(instrument=data["instrument"], date=data["date"], type=data["type"])
+            return self.model.objects.get(instrument=data["instrument"], date=data["date"], type=data["type"])
 
 
 class OptionImportHandler(ImportExportHandler):
@@ -46,7 +46,7 @@ class OptionImportHandler(ImportExportHandler):
 
     def _get_instance(self, data: Dict[str, Any], history: Optional[models.QuerySet] = None, **kwargs) -> models.Model:
         with suppress(ObjectDoesNotExist):
-            return self.model.objects.filter(
+            return self.model.objects.get(
                 instrument=data["instrument"],
                 contract_identifier=data["contract_identifier"],
                 date=data["date"],

wbfdm/import_export/parsers/cbinsights/equities.py

@@ -1,4 +1,5 @@
 import json
+from contextlib import suppress
 
 from wbcore.contrib.geography.models import Geography
 
@@ -44,14 +45,12 @@ def parse(import_source):
                         d["headquarter_address"] = f"{street}, {postal_code} {city_name}"
 
             if sector_id := summary.get("sectorId", None):
-                try:
+                with suppress(Exception):
                     code_aggregated = f"{int(sector_id):03}"
                     if industry_id := summary.get("industryId", None):
                         code_aggregated += f"{int(industry_id):03}"
                         if subindustry_id := summary.get("subindustryId", None):
                             code_aggregated += f"{int(subindustry_id):03}"
-                except Exception:
-                    pass
                 d["classifications"] = [{"code_aggregated": code_aggregated, "group": cbinsight_group.id}]
         data.append(d)
     return {"data": data}

wbfdm/jinja2.py

@@ -3,5 +3,6 @@ from jinjasql import JinjaSql  # type: ignore
 
 
 def get_environment(**options):
-    env = Environment(**options)
+    # we except only SQL template, so we need to not escape special characters (possible XSS attack for HTML template)
+    env = Environment(**options)  # noqa: S701
     return JinjaSql(env=env, param_style="format").env

wbfdm/locale/de/LC_MESSAGES/django.po

@@ -0,0 +1,257 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2025-05-30 11:37+0200\n"
+"PO-Revision-Date: 2025-05-30 09:40+0000\n"
+"Language-Team: German (https://app.transifex.com/stainly/teams/171242/de/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: de\n"
+"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+
+#: contrib/metric/filters.py:27
+msgid "Basket Content Type"
+msgstr ""
+
+#: contrib/metric/viewsets/configs/utils.py:127
+msgid "Fundamentals"
+msgstr ""
+
+#: models/exchanges/exchanges.py:184
+#: viewsets/configs/display/instruments.py:77
+msgid "Exchange"
+msgstr ""
+
+#: models/exchanges/exchanges.py:185
+msgid "Exchanges"
+msgstr ""
+
+#: models/instruments/instrument_requests.py:116
+#: models/instruments/instrument_requests.py:182
+msgid "An instrument already exists with the proposed identifier"
+msgstr ""
+
+#: viewsets/configs/buttons/classifications.py:10
+msgid "Tree Chart"
+msgstr ""
+
+#: viewsets/configs/buttons/classifications.py:11
+msgid "Icicle Chart"
+msgstr ""
+
+#: viewsets/configs/display/classifications.py:40
+msgid "Child Classifications"
+msgstr ""
+
+#: viewsets/configs/display/classifications.py:43
+msgid "Instruments"
+msgstr ""
+
+#: viewsets/configs/display/classifications.py:69
+msgid "Classification"
+msgstr ""
+
+#: viewsets/configs/display/classifications.py:112
+msgid "Related Instruments"
+msgstr ""
+
+#: viewsets/configs/display/esg.py:14
+msgid "Initiated"
+msgstr ""
+
+#: viewsets/configs/display/esg.py:15
+msgid "Review"
+msgstr ""
+
+#: viewsets/configs/display/esg.py:16
+msgid "Headline"
+msgstr ""
+
+#: viewsets/configs/display/esg.py:17
+msgid "Narrative"
+msgstr ""
+
+#: viewsets/configs/display/esg.py:18
+msgid "Source"
+msgstr ""
+
+#: viewsets/configs/display/esg.py:19
+msgid "Status"
+msgstr ""
+
+#: viewsets/configs/display/esg.py:20
+msgid "Type"
+msgstr ""
+
+#: viewsets/configs/display/esg.py:21
+msgid "Assessment"
+msgstr ""
+
+#: viewsets/configs/display/esg.py:22
+msgid "Response"
+msgstr ""
+
+#: viewsets/configs/display/esg.py:70
+msgid "Section"
+msgstr ""
+
+#: viewsets/configs/display/esg.py:71
+msgid "Adverse sustainability indicator"
+msgstr ""
+
+#: viewsets/configs/display/esg.py:72
+msgid "Metric"
+msgstr ""
+
+#: viewsets/configs/display/esg.py:73
+msgid "Factor"
+msgstr ""
+
+#: viewsets/configs/display/esg.py:74
+msgid "Value"
+msgstr ""
+
+#: viewsets/configs/display/instrument_lists.py:34
+msgid "Exclusion List"
+msgstr ""
+
+#: viewsets/configs/display/instrument_lists.py:105
+msgid "Validated"
+msgstr ""
+
+#: viewsets/configs/display/instrument_lists.py:109
+msgid "Not Validated"
+msgstr ""
+
+#: viewsets/configs/display/instrument_requests.py:95
+msgid "Instrument Data"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:70
+#: viewsets/configs/display/officers.py:11
+msgid "Name"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:73
+msgid "Information"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:76
+msgid "Instrument Type"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:78
+msgid "ISIN"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:79
+msgid "Ticker"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:80
+msgid "RIC"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:81
+msgid "Refinitiv Mnemonic"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:82
+msgid "Description"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:83
+msgid "Currency"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:84
+msgid "Country"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:91
+msgid "Extra"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:94
+msgid "Primary"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:95
+msgid "Investable Universe"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:96
+msgid "Security"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:97
+msgid "Internally Managed"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:100
+msgid "Performance"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:330
+msgid "Hierarchy"
+msgstr ""
+
+#: viewsets/configs/display/instruments.py:343
+msgid "News"
+msgstr ""
+
+#: viewsets/configs/display/officers.py:10
+msgid "Position"
+msgstr ""
+
+#: viewsets/configs/display/officers.py:12
+msgid "Age"
+msgstr ""
+
+#: viewsets/configs/display/officers.py:13
+msgid "Sex"
+msgstr ""
+
+#: viewsets/configs/display/officers.py:14
+msgid "Start"
+msgstr ""
+
+#: viewsets/configs/display/prices.py:12
+msgid "Date"
+msgstr ""
+
+#: viewsets/configs/display/prices.py:13
+msgid "Open"
+msgstr ""
+
+#: viewsets/configs/display/prices.py:14
+msgid "High"
+msgstr ""
+
+#: viewsets/configs/display/prices.py:15
+msgid "Low"
+msgstr ""
+
+#: viewsets/configs/display/prices.py:16
+msgid "Close"
+msgstr ""
+
+#: viewsets/configs/display/prices.py:17
+msgid "Volume"
+msgstr ""
+
+#: viewsets/configs/display/prices.py:18
+msgid "Oustanding Shares"
+msgstr ""
+
+#: viewsets/configs/display/prices.py:19
+msgid "Market Cap."
+msgstr ""