wbfdm 1.43.1__py2.py3-none-any.whl

wbfdm/dataloaders/cache.py

@@ -0,0 +1,129 @@
+import itertools
+from typing import Any, Generic, Iterable, TypedDict, TypeVar, cast
+
+from django.core.cache import cache
+
+T = TypeVar("T", bound=TypedDict)
+
+
+class Cache(Generic[T]):
+    """
+    A Cache Class to handle 3-Dimensional data (identifier-key-value)
+    """
+
+    def __init__(
+        self,
+        identifier_key: str = "instrument_id",
+        symbol_key: str = "factor_code",
+        value_key: str = "value",
+        timeout: int | None = None,
+    ):
+        """
+        Constructor of the Cache Class
+        Args:
+            identifier_key: The lookup identifier field. Default to `instrument_id`
+            symbol_key: The symbol lookup field. Default to `factor_code`
+            value_key: The value lookup field. Default to `value`
+            timeout: The cache timeout configuration value. Default to None (never expired)
+        """
+        self.identifier_key = identifier_key
+        self.symbol_key = symbol_key
+        self.value_key = value_key
+        self.timeout = timeout
+        self.id_symbol_pairs: list[tuple[str, str]] = []
+        self.missing_keys = []
+        self.missing_ids = set()
+        self.missing_symbols = set()
+
+    def _get_cache_key(self, id: str, symbol: str) -> str:
+        """
+        Generate the key used in the caching layer based on the given ID and symbol.
+
+        Args:
+            id (str): The ID to be used in the cache key.
+            symbol (str): The symbol to be used in the cache key.
+
+        Returns:
+            str: A cache key in the format "id_symbol" where both the ID and symbol
+            are in lowercase.
+        """
+        return f"{str(id).lower()}_{symbol.lower()}"
+
+    def _deserialize_cache(self, id: str, symbol: str, value: Any) -> T:
+        """
+        Data retreived from cache is deserialized using this method. We expect a key value format which is converted into a dictionary with keys as identifier, symbol and value
+
+        Args:
+            id: identifier
+            symbol: symbol
+            value: value
+
+        Returns:
+            a dictionary with keys as identifier, symbol and value
+        """
+        res = {self.identifier_key: int(id), self.symbol_key: symbol, self.value_key: value}
+        return cast(T, res)
+
+    def initialize(self, ids: list[int], symbols: list[str]):
+        """
+        Initialize the instance with a list of IDs and symbols.
+
+        This method takes a list of identifiers and a list of symbols and creates a list of
+        tuples containing all possible pairs of identifiers (converted to strings) and symbols.
+        It also initializes sets for symbol and identifiers not present in cache
+
+        Args:
+            ids (list[int]): A list of integer IDs.
+            symbols (list[str]): A list of symbol strings.
+        """
+        self.id_symbol_pairs = list(map(lambda x: (str(x[0]), x[1]), itertools.product(ids, symbols)))
+        self.missing_ids = set()
+        self.missing_symbols = set()
+        self.missing_keys = list()
+
+    def fetch_from_cache(self) -> Iterable[T]:
+        """
+        Fetch values already stored in the cache.
+
+        This method iterates over all identifiers and symbols pair and return the cached value if it exists or mark this pair as missing
+
+        Yields:
+            T: Deserialized cached values.
+        """
+        sentinel = object()
+        for id, symbol in self.id_symbol_pairs:
+            key = self._get_cache_key(id, symbol)
+            cached_value = cache.get(key, sentinel)
+            if cached_value is not None and cached_value is not sentinel:
+                yield self._deserialize_cache(id, symbol, cached_value)
+            elif (
+                cached_value is sentinel
+            ):  # otherwise, it's literal None and then it means the cache contains "None" for that key
+                self.missing_symbols.add(symbol)
+                self.missing_ids.add(id)
+                self.missing_keys.append(key)
+
+    def write(self, row: T) -> T:
+        """
+        Write given row into the cache
+
+        Args:
+            row: a dictionary typed object
+
+        Returns:
+            a dictionary typed object
+        """
+        if (key := row.get(self.identifier_key)) and (symbol := row.get(self.symbol_key)):
+            value = row.get(self.value_key)
+            key = self._get_cache_key(str(key), str(symbol))
+            cache.set(key, value, timeout=self.timeout)
+            if key in self.missing_keys:  # mark row's key are "handled"
+                self.missing_keys.remove(key)
+        return row
+
+    def close(self):
+        """
+        Close cache stream by fixing missing keys not handled as None value in the cache (it shows that the value was actually fetched but didn't returned any value)
+        """
+        for key in self.missing_keys:
+            cache.set(key, None, timeout=self.timeout)

wbfdm/dataloaders/protocols.py

@@ -0,0 +1,112 @@
+from datetime import date
+from typing import Iterator, Protocol
+
+from wbfdm.dataloaders.types import (
+    AdjustmentDataDict,
+    CorporateActionDataDict,
+    ESGControversyDataDict,
+    ESGDataDict,
+    FinancialDataDict,
+    MarketDataDict,
+    OfficerDataDict,
+    ReportDateDataDict,
+    StatementDataDict,
+)
+from wbfdm.enums import (
+    ESG,
+    CalendarType,
+    DataType,
+    EstimateType,
+    Financial,
+    Frequency,
+    MarketData,
+    PeriodType,
+    SeriesType,
+    StatementType,
+)
+
+
+class ReportDateProtocol(Protocol):
+    def reporting_dates(self, only_next: bool = True) -> Iterator[ReportDateDataDict]:
+        ...
+
+
+class AdjustmentsProtocol(Protocol):
+    def adjustments(self, from_date: date, to_date: date) -> Iterator[AdjustmentDataDict]:
+        ...
+
+
+class MarketDataProtocol(Protocol):
+    def market_data(
+        self,
+        values: list[MarketData] | None = [MarketData.CLOSE],
+        from_date: date | None = None,
+        to_date: date | None = None,
+        exact_date: date | None = None,
+        frequency: Frequency = Frequency.DAILY,
+        target_currency: str | None = None,
+    ) -> Iterator[MarketDataDict]:
+        ...
+
+
+class CorporateActionsProtocol(Protocol):
+    def corporate_actions(
+        self, from_date: date | None = None, to_date: date | None = None
+    ) -> Iterator[CorporateActionDataDict]:
+        ...
+
+
+class OfficersProtocol(Protocol):
+    def officers(self) -> Iterator[OfficerDataDict]:
+        ...
+
+
+class StatementsProtocol(Protocol):
+    def statements(
+        self,
+        statement_type: StatementType | None = None,
+        from_date: date | None = None,
+        to_date: date | None = None,
+        from_year: int | None = None,
+        to_year: int | None = None,
+        period_type: PeriodType = PeriodType.ALL,
+        data_type: DataType = DataType.STANDARDIZED,
+        financials: list[Financial] | None = None,
+        target_currency: str | None = None,
+    ) -> Iterator[StatementDataDict]:
+        ...
+
+
+class FinancialsProtocol(Protocol):
+    def financials(
+        self,
+        values: list[Financial],
+        from_date: date | None = None,
+        to_date: date | None = None,
+        from_year: int | None = None,
+        to_year: int | None = None,
+        from_index: int | None = None,
+        to_index: int | None = None,
+        from_valid: date | None = None,
+        to_valid: date | None = None,
+        period_type: PeriodType = PeriodType.ANNUAL,
+        calendar_type: CalendarType = CalendarType.FISCAL,
+        series_type: SeriesType = SeriesType.COMPLETE,
+        data_type: DataType = DataType.STANDARDIZED,
+        estimate_type: EstimateType = EstimateType.VALID,
+        target_currency: str | None = None,
+    ) -> Iterator[FinancialDataDict]:
+        ...
+
+
+class ESGControversyProtocol(Protocol):
+    def esg_controversies(self) -> Iterator[ESGControversyDataDict]:
+        ...
+
+
+class ESGProtocol(Protocol):
+    def esg(
+        self,
+        values: list[ESG],
+    ) -> Iterator[ESGDataDict]:
+        ...

wbfdm/dataloaders/proxies.py

@@ -0,0 +1,201 @@
+from datetime import date
+from typing import Iterator
+
+from wbcore.contrib.dataloader.dataloaders import DataloaderProxy
+from wbfdm.dataloaders.protocols import (
+    AdjustmentsProtocol,
+    CorporateActionsProtocol,
+    ESGControversyProtocol,
+    ESGProtocol,
+    FinancialsProtocol,
+    MarketDataProtocol,
+    OfficersProtocol,
+    ReportDateProtocol,
+    StatementsProtocol,
+)
+from wbfdm.dataloaders.types import (
+    AdjustmentDataDict,
+    CorporateActionDataDict,
+    ESGControversyDataDict,
+    ESGDataDict,
+    FinancialDataDict,
+    MarketDataDict,
+    OfficerDataDict,
+    ReportDateDataDict,
+    StatementDataDict,
+)
+from wbfdm.enums import (
+    ESG,
+    CalendarType,
+    DataType,
+    EstimateType,
+    Financial,
+    Frequency,
+    MarketData,
+    PeriodType,
+    SeriesType,
+    StatementType,
+)
+
+from .cache import Cache
+
+
+def _market_data_row_parser(row):
+    if row.get("close") is None and (bid := row.get("bid")) is not None and (ask := row.get("ask")) is not None:
+        price = (bid + ask) / 2
+        row["close"] = price
+        row["open"] = price
+        row["low"] = price
+        row["high"] = price
+    return row
+
+
+class InstrumentDataloaderProxy(
+    DataloaderProxy[
+        AdjustmentsProtocol
+        | MarketDataProtocol
+        | CorporateActionsProtocol
+        | OfficersProtocol
+        | StatementsProtocol
+        | FinancialsProtocol
+        | ReportDateProtocol
+        | ESGControversyProtocol
+        | ESGProtocol
+    ]
+):
+    def reporting_dates(self, only_next: bool = True) -> Iterator[ReportDateDataDict]:
+        for dl in self.iterate_dataloaders("reporting_dates"):
+            yield from dl.reporting_dates(only_next=only_next)
+
+    def adjustments(self, from_date: date, to_date: date) -> Iterator[AdjustmentDataDict]:
+        for dl in self.iterate_dataloaders("adjustments"):
+            yield from dl.adjustments(from_date=from_date, to_date=to_date)
+
+    def market_data(
+        self,
+        values: list[MarketData] | None = None,
+        from_date: date | None = None,
+        to_date: date | None = None,
+        exact_date: date | None = None,
+        frequency: Frequency = Frequency.DAILY,
+        target_currency: str | None = None,
+    ) -> Iterator[MarketDataDict]:
+        if not values:
+            values = list(MarketData)
+        for dl in self.iterate_dataloaders("market_data"):
+            yield from map(
+                lambda row: _market_data_row_parser(row),
+                dl.market_data(
+                    values=values,
+                    from_date=from_date,
+                    to_date=to_date,
+                    exact_date=exact_date,
+                    frequency=frequency,
+                    target_currency=target_currency,
+                ),
+            )
+
+    def corporate_actions(
+        self, from_date: date | None = None, to_date: date | None = None
+    ) -> Iterator[CorporateActionDataDict]:
+        for dl in self.iterate_dataloaders("corporate_actions"):
+            yield from dl.corporate_actions(from_date=from_date, to_date=to_date)
+
+    def officers(self) -> Iterator[OfficerDataDict]:
+        for dl in self.iterate_dataloaders("officers"):
+            yield from dl.officers()
+
+    def statements(
+        self,
+        statement_type: StatementType | None = None,
+        from_date: date | None = None,
+        to_date: date | None = None,
+        from_year: int | None = None,
+        to_year: int | None = None,
+        period_type: PeriodType = PeriodType.ALL,
+        data_type: DataType = DataType.STANDARDIZED,
+        financials: list[Financial] | None = None,
+        target_currency: str | None = None,
+    ) -> Iterator[StatementDataDict]:
+        for dl in self.iterate_dataloaders("statements"):
+            yield from dl.statements(
+                statement_type=statement_type,
+                from_date=from_date,
+                to_date=to_date,
+                from_year=from_year,
+                to_year=to_year,
+                period_type=period_type,
+                data_type=data_type,
+                financials=financials,
+                target_currency=target_currency,
+            )
+
+    def financials(
+        self,
+        values: list[Financial],
+        from_date: date | None = None,
+        to_date: date | None = None,
+        from_year: int | None = None,
+        to_year: int | None = None,
+        from_index: int | None = None,
+        to_index: int | None = None,
+        from_valid: date | None = None,
+        to_valid: date | None = None,
+        period_type: PeriodType = PeriodType.ANNUAL,
+        calendar_type: CalendarType = CalendarType.FISCAL,
+        series_type: SeriesType = SeriesType.COMPLETE,
+        data_type: DataType = DataType.STANDARDIZED,
+        estimate_type: EstimateType = EstimateType.VALID,
+        target_currency: str | None = None,
+    ) -> Iterator[FinancialDataDict]:
+        for dl in self.iterate_dataloaders("financials"):
+            yield from dl.financials(
+                values=values,
+                from_date=from_date,
+                to_date=to_date,
+                from_year=from_year,
+                to_year=to_year,
+                from_index=from_index,
+                to_index=to_index,
+                from_valid=from_valid,
+                to_valid=to_valid,
+                period_type=period_type,
+                calendar_type=calendar_type,
+                series_type=series_type,
+                data_type=data_type,
+                estimate_type=estimate_type,
+                target_currency=target_currency,
+            )
+
+    def esg_controversies(self) -> Iterator[ESGControversyDataDict]:
+        for dl in self.iterate_dataloaders("esg_controversies"):
+            yield from dl.esg_controversies()
+
+    def esg(
+        self,
+        values: list[ESG],
+    ) -> Iterator[ESGDataDict]:
+        for dl in self.iterate_dataloaders("esg"):
+            if (
+                (cache_identifier_key := getattr(dl, "CACHE_IDENTIFIER_KEY", None))
+                and (cache_symbol_key := getattr(dl, "CACHE_SYMBOL_KEY", None))
+                and (cache_value_key := getattr(dl, "CACHE_VALUE_KEY", None))
+            ):
+                cache = Cache(
+                    identifier_key=cache_identifier_key,
+                    symbol_key=cache_symbol_key,
+                    value_key=cache_value_key,
+                    timeout=getattr(dl, "CACHE_TIMEOUT", 10 * 24 * 3600),  # default to 10days
+                )
+                cache.initialize(dl.entity_ids, [v.value for v in values])
+
+                yield from cache.fetch_from_cache()
+                if len(cache.missing_symbols) > 0 and len(cache.missing_ids) > 0:
+                    dl.entities = dl.entities.filter(id__in=cache.missing_ids)
+                    yield from map(
+                        lambda row: cache.write(row),
+                        dl.esg(values=[v for v in values if v.value in cache.missing_symbols]),
+                    )
+                cache.close()
+            else:
+                yield from dl.esg(values=values)

wbfdm/dataloaders/types.py

@@ -0,0 +1,209 @@
+from datetime import date
+from typing import Literal, TypedDict
+
+from typing_extensions import NotRequired
+
+
+class BaseDict(TypedDict):
+    """
+    Represents a base dictionary with common fields.
+
+    Attributes:
+        id: str | int
+            The unique identifier.
+        external_id: str | int
+            The external identifier.
+        source: str
+            The source of the data.
+        currency: str
+            The currency used.
+    """
+
+    id: str | int
+    instrument_id: int
+    external_id: str | int
+    source: str
+    currency: NotRequired[str]
+
+
+class MarketDataDict(BaseDict):
+    """
+    Represents a dictionary for daily valuation data.
+
+    Attributes:
+        valuation_date: date
+            The date of valuation.
+        open: float | None
+            The opening value (if available).
+        close: float | None
+            The closing value (if available).
+        high: float | None
+            The highest value (if available).
+        low: float | None
+            The lowest value (if available).
+        bid: float | None
+            The bid value (if available).
+        ask: float | None
+            The ask value (if available).
+        volume: float | None
+            The volume (if available).
+        market_cap: float | None
+            The market capitalization (if available).
+    """
+
+    valuation_date: date
+
+    open: NotRequired[float]
+    close: NotRequired[float]
+    high: NotRequired[float]
+    low: NotRequired[float]
+    bid: NotRequired[float]
+    ask: NotRequired[float]
+    volume: NotRequired[float]
+    market_cap: NotRequired[float]
+    calculated: NotRequired[bool]
+
+
+class CorporateActionDataDict(BaseDict):
+    old_shares: float
+    new_shares: float
+
+    action_code: str
+    event_code: str
+    valuation_date: date
+
+
+class AdjustmentDataDict(BaseDict):
+    adjustment_factor: float
+    cumulative_adjustment_factor: float
+
+    adjustment_date: date
+    adjustment_end_date: date
+
+
+class FinancialDataDict(BaseDict):
+    """
+    Represents a dictionary for financial data.
+
+    Attributes:
+        period_end_date: date
+            The period end date / The reporting date
+        valid_until: date | None
+            The validity date (if available).
+        fiscal_year: int
+            The fiscal year.
+        interim: int
+            The interim period. 0 means yearly data.
+        estimate: bool
+            Indicates if the data is an estimate or actual data
+        reported: bool
+            Indicates if the data is reported or standardized
+        value: str | float | int
+            The value.
+        financial: str
+            The type of financial data.
+    """
+
+    valid_until: NotRequired[date]
+    year: int
+    interim: int
+    estimate: NotRequired[bool]
+    # reported: bool
+
+    value: str | float | int
+    financial: str
+
+
+class StatementDataDict(BaseDict):
+    """
+    Represents a dictionary for statement data
+
+    Attributes:
+        external_code: str
+            The code that is used by the data vendor to identify this type of financial
+        external_ordering: int
+            The ordering that is supplied by the data vendor to sort this financial
+        external_description: str
+            The description supplied by the data vendor to describe this financial
+        period_end_date: date
+            The period end date / The reporting date
+        year: int
+            The financial year of the statement.
+        interim: int
+            The interim period. 0 means yearly data.
+        reported: bool
+            Indicates if the data is reported or standardized.
+        value: float | int
+            The value of the datapoint of the statement.
+
+    """
+
+    external_code: str
+    external_ordering: int
+    external_description: str
+
+    period_end_date: date
+    year: int
+    interim: int
+    value: float | int
+    reported: bool
+
+
+class ReportDateDataDict(TypedDict):
+    instrument_id: int
+    external_id: int
+    source: str
+    period_end_date: date
+    is_interim: bool
+    start_date: date
+    end_date: date
+    market_phase: Literal["before_market"] | Literal["after_market"] | None
+    status: Literal["confirmed"] | Literal["tentative"]
+
+
+class OfficerDataDict(TypedDict):
+    """
+    Represents a dictionary for officers
+
+    Attributes:
+        instrument_id: int
+            An identifier to uniquely identify the instrument linked to an officer
+        position: str
+            The title of the position
+        name: str
+            The name of the officer
+        age: int
+            The current age of the officer
+        sex: "M" | "F"
+            Indicates the sex of the officer
+        start: date
+            The date when the officer started the position
+    """
+
+    instrument_id: int
+    position: str
+    name: str
+    age: int
+    sex: Literal["M"] | Literal["F"]
+    start: date
+
+
+class ESGDataDict(TypedDict):
+    instrument_id: int
+    factor_code: str
+    value: str | float | int | None
+
+
+class ESGControversyDataDict(TypedDict):
+    id: str | int
+    instrument_id: str
+    headline: str
+    narrative: str
+    source: str
+    response: str
+    status: str | None  # TODO: Move to enum?
+    type: str | None  # TODO: Move to enum?
+    assessment: str | None  # TODO: Move to enum?
+    review: date | None
+    initiated: date | None
+    flag: str | None  # TODO: Move to enum?

wbfdm/dynamic_preferences_registry.py

@@ -0,0 +1,45 @@
+from datetime import date
+
+from dynamic_preferences.preferences import Section
+from dynamic_preferences.registries import global_preferences_registry
+from dynamic_preferences.types import DatePreference, IntPreference, StringPreference
+
+fdm = Section("wbfdm")
+
+
+@global_preferences_registry.register
+class DefaultClassificationGroup(IntPreference):
+    section = fdm
+    name = "default_classification_group"
+    default = 0
+
+    verbose_name = "Default Classification Group"
+
+
+@global_preferences_registry.register
+class DefaultStartDateHistoricalImport(DatePreference):
+    section = fdm
+    name = "default_start_date_historical_import"
+    default = date(2015, 1, 1)
+
+    verbose_name = "Default Start Date"
+    help_text = "Default start date in historical import"
+
+
+@global_preferences_registry.register
+class NonTickerWords(StringPreference):
+    section = fdm
+    name = "non_ticker_words"
+    default = ""
+
+    verbose_name = "Non Ticker Words"
+    help_text = "Comma Separated list of non-ticker words"
+
+
+class FinancialSummarySectionName(StringPreference):
+    section = fdm
+    name = "financial_summary_section_name"
+    default = "Financial Summary"
+
+    verbose_name = "Financial Summary Section Name"
+    help_text = "This name set the tab section name shown from the instance view"