wbfdm 2.2.1__py2.py3-none-any.whl

wbfdm/contrib/qa/dataloaders/reporting_dates.py

@@ -0,0 +1,67 @@
+from typing import Iterator
+
+from django.db import connections
+from jinjasql import JinjaSql
+from wbcore.contrib.dataloader.dataloaders import Dataloader
+from wbcore.contrib.dataloader.utils import dictfetchall
+from wbfdm.dataloaders.protocols import ReportDateProtocol
+from wbfdm.dataloaders.types import ReportDateDataDict
+
+
+class IbesReportingDateDataloader(ReportDateProtocol, Dataloader):
+    def reporting_dates(self, only_next: bool = True) -> Iterator[ReportDateDataDict]:
+        lookup = {k: v for k, v in self.entities.values_list("dl_parameters__reporting_dates__parameters", "id")}
+
+        sql = """
+            with next_events as (
+                select
+                    *,
+                    row_number() over (partition by EstPermID, PerType order by PerEndDate) as rn
+                from TREExpectedRptDate
+                where StartDate > getdate()
+            )
+
+            select
+                rp.EstPermID as external_id,
+                'qa-ibes' as source,
+                convert(date, rp.PerEndDate) as per_end_date,
+                convert(date, rp.StartDate) as start_date,
+                convert(date, rp.EndDate) as end_date,
+                iif(rp.PerType=4, 0, 1) as interim,
+                case
+                    when rp.MarketPhase = 'AMC' then 'after_market'
+                    when rp.MarketPhase = 'BMO' then 'before_market'
+                    else null
+                end as market_phase,
+                lower(rp.Status) as status
+            {% if only_next %}
+                from next_events as rp
+            {% else %}
+                from TREExpectedRptDate as rp
+            {% endif %}
+            where rp.EstPermID in (
+            {% for instrument in instruments %}
+                {{instrument}} {% if not loop.last %}, {% endif %}
+            {% endfor %}
+            )
+            {% if only_next %}
+                AND rp.rn = 1
+            {% endif %}
+        """
+
+        query, bind_params = JinjaSql(param_style="format").prepare_query(
+            sql,
+            {
+                "instruments": lookup.keys(),
+                "only_next": only_next,
+            },
+        )
+        with connections["qa"].cursor() as cursor:
+            cursor.execute(
+                query,
+                bind_params,
+            )
+            for row in dictfetchall(cursor):
+                row["instrument_id"] = lookup[row["external_id"]]
+                row["interim"] = bool(row["interim"])
+                yield row

wbfdm/contrib/qa/dataloaders/statements.py

@@ -0,0 +1,267 @@
+from datetime import date
+from enum import Enum
+from typing import Iterator
+
+from django.db import connections
+from jinjasql import JinjaSql  # type: ignore
+from wbcore.contrib.dataloader.dataloaders import Dataloader
+from wbcore.contrib.dataloader.utils import dictfetchall
+from wbfdm.dataloaders.protocols import StatementsProtocol
+from wbfdm.dataloaders.types import StatementDataDict
+from wbfdm.enums import DataType, Financial, PeriodType, StatementType
+
+
+class RKDStatementType(Enum):
+    INCOME_STATEMENT = 1
+    CASHFLOW_STATEMENT = 2
+    BALANCE_SHEET = 3
+
+
+class RKDFinancial(Enum):
+    EMPLOYEES = "METL"
+    CASH_AND_SHORT_TERM_INVESTMENT = "SCSI"
+    DILUTED_WEIGHTED_AVG_SHARES = "SDWS"
+    TOTAL_DEBT = "STLD"
+    NET_DEBT = "SNTD"
+    STOCK_COMPENSATION = "VSCP"
+    TANGIBLE_BOOK_VALUE_PER_SHARE = "STBP"
+    REVENUE = "RTLR"
+    SHARES_OUTSTANDING = "QTCO"
+    EPS = "SDAI"
+    CASH_FLOW_FROM_OPERATIONS = "OTLO"
+    CAPEX = "SCEX"
+
+
+reported_sql = """
+    SELECT
+        CONCAT(val.Code, '-', CONVERT(DATE, val.PerEndDt), '-', fil.InterimNo, '-', val.LineID) as id,
+        val.LineID as external_ordering,
+        itemsub.Desc_ as external_description,
+        itemsub.StdCOA as external_code,
+        val.Code as external_identifier,
+        CONVERT(DATE, val.PerEndDt) as period_end_date,
+        per.Fyr as year,
+        fil.InterimNo as interim,
+        CASE
+            when fil.PerTypeCode = 1 THEN 'Y'
+            when fil.PerTypeCode = 2 THEN 'Q'
+            when fil.PerTypeCode = 3 THEN 'S'
+            when fil.PerTypeCode = 4 THEN 'T'
+            when fil.PerTypeCode = 5 THEN 'Q'
+            when fil.PerTypeCode = 6 THEN 'Y'
+        END as period_type,
+        CASE
+            WHEN fil.UnitsConvToCode = 'T' AND item.ItemPrecision in (1,2) THEN val.Value_ * 1e3
+            WHEN fil.UnitsConvToCode = 'M' AND item.ItemPrecision in (1,2) THEN val.Value_ * 1e6
+            WHEN fil.UnitsConvToCode = 'B' AND item.ItemPrecision in (1,2) THEN val.Value_ * 1e9
+            ELSE val.Value_
+        END as value,
+        'qa-rkd' as source,
+        CASE
+            WHEN item.IsCurrency = 1 THEN code.Desc_
+            ELSE NULL
+        END AS currency
+
+    FROM RKDFndCSFFinVal AS val
+
+    JOIN RKDFndCSFStmt AS stmt ON
+        val.Code = stmt.Code
+        AND val.PerEndDt = stmt.PerEndDt
+        AND val.PerTypeCode = stmt.PerTypeCode
+        AND val.StmtDt = stmt.StmtDt
+        AND val.StmtTypeCode = stmt.StmtTypeCode
+        AND stmt.CompStmtCode = 1
+
+    JOIN RKDFndCSFPerFiling AS fil ON
+        stmt.Code = fil.Code
+        AND stmt.PerEndDt = fil.PerEndDt
+        AND stmt.PerTypeCode = fil.PerTypeCode
+        AND stmt.StmtDt = fil.StmtDt
+
+    LEFT JOIN RKDFNDCSFITEMSUB AS itemsub ON
+        val.Code = itemsub.Code
+        AND val.PerTypeCode = itemsub.PerTypeCode
+        AND val.StmtTypeCode = itemsub.StmtTypeCode
+        AND val.LineID = itemsub.LineID
+
+    LEFT JOIN RKDFndCSFItem AS item ON
+        itemsub.Item = item.Item
+
+    LEFT JOIN RKDFndCode AS code ON
+        fil.CurrConvToCode = code.Code
+        AND code.Type_ = 58
+
+    LEFT JOIN RKDFndCsfPeriod AS per ON
+        per.Code = fil.Code
+        AND per.PerEndDt = fil.PerEndDt
+        AND per.PerTypeCode = (
+            CASE
+                WHEN fil.PerTypeCode = 1 THEN 1
+                WHEN fil.PerTypeCode in (2,3,4,5) THEN 5
+            END
+        )
+
+    WHERE
+        val.Code in (
+        {% for instrument in instruments %}
+            {{instrument}} {% if not loop.last %}, {% endif %}
+        {% endfor %})
+        AND val.StmtTypeCode = {{ statement_type }}
+        {% if from_year %}AND per.Fyr >= {{ from_year }} {% endif %}
+        {% if to_year %}AND per.Fyr <= {{ to_year }} {% endif %}
+        {% if from_date %}AND val.PerEndDt >= {{ from_date }} {% endif %}
+        {% if to_date %}AND val.PerEndDt <= {{ to_date }} {% endif %}
+        {% if period_type == 'interim' %}AND fil.InterimNo > 0{% elif period_type == 'annual' %}AND (fil.InterimNo = 0 OR fil.InterimNo IS NULL){% endif %}
+    ORDER BY val.LineID
+"""
+standardized_sql = """
+    SELECT
+        CONCAT(val.Code, '-', CONVERT(DATE, val.PerEndDt), '-', fil.InterimNo, '-', item.LineID) as id,
+        item.LineID as external_ordering,
+        item.Desc_ as external_description,
+        item.COA as external_code,
+        val.Code as external_identifier,
+        CONVERT(DATE, val.PerEndDt) as period_end_date,
+        per.Fyr as year,
+        CASE
+            when fil.PerTypeCode = 1 THEN 'Y'
+            when fil.PerTypeCode = 2 THEN 'Q'
+            when fil.PerTypeCode = 3 THEN 'S'
+            when fil.PerTypeCode = 4 THEN 'T'
+            when fil.PerTypeCode = 5 THEN 'Q'
+            when fil.PerTypeCode = 6 THEN 'Y'
+        END as period_type,
+        fil.InterimNo as interim,
+        CASE
+            WHEN fil.UnitsConvToCode = 'T' AND item.ItemPrecision in (1,2) THEN val.Value_ * 1e3
+            WHEN fil.UnitsConvToCode = 'M' AND item.ItemPrecision in (1,2) THEN val.Value_ * 1e6
+            WHEN fil.UnitsConvToCode = 'B' AND item.ItemPrecision in (1,2) THEN val.Value_ * 1e9
+            ELSE val.Value_
+        END as value,
+        'qa-rkd' as source,
+        CASE
+            WHEN item.IsCurrency = 1 THEN code.Desc_
+            ELSE NULL
+        END AS currency
+
+    FROM RKDFndStdFinVal AS val
+
+    LEFT JOIN RKDFndStdPeriod AS per ON
+        per.Code = val.Code
+        AND per.PerEndDt = val.PerEndDt
+        AND per.PerTypeCode = (
+            CASE
+                WHEN val.PerTypeCode = 1 THEN 1
+                WHEN val.PerTypeCode in (2,3,4,5) THEN 5
+            END
+        )
+        AND per.PerEnddt = (
+            SELECT TOP 1 per2.PerEndDt
+            FROM RKDFndStdPeriod AS per2
+            WHERE per.Code = per2.Code
+              AND per.PerTypeCode = per2.PerTypeCode
+              AND per.Fyr = per2.Fyr
+              AND (
+                per2.InterimNo = per.InterimNo
+                    OR (
+                    per2.InterimNo IS NULL AND per.InterimNo IS NULL
+                    )
+                )
+            ORDER BY per2.PerEndDt DESC
+        )
+
+    JOIN RKDFndStdStmt AS stmt ON
+        val.Code = stmt.Code
+        AND val.PerEndDt = stmt.PerEndDt
+        AND val.PerTypeCode = stmt.PerTypeCode
+        AND val.StmtDt = stmt.StmtDt
+        AND val.StmtTypeCode = stmt.StmtTypeCode
+        AND stmt.CompStmtCode = 1
+        AND stmt.StmtDt = (
+            select top 1 stmt2.StmtDt
+            from RKDFndStdStmt as stmt2
+            where stmt2.Code = stmt.Code
+                and stmt2.PerEndDt = stmt.PerEndDt
+                and stmt2.PerTypeCode = stmt.PerTypeCode
+                and stmt2.StmtTypeCode = stmt.StmtTypeCode
+                and stmt2.CompStmtCode = stmt.CompStmtCode
+            order by stmt2.StmtDt desc
+        )
+
+    JOIN RKDFndStdPerFiling AS fil ON
+        stmt.Code = fil.Code
+        AND stmt.PerEndDt = fil.PerEndDt
+        AND stmt.PerTypeCode = fil.PerTypeCode
+        AND stmt.StmtDt = fil.StmtDt
+
+    LEFT JOIN RKDFndStdItem AS item ON
+        val.Item = item.Item
+
+    LEFT JOIN RKDFndCode AS code ON
+        fil.CurrConvToCode = code.Code
+        AND code.Type_ = 58
+
+    WHERE
+        val.Code in (
+        {% for instrument in instruments %}
+            {{instrument}} {% if not loop.last %}, {% endif %}
+        {% endfor %})
+        {% if statement_type %} AND val.StmtTypeCode = {{ statement_type }}{% endif %}
+        {% if from_year %}AND per.Fyr >= {{ from_year }} {% endif %}
+        {% if to_year %}AND per.Fyr <= {{ to_year }} {% endif %}
+        {% if from_date %}AND val.PerEndDt >= {{ from_date }} {% endif %}
+        {% if to_date %}AND val.PerEndDt <= {{ to_date }} {% endif %}
+        {% if period_type == 'interim' %}AND fil.InterimNo > 0{% elif period_type == 'annual' %}AND (fil.InterimNo = 0 OR fil.InterimNo IS NULL){% endif %}
+        {% if external_codes %}
+            AND item.COA in (
+                {% for external_code in external_codes %}
+                    {{ external_code }} {% if not loop.last %}, {% endif %}
+                {% endfor %}
+            )
+        {% endif %}
+    ORDER BY item.LineID
+"""
+
+
+class RKDStatementsDataloader(StatementsProtocol, Dataloader):
+    def statements(
+        self,
+        statement_type: StatementType | None = None,
+        from_date: date | None = None,
+        to_date: date | None = None,
+        from_year: int | None = None,
+        to_year: int | None = None,
+        period_type: PeriodType = PeriodType.ALL,
+        data_type: DataType = DataType.STANDARDIZED,
+        financials: list[Financial] | None = None,
+        target_currency: str | None = None,
+    ) -> Iterator[StatementDataDict]:
+        lookup = {k: v for k, v in self.entities.values_list("dl_parameters__statements__parameters", "id")}
+        sql = reported_sql if data_type is DataType.REPORTED else standardized_sql
+        query, bind_params = JinjaSql(param_style="format").prepare_query(
+            sql,
+            {
+                "instruments": lookup.keys(),
+                "statement_type": RKDStatementType[statement_type.name].value if statement_type else None,
+                "from_year": from_year,
+                "to_year": to_year,
+                "from_date": from_date,
+                "to_date": to_date,
+                "period_type": period_type.value,
+                "external_codes": [RKDFinancial[fin.name].value for fin in financials or []],
+            },
+        )
+        with connections["qa"].cursor() as cursor:
+            cursor.execute(
+                query,
+                bind_params,
+            )
+            for row in dictfetchall(cursor):
+                if row["interim"] is None:
+                    row["interim"] = 0
+                # sometime we get None for the year. We default to the period end date year then
+                row["year"] = int(row["year"] or row["period_end_date"].year)
+                row["instrument_id"] = lookup[row["external_identifier"]]
+                if financials:
+                    row["financial"] = Financial[RKDFinancial(row["external_code"]).name].value
+                yield row

wbfdm/dataloaders/cache.py

@@ -0,0 +1,129 @@
+import itertools
+from typing import Any, Generic, Iterable, TypedDict, TypeVar, cast
+
+from django.core.cache import cache
+
+T = TypeVar("T", bound=TypedDict)
+
+
+class Cache(Generic[T]):
+    """
+    A Cache Class to handle 3-Dimensional data (identifier-key-value)
+    """
+
+    def __init__(
+        self,
+        identifier_key: str = "instrument_id",
+        symbol_key: str = "factor_code",
+        value_key: str = "value",
+        timeout: int | None = None,
+    ):
+        """
+        Constructor of the Cache Class
+        Args:
+            identifier_key: The lookup identifier field. Default to `instrument_id`
+            symbol_key: The symbol lookup field. Default to `factor_code`
+            value_key: The value lookup field. Default to `value`
+            timeout: The cache timeout configuration value. Default to None (never expired)
+        """
+        self.identifier_key = identifier_key
+        self.symbol_key = symbol_key
+        self.value_key = value_key
+        self.timeout = timeout
+        self.id_symbol_pairs: list[tuple[str, str]] = []
+        self.missing_keys = []
+        self.missing_ids = set()
+        self.missing_symbols = set()
+
+    def _get_cache_key(self, id: str, symbol: str) -> str:
+        """
+        Generate the key used in the caching layer based on the given ID and symbol.
+
+        Args:
+            id (str): The ID to be used in the cache key.
+            symbol (str): The symbol to be used in the cache key.
+
+        Returns:
+            str: A cache key in the format "id_symbol" where both the ID and symbol
+            are in lowercase.
+        """
+        return f"{str(id).lower()}_{symbol.lower()}"
+
+    def _deserialize_cache(self, id: str, symbol: str, value: Any) -> T:
+        """
+        Data retreived from cache is deserialized using this method. We expect a key value format which is converted into a dictionary with keys as identifier, symbol and value
+
+        Args:
+            id: identifier
+            symbol: symbol
+            value: value
+
+        Returns:
+            a dictionary with keys as identifier, symbol and value
+        """
+        res = {self.identifier_key: int(id), self.symbol_key: symbol, self.value_key: value}
+        return cast(T, res)
+
+    def initialize(self, ids: list[int], symbols: list[str]):
+        """
+        Initialize the instance with a list of IDs and symbols.
+
+        This method takes a list of identifiers and a list of symbols and creates a list of
+        tuples containing all possible pairs of identifiers (converted to strings) and symbols.
+        It also initializes sets for symbol and identifiers not present in cache
+
+        Args:
+            ids (list[int]): A list of integer IDs.
+            symbols (list[str]): A list of symbol strings.
+        """
+        self.id_symbol_pairs = list(map(lambda x: (str(x[0]), x[1]), itertools.product(ids, symbols)))
+        self.missing_ids = set()
+        self.missing_symbols = set()
+        self.missing_keys = list()
+
+    def fetch_from_cache(self) -> Iterable[T]:
+        """
+        Fetch values already stored in the cache.
+
+        This method iterates over all identifiers and symbols pair and return the cached value if it exists or mark this pair as missing
+
+        Yields:
+            T: Deserialized cached values.
+        """
+        sentinel = object()
+        for id, symbol in self.id_symbol_pairs:
+            key = self._get_cache_key(id, symbol)
+            cached_value = cache.get(key, sentinel)
+            if cached_value is not None and cached_value is not sentinel:
+                yield self._deserialize_cache(id, symbol, cached_value)
+            elif (
+                cached_value is sentinel
+            ):  # otherwise, it's literal None and then it means the cache contains "None" for that key
+                self.missing_symbols.add(symbol)
+                self.missing_ids.add(id)
+                self.missing_keys.append(key)
+
+    def write(self, row: T) -> T:
+        """
+        Write given row into the cache
+
+        Args:
+            row: a dictionary typed object
+
+        Returns:
+            a dictionary typed object
+        """
+        if (key := row.get(self.identifier_key)) and (symbol := row.get(self.symbol_key)):
+            value = row.get(self.value_key)
+            key = self._get_cache_key(str(key), str(symbol))
+            cache.set(key, value, timeout=self.timeout)
+            if key in self.missing_keys:  # mark row's key are "handled"
+                self.missing_keys.remove(key)
+        return row
+
+    def close(self):
+        """
+        Close cache stream by fixing missing keys not handled as None value in the cache (it shows that the value was actually fetched but didn't returned any value)
+        """
+        for key in self.missing_keys:
+            cache.set(key, None, timeout=self.timeout)

wbfdm/dataloaders/protocols.py

@@ -0,0 +1,112 @@
+from datetime import date
+from typing import Iterator, Protocol
+
+from wbfdm.dataloaders.types import (
+    AdjustmentDataDict,
+    CorporateActionDataDict,
+    ESGControversyDataDict,
+    ESGDataDict,
+    FinancialDataDict,
+    MarketDataDict,
+    OfficerDataDict,
+    ReportDateDataDict,
+    StatementDataDict,
+)
+from wbfdm.enums import (
+    ESG,
+    CalendarType,
+    DataType,
+    EstimateType,
+    Financial,
+    Frequency,
+    MarketData,
+    PeriodType,
+    SeriesType,
+    StatementType,
+)
+
+
+class ReportDateProtocol(Protocol):
+    def reporting_dates(self, only_next: bool = True) -> Iterator[ReportDateDataDict]:
+        ...
+
+
+class AdjustmentsProtocol(Protocol):
+    def adjustments(self, from_date: date, to_date: date) -> Iterator[AdjustmentDataDict]:
+        ...
+
+
+class MarketDataProtocol(Protocol):
+    def market_data(
+        self,
+        values: list[MarketData] | None = [MarketData.CLOSE],
+        from_date: date | None = None,
+        to_date: date | None = None,
+        exact_date: date | None = None,
+        frequency: Frequency = Frequency.DAILY,
+        target_currency: str | None = None,
+    ) -> Iterator[MarketDataDict]:
+        ...
+
+
+class CorporateActionsProtocol(Protocol):
+    def corporate_actions(
+        self, from_date: date | None = None, to_date: date | None = None
+    ) -> Iterator[CorporateActionDataDict]:
+        ...
+
+
+class OfficersProtocol(Protocol):
+    def officers(self) -> Iterator[OfficerDataDict]:
+        ...
+
+
+class StatementsProtocol(Protocol):
+    def statements(
+        self,
+        statement_type: StatementType | None = None,
+        from_date: date | None = None,
+        to_date: date | None = None,
+        from_year: int | None = None,
+        to_year: int | None = None,
+        period_type: PeriodType = PeriodType.ALL,
+        data_type: DataType = DataType.STANDARDIZED,
+        financials: list[Financial] | None = None,
+        target_currency: str | None = None,
+    ) -> Iterator[StatementDataDict]:
+        ...
+
+
+class FinancialsProtocol(Protocol):
+    def financials(
+        self,
+        values: list[Financial],
+        from_date: date | None = None,
+        to_date: date | None = None,
+        from_year: int | None = None,
+        to_year: int | None = None,
+        from_index: int | None = None,
+        to_index: int | None = None,
+        from_valid: date | None = None,
+        to_valid: date | None = None,
+        period_type: PeriodType = PeriodType.ANNUAL,
+        calendar_type: CalendarType = CalendarType.FISCAL,
+        series_type: SeriesType = SeriesType.COMPLETE,
+        data_type: DataType = DataType.STANDARDIZED,
+        estimate_type: EstimateType = EstimateType.VALID,
+        target_currency: str | None = None,
+    ) -> Iterator[FinancialDataDict]:
+        ...
+
+
+class ESGControversyProtocol(Protocol):
+    def esg_controversies(self) -> Iterator[ESGControversyDataDict]:
+        ...
+
+
+class ESGProtocol(Protocol):
+    def esg(
+        self,
+        values: list[ESG],
+    ) -> Iterator[ESGDataDict]:
+        ...