marketmyna 0.2.0__py3-none-any.whl

marketmyna/data/resilience.py

@@ -0,0 +1,173 @@
+"""Circuit-breaker-protected fallback chains for data fetchers.
+
+Mechanics borrowed from daily_stock_analysis's data_provider circuit
+breaker (closed -> open -> half_open state machine, per-source failure
+count + cooldown), reimplemented here against MarketMyna's own,
+differently-shaped DataFetcher interface rather than copied verbatim.
+
+Key design choice: state is keyed by an arbitrary string, not just the
+source name, so callers can compose "{source}:{method}" keys. A source
+failing on get_quote should not necessarily open the circuit for that
+same source's get_history - the two calls can have very different
+failure modes (e.g. a quote endpoint rate-limited while history is
+fine), so isolating per-method avoids over-tripping the breaker.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from threading import RLock
+from typing import Any, Callable, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class DataSourceUnavailable(Exception):
+    """Every source in a FallbackChain failed or is circuit-open."""
+
+
+class CircuitBreaker:
+    """Per-key closed/open/half_open state machine.
+
+    - CLOSED: normal, calls go through.
+    - OPEN: skip calls for `cooldown_seconds` after `failure_threshold`
+      consecutive failures.
+    - HALF_OPEN: after cooldown, allow `half_open_max_calls` probe(s);
+      success fully recovers, failure re-opens for another cooldown.
+    """
+
+    CLOSED = "closed"
+    OPEN = "open"
+    HALF_OPEN = "half_open"
+
+    def __init__(
+        self,
+        failure_threshold: int = 3,
+        cooldown_seconds: float = 300.0,
+        half_open_max_calls: int = 1,
+    ):
+        self.failure_threshold = failure_threshold
+        self.cooldown_seconds = cooldown_seconds
+        self.half_open_max_calls = half_open_max_calls
+        self._states: dict[str, dict[str, Any]] = {}
+        self._lock = RLock()
+
+    def _state(self, key: str) -> dict[str, Any]:
+        if key not in self._states:
+            self._states[key] = {
+                "state": self.CLOSED,
+                "failures": 0,
+                "last_failure_time": 0.0,
+                "half_open_calls": 0,
+            }
+        return self._states[key]
+
+    def is_available(self, key: str) -> bool:
+        """True if a call should be attempted for this key."""
+        with self._lock:
+            state = self._state(key)
+            now = time.time()
+
+            if state["state"] == self.CLOSED:
+                return True
+
+            if state["state"] == self.OPEN:
+                elapsed = now - state["last_failure_time"]
+                if elapsed < self.cooldown_seconds:
+                    return False
+                state["state"] = self.HALF_OPEN
+                state["half_open_calls"] = 0
+                logger.info("Circuit breaker %s: cooldown elapsed, half-opening", key)
+
+            # HALF_OPEN
+            if state["half_open_calls"] < self.half_open_max_calls:
+                state["half_open_calls"] += 1
+                return True
+            return False
+
+    def record_success(self, key: str) -> None:
+        with self._lock:
+            state = self._state(key)
+            if state["state"] == self.HALF_OPEN:
+                logger.info("Circuit breaker %s: probe succeeded, closing", key)
+            state["state"] = self.CLOSED
+            state["failures"] = 0
+            state["half_open_calls"] = 0
+
+    def record_failure(self, key: str, error: Optional[str] = None) -> None:
+        with self._lock:
+            state = self._state(key)
+            state["failures"] += 1
+            state["last_failure_time"] = time.time()
+
+            if state["state"] == self.HALF_OPEN:
+                state["state"] = self.OPEN
+                state["half_open_calls"] = 0
+                logger.warning("Circuit breaker %s: probe failed, re-opening", key)
+            elif state["failures"] >= self.failure_threshold:
+                state["state"] = self.OPEN
+                logger.warning(
+                    "Circuit breaker %s: %d consecutive failures, opening for %.0fs (%s)",
+                    key, state["failures"], self.cooldown_seconds, error or "",
+                )
+
+    def record_inconclusive(self, key: str) -> None:
+        """An empty/unusable (but non-exception) result - only matters in
+        HALF_OPEN, where it re-opens the breaker rather than fully closing."""
+        with self._lock:
+            state = self._state(key)
+            if state["state"] == self.HALF_OPEN:
+                state["state"] = self.OPEN
+                state["half_open_calls"] = 0
+                state["last_failure_time"] = time.time()
+
+    def status(self) -> dict[str, str]:
+        with self._lock:
+            return {key: info["state"] for key, info in self._states.items()}
+
+
+# Shared default breaker: fetcher instances come and go (a new
+# PortfolioManager/StockDataFetcher per API request, per Phase 0's
+# get_portfolio_manager pattern) but failure history should persist
+# across those instances for the breaker to mean anything.
+default_breaker = CircuitBreaker()
+
+
+class FallbackChain:
+    """An ordered list of named callables for one logical operation
+    (e.g. "get_quote"), tried in order with circuit-breaker protection."""
+
+    def __init__(self, method: str, breaker: Optional[CircuitBreaker] = None):
+        self.method = method
+        self.breaker = breaker or default_breaker
+        self._sources: list[tuple[str, Callable[..., Any]]] = []
+
+    def add(self, name: str, func: Callable[..., Any]) -> "FallbackChain":
+        self._sources.append((name, func))
+        return self
+
+    def call(self, *args: Any, is_empty: Optional[Callable[[Any], bool]] = None, **kwargs: Any) -> Any:
+        last_exc: Optional[Exception] = None
+        for name, func in self._sources:
+            key = f"{name}:{self.method}"
+            if not self.breaker.is_available(key):
+                continue
+            try:
+                result = func(*args, **kwargs)
+            except Exception as exc:  # noqa: BLE001 - deliberately broad, this is the fallback boundary
+                self.breaker.record_failure(key, str(exc))
+                last_exc = exc
+                continue
+
+            if is_empty is not None and is_empty(result):
+                self.breaker.record_inconclusive(key)
+                last_exc = last_exc or RuntimeError(f"{name} returned an empty result")
+                continue
+
+            self.breaker.record_success(key)
+            return result
+
+        raise DataSourceUnavailable(
+            f"All sources exhausted for {self.method}"
+        ) from last_exc

marketmyna/data/sec_facts.py

@@ -0,0 +1,252 @@
+"""US financial statements from SEC XBRL company facts (data.sec.gov, no key).
+
+companyfacts holds every number a company has tagged in its 10-K / 10-Q
+filings since about 2009. This turns the common US-GAAP tags into the same
+three DataFrames yfinance returns (rows named like Yahoo's, columns = period
+end dates, newest first), so marketmyna/services/financials.py and the quality
+scores can use either source. Used when Yahoo has no statements and for
+longer histories (Yahoo keeps 4-5 years).
+
+Total debt excludes lease liabilities (Yahoo includes them).
+Annual = 10-K full-year figures. Quarterly = three-month figures; the fourth
+quarter, which companies do not file separately, is the year minus the first
+three quarters. Flows that are outflows (capex, dividends, buybacks, debt
+repaid) are made negative, as Yahoo shows them. SEC needs SEC_USER_AGENT
+(a name and contact email), see marketmyna/services/sec_mdna.py.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from datetime import date
+from typing import Optional
+
+import pandas as pd
+
+FACTS_URL = "https://data.sec.gov/api/xbrl/companyfacts/CIK{cik:010d}.json"
+CACHE_SECONDS = 12 * 3600
+
+# Yahoo row name -> US-GAAP tags, first one with data wins
+INCOME = {
+    "Total Revenue": ["Revenues", "RevenueFromContractWithCustomerExcludingAssessedTax", "SalesRevenueNet",
+                      "RevenueFromContractWithCustomerIncludingAssessedTax", "SalesRevenueGoodsNet",
+                      "RevenuesNetOfInterestExpense"],
+    "Cost Of Revenue": ["CostOfRevenue", "CostOfGoodsAndServicesSold", "CostOfGoodsSold"],
+    "Gross Profit": ["GrossProfit"],
+    "Operating Expense": ["OperatingExpenses"],
+    "Selling General And Administration": ["SellingGeneralAndAdministrativeExpense"],
+    "Operating Income": ["OperatingIncomeLoss"],
+    "Interest Expense": ["InterestExpense", "InterestExpenseNonoperating", "InterestExpenseDebt"],
+    "Pretax Income": ["IncomeLossFromContinuingOperationsBeforeIncomeTaxesExtraordinaryItemsNoncontrollingInterest",
+                      "IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments"],
+    "Tax Provision": ["IncomeTaxExpenseBenefit"],
+    "Net Income": ["NetIncomeLoss", "ProfitLoss"],
+    "Diluted EPS": ["EarningsPerShareDiluted", "EarningsPerShareBasicAndDiluted"],
+    "Basic EPS": ["EarningsPerShareBasic"],
+    "Diluted Average Shares": ["WeightedAverageNumberOfDilutedSharesOutstanding"],
+}
+BALANCE = {
+    "Total Assets": ["Assets"],
+    "Current Assets": ["AssetsCurrent"],
+    "Cash And Cash Equivalents": ["CashAndCashEquivalentsAtCarryingValue", "Cash"],
+    "Short Term Investments": ["ShortTermInvestments", "MarketableSecuritiesCurrent", "AvailableForSaleSecuritiesDebtSecuritiesCurrent"],
+    "Accounts Receivable": ["AccountsReceivableNetCurrent", "ReceivablesNetCurrent"],
+    "Inventory": ["InventoryNet"],
+    "Net PPE": ["PropertyPlantAndEquipmentNet"],
+    "Total Liabilities Net Minority Interest": ["Liabilities"],
+    "Current Liabilities": ["LiabilitiesCurrent"],
+    "Long Term Debt": ["LongTermDebtNoncurrent", "LongTermDebt"],
+    "Current Debt": ["LongTermDebtCurrent", "DebtCurrent"],
+    "Commercial Paper": ["CommercialPaper", "ShortTermBorrowings"],
+    "Stockholders Equity": ["StockholdersEquity", "StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest"],
+    "Retained Earnings": ["RetainedEarningsAccumulatedDeficit"],
+    "Goodwill": ["Goodwill"],
+    "Other Intangible Assets": ["IntangibleAssetsNetExcludingGoodwill", "FiniteLivedIntangibleAssetsNet"],
+    "Ordinary Shares Number": ["CommonStockSharesOutstanding"],
+}
+CASHFLOW = {
+    "Operating Cash Flow": ["NetCashProvidedByUsedInOperatingActivities",
+                            "NetCashProvidedByUsedInOperatingActivitiesContinuingOperations"],
+    "Capital Expenditure": ["PaymentsToAcquirePropertyPlantAndEquipment", "PaymentsToAcquireProductiveAssets"],
+    "Investing Cash Flow": ["NetCashProvidedByUsedInInvestingActivities",
+                            "NetCashProvidedByUsedInInvestingActivitiesContinuingOperations"],
+    "Financing Cash Flow": ["NetCashProvidedByUsedInFinancingActivities",
+                            "NetCashProvidedByUsedInFinancingActivitiesContinuingOperations"],
+    "Cash Dividends Paid": ["PaymentsOfDividends", "PaymentsOfDividendsCommonStock"],
+    "Repurchase Of Capital Stock": ["PaymentsForRepurchaseOfCommonStock"],
+    "Issuance Of Debt": ["ProceedsFromIssuanceOfLongTermDebt", "ProceedsFromIssuanceOfDebt"],
+    "Repayment Of Debt": ["RepaymentsOfLongTermDebt", "RepaymentsOfDebt"],
+    "Stock Based Compensation": ["ShareBasedCompensation", "AllocatedShareBasedCompensationExpense"],
+    "Depreciation And Amortization": ["DepreciationDepletionAndAmortization", "DepreciationAndAmortization",
+                                      "DepreciationAmortizationAndAccretionNet"],
+    "Changes In Cash": ["CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect",
+                        "CashAndCashEquivalentsPeriodIncreaseDecrease"],
+}
+OUTFLOWS = {"Capital Expenditure", "Cash Dividends Paid", "Repurchase Of Capital Stock", "Repayment Of Debt"}
+PER_SHARE = {"Diluted EPS", "Basic EPS"}
+# Not sums over time: no quarter is derived from year-to-date or full-year figures
+NON_ADDITIVE = PER_SHARE | {"Diluted Average Shares"}
+
+_cache: dict[int, tuple[float, dict]] = {}
+_lock = threading.Lock()
+
+
+def company_facts(symbol: str) -> dict:
+    from ..services.sec_mdna import _get, cik_for
+
+    cik = cik_for(symbol)
+    with _lock:
+        hit = _cache.get(cik)
+    if hit and time.time() - hit[0] < CACHE_SECONDS:
+        return hit[1]
+    facts = _get(FACTS_URL.format(cik=cik)).json()
+    with _lock:
+        _cache[cik] = (time.time(), facts)
+    return facts
+
+
+def _days(start: str, end: str) -> int:
+    return (date.fromisoformat(end) - date.fromisoformat(start)).days
+
+
+def _entries(facts: dict, tag: str) -> list[dict]:
+    node = ((facts.get("facts") or {}).get("us-gaap") or {}).get(tag) or {}
+    units = node.get("units") or {}
+    for unit in ("USD", "USD/shares", "shares"):
+        if unit in units:
+            return units[unit]
+    return next(iter(units.values()), [])
+
+
+def _latest_by_end(rows: list[dict]) -> dict[str, float]:
+    """end date -> value, keeping the most recently filed figure (restatements win)."""
+    best: dict[str, tuple[str, float]] = {}
+    for r in rows:
+        end, filed = r.get("end"), r.get("filed") or ""
+        if end and (end not in best or filed >= best[end][0]):
+            best[end] = (filed, float(r["val"]))
+    return {end: v for end, (_, v) in best.items()}
+
+
+def _flow_one(facts: dict, tag: str, quarterly: bool, additive: bool = True) -> dict[str, float]:
+    entries = [e for e in _entries(facts, tag) if e.get("start") and e.get("end")]
+    if not entries:
+        return {}
+    yearly = [e for e in entries if 350 <= _days(e["start"], e["end"]) <= 380]
+    annual = _latest_by_end([e for e in yearly if (e.get("form") or "").startswith("10-K")])
+    if not quarterly:
+        return annual
+    quarters = _latest_by_end([e for e in entries if 80 <= _days(e["start"], e["end"]) <= 100])
+    if not additive:
+        return quarters
+    # Cash-flow statements in a 10-Q are year to date (6 or 9 months): a quarter
+    # is the year-to-date figure minus the one before it from the same start.
+    by_start: dict[str, dict[str, float]] = {}
+    for e in entries:
+        by_start.setdefault(e["start"], {})
+    for start in by_start:
+        by_start[start] = _latest_by_end([e for e in entries if e["start"] == start])
+    for start, ytd in by_start.items():
+        ends = sorted(ytd)
+        for prev, end in zip(ends, ends[1:]):
+            if end not in quarters and 80 <= _days(prev, end) <= 100:
+                quarters[end] = ytd[end] - ytd[prev]
+    # Q4 = year minus the three quarters inside it
+    for end, total in annual.items():
+        if end in quarters:
+            continue
+        start = next(e["start"] for e in yearly if e["end"] == end)
+        inside = [v for q_end, v in quarters.items() if start < q_end < end]
+        if len(inside) == 3:
+            quarters[end] = round(total - sum(inside), 4)
+    return quarters
+
+
+def _instant_one(facts: dict, tag: str, quarterly: bool) -> dict[str, float]:
+    entries = [e for e in _entries(facts, tag) if e.get("end") and not e.get("start")]
+    if not quarterly:
+        entries = [e for e in entries if (e.get("form") or "").startswith("10-K") and e.get("fp") == "FY"]
+    return _latest_by_end(entries)
+
+
+def _merge(series: list[dict[str, float]]) -> dict[str, float]:
+    """Companies switch tags over the years (Apple's revenue moved from Revenues to
+    RevenueFromContractWithCustomer... in 2018): each period takes the first tag,
+    in priority order, that has it."""
+    out: dict[str, float] = {}
+    for values in series:
+        for end, v in values.items():
+            out.setdefault(end, v)
+    return out
+
+
+def flow_series(facts: dict, tags: list[str], quarterly: bool, additive: bool = True) -> dict[str, float]:
+    """Figures over a period (income, cash flow) by period end."""
+    return _merge([_flow_one(facts, t, quarterly, additive) for t in tags])
+
+
+def instant_series(facts: dict, tags: list[str], quarterly: bool) -> dict[str, float]:
+    """Balances at a date (balance sheet), from 10-K (annual) or any filing (quarterly)."""
+    return _merge([_instant_one(facts, t, quarterly) for t in tags])
+
+
+def _frame(series: dict[str, dict[str, float]], periods: list[str]) -> pd.DataFrame:
+    cols = [pd.Timestamp(p) for p in periods]
+    data = {name: [values.get(p) for p in periods] for name, values in series.items() if values}
+    return pd.DataFrame.from_dict(data, orient="index", columns=cols) if data else pd.DataFrame(columns=cols)
+
+
+def statements(symbol: str, quarterly: bool = False, max_periods: Optional[int] = None,
+               facts: Optional[dict] = None) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+    """(income, balance, cashflow) like yfinance's, newest period first."""
+    facts = facts if facts is not None else company_facts(symbol)
+    limit = max_periods or (5 if quarterly else 10)
+
+    income = {name: flow_series(facts, tags, quarterly, name not in NON_ADDITIVE) for name, tags in INCOME.items()}
+    cash = {name: flow_series(facts, tags, quarterly) for name, tags in CASHFLOW.items()}
+    balance = {name: instant_series(facts, tags, quarterly) for name, tags in BALANCE.items()}
+    for name in OUTFLOWS:
+        cash[name] = {k: -abs(v) for k, v in cash[name].items()}
+
+    # Revenue or net income: banks stop tagging quarterly revenue under one tag for years
+    periods = sorted(set(income["Total Revenue"]) | set(income["Net Income"]), reverse=True)[:limit]
+    if not periods:
+        raise ValueError(f"No SEC financial statements tagged for {symbol}")
+
+    # Balance sheets dated a few days off the income period end are matched to it.
+    def align(values: dict[str, float]) -> dict[str, float]:
+        out = {}
+        for p in periods:
+            if p in values:
+                out[p] = values[p]
+                continue
+            near = [d for d in values if abs(_days(d, p)) <= 7]
+            if near:
+                out[p] = values[min(near, key=lambda d: abs(_days(d, p)))]
+        return out
+
+    balance = {k: align(v) for k, v in balance.items()}
+    cash = {k: align(v) for k, v in cash.items()}
+
+    # Derived rows Yahoo also shows
+    def combine(*rows: dict[str, float]) -> dict[str, float]:
+        return {p: sum(r[p] for r in rows if p in r) for p in periods if any(p in r for r in rows)}
+
+    if not income["Gross Profit"] and income["Total Revenue"] and income["Cost Of Revenue"]:
+        income["Gross Profit"] = {p: income["Total Revenue"][p] - income["Cost Of Revenue"][p]
+                                  for p in periods if p in income["Total Revenue"] and p in income["Cost Of Revenue"]}
+    if income["Operating Income"] and cash["Depreciation And Amortization"]:
+        income["EBITDA"] = combine(income["Operating Income"], cash["Depreciation And Amortization"])
+    balance["Total Debt"] = combine(balance.pop("Long Term Debt"), balance.pop("Current Debt"),
+                                    balance.pop("Commercial Paper"))
+    balance["Cash Cash Equivalents And Short Term Investments"] = combine(
+        balance["Cash And Cash Equivalents"], balance.pop("Short Term Investments"))
+    if balance["Total Debt"]:
+        balance["Net Debt"] = {p: balance["Total Debt"][p] - balance["Cash And Cash Equivalents"].get(p, 0)
+                               for p in balance["Total Debt"]}
+    balance["Goodwill And Other Intangible Assets"] = combine(balance["Goodwill"], balance.pop("Other Intangible Assets"))
+    if cash["Operating Cash Flow"] and cash["Capital Expenditure"]:
+        cash["Free Cash Flow"] = combine(cash["Operating Cash Flow"], cash["Capital Expenditure"])
+
+    return _frame(income, periods), _frame(balance, periods), _frame(cash, periods)