marketmyna 0.2.0__py3-none-any.whl

marketmyna/services/market_valuation.py

@@ -0,0 +1,231 @@
+"""How expensive the whole market is, against its own history.
+
+US: S&P 500 P/E, Shiller P/E (CAPE), dividend yield and earnings yield,
+monthly since 1871, from multpl.com tables (no key). Each reading is placed
+as a percentile of its full history and of the years since 1990.
+
+India: P/E, P/B and dividend yield of the main Nifty indices from NSE's
+allIndices feed (current values only). One snapshot a day is kept under
+~/.marketmyna/valuation/, so a percentile appears once a year of history
+has built up; until then the card shows the reading and the date the
+history starts. Failed fetches serve the last good copy marked stale.
+
+Readings are descriptive (where today sits in the record), not a forecast.
+"""
+
+from __future__ import annotations
+
+import html
+import json
+import logging
+import re
+import threading
+import time
+from datetime import date, datetime
+from pathlib import Path
+from typing import Callable, Optional
+
+import requests
+
+from marketmyna.core.home import app_home
+
+logger = logging.getLogger(__name__)
+
+CACHE_SECONDS = 6 * 3600
+MIN_HISTORY_DAYS = 250  # trading days of India snapshots before a percentile is shown
+
+US_SERIES = [
+    ("shiller-pe", "Shiller P/E (CAPE)", "Price over 10-year average inflation-adjusted earnings"),
+    ("s-p-500-pe-ratio", "S&P 500 P/E", "Price over trailing 12-month earnings"),
+    ("s-p-500-earnings-yield", "Earnings yield", "Trailing earnings over price, %"),
+    ("s-p-500-dividend-yield", "Dividend yield", "Trailing dividends over price, %"),
+]
+INDIA_INDICES = ["NIFTY 50", "NIFTY NEXT 50", "NIFTY 500", "NIFTY BANK", "NIFTY MIDCAP 100", "NIFTY SMALLCAP 100"]
+_UA = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36"}
+
+_cache: dict[str, tuple[float, dict]] = {}
+_lock = threading.Lock()
+
+
+def _dir() -> Path:
+    path = app_home() / "valuation"
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def _read(name: str):
+    try:
+        return json.loads((_dir() / name).read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+
+
+def _write(name: str, data) -> None:
+    try:
+        tmp = _dir() / f".{name}.tmp"
+        tmp.write_text(json.dumps(data), encoding="utf-8")
+        tmp.replace(_dir() / name)
+    except OSError as exc:
+        logger.warning("Could not save %s: %s", name, exc)
+
+
+# ---------------------------------------------------------------- US (multpl)
+
+
+def parse_multpl(page: str) -> list[tuple[str, float, bool]]:
+    """(ISO date, value, estimated) rows, newest first, from a /table/by-month page."""
+    rows = []
+    for when, cell in re.findall(r"<td>([A-Z][a-z]{2} \d{1,2}, \d{4})</td>\s*<td>(.*?)</td>", page, re.S):
+        text = re.sub(r"<[^>]+>|\s", "", html.unescape(cell)).replace(" ", "")
+        estimated = "†" in text  # dagger: multpl's estimate marker
+        try:
+            value = float(text.replace("†", "").replace("%", ""))
+        except ValueError:
+            continue
+        rows.append((datetime.strptime(when, "%b %d, %Y").date().isoformat(), value, estimated))
+    return rows
+
+
+def _multpl(slug: str) -> str:
+    r = requests.get(f"https://www.multpl.com/{slug}/table/by-month", headers=_UA, timeout=20)
+    r.raise_for_status()
+    return r.text
+
+
+def percentile(values: list[float], value: float) -> Optional[float]:
+    """Share of the history at or below `value`, 0-100."""
+    if not values:
+        return None
+    return round(sum(v <= value for v in values) / len(values) * 100, 1)
+
+
+def _median(values: list[float]) -> Optional[float]:
+    if not values:
+        return None
+    s = sorted(values)
+    mid = len(s) // 2
+    return s[mid] if len(s) % 2 else (s[mid - 1] + s[mid]) / 2
+
+
+def us_series(slug: str, label: str, note: str, rows: list[tuple[str, float, bool]]) -> Optional[dict]:
+    if not rows:
+        return None
+    when, value, estimated = rows[0]
+    history = [v for _, v, _ in rows[1:]]  # the monthly record, excluding today's reading
+    since_1990 = [v for d, v, _ in rows[1:] if d >= "1990-01-01"]
+    spark = [{"date": d, "value": v} for d, v, _ in reversed(rows[:241])]  # about 20 years, monthly
+    return {
+        "key": slug, "label": label, "note": note, "value": value, "date": when, "estimated": estimated,
+        "percentile_all": percentile(history, value), "percentile_1990": percentile(since_1990, value),
+        "median_all": _median(history), "median_1990": _median(since_1990),
+        "since": rows[-1][0], "spark": spark,
+    }
+
+
+def _us(fetch: Callable[[str], str]) -> dict:
+    out, failed = [], []
+    for slug, label, note in US_SERIES:
+        try:
+            rows = parse_multpl(fetch(slug))
+            if not rows:
+                raise ValueError("no rows")
+            _write(f"multpl-{slug}.json", rows)
+            stale = False
+        except Exception as exc:  # noqa: BLE001 - fall back to the last good copy
+            logger.info("multpl %s failed: %s", slug, exc)
+            rows, stale = [tuple(r) for r in (_read(f"multpl-{slug}.json") or [])], True
+        item = us_series(slug, label, note, rows)
+        if item:
+            item["stale"] = stale
+            out.append(item)
+        else:
+            failed.append(label)
+    return {"series": out, "failed": failed, "source": "multpl.com (S&P 500 data back to 1871)"}
+
+
+# ------------------------------------------------------------------ India (NSE)
+
+
+def _nse_indices() -> list[dict]:
+    from ..trading.options import _nse
+
+    return (_nse._get("allIndices", {}) or {}).get("data") or []
+
+
+def _num(v) -> Optional[float]:
+    try:
+        f = float(v)
+    except (TypeError, ValueError):
+        return None
+    return f if f > 0 else None
+
+
+def _india(fetch: Callable[[], list[dict]], today: Optional[date] = None) -> dict:
+    today = today or date.today()
+    history: dict[str, dict] = _read("nse-history.json") or {}
+    stale = False
+    try:
+        rows = {r.get("index"): r for r in fetch()}
+        snap = {name: {"pe": _num(r.get("pe")), "pb": _num(r.get("pb")), "dy": _num(r.get("dy")),
+                       "last": _num(r.get("last"))}
+                for name in INDIA_INDICES if (r := rows.get(name)) and _num(r.get("pe"))}
+        if not snap:
+            raise ValueError("NSE returned no index valuations")
+        history[today.isoformat()] = snap
+        _write("nse-history.json", history)
+    except Exception as exc:  # noqa: BLE001 - serve the latest saved day
+        logger.info("NSE allIndices failed: %s", exc)
+        stale = True
+    if not history:
+        return {"indices": [], "stale": True, "source": "NSE"}
+
+    days = sorted(history)
+    latest_day = days[-1]
+    out = []
+    for name in INDIA_INDICES:
+        now = history[latest_day].get(name)
+        if not now:
+            continue
+        item = {"index": name, "date": latest_day, "last": now.get("last"), "history_from": days[0],
+                "history_days": len(days)}
+        for metric in ("pe", "pb", "dy"):
+            past = [history[d][name][metric] for d in days[:-1] if history[d].get(name, {}).get(metric)]
+            item[metric] = now.get(metric)
+            item[f"{metric}_percentile"] = (percentile(past, now[metric])
+                                            if now.get(metric) and len(past) >= MIN_HISTORY_DAYS else None)
+        year_ago = [d for d in days if d <= (today.replace(year=today.year - 1)).isoformat()]
+        item["pe_year_ago"] = history[year_ago[-1]][name]["pe"] if year_ago and name in history[year_ago[-1]] else None
+        out.append(item)
+    return {"indices": out, "stale": stale, "as_of": latest_day,
+            "source": "NSE index P/E, P/B and dividend yield; daily snapshots kept from " + days[0]}
+
+
+def build(fetch_us: Callable[[str], str] = _multpl, fetch_india: Callable[[], list[dict]] = _nse_indices,
+          use_cache: bool = True) -> dict:
+    if use_cache:
+        with _lock:
+            hit = _cache.get("all")
+        if hit and time.time() - hit[0] < CACHE_SECONDS:
+            return hit[1]
+    out = {"us": _us(fetch_us), "india": _india(fetch_india), "updated": time.strftime("%Y-%m-%dT%H:%M:%S")}
+    if use_cache and (out["us"]["series"] or out["india"]["indices"]):
+        with _lock:
+            _cache["all"] = (time.time(), out)
+    return out
+
+
+def snapshot_loop(check_seconds: int = 3 * 3600) -> None:
+    """Background thread: save today's NSE valuations once, after the close
+    (16:00 IST), on weekdays, so the history builds without visitors."""
+    from zoneinfo import ZoneInfo
+
+    time.sleep(180)  # let start-up traffic settle
+    while True:
+        try:
+            now = datetime.now(ZoneInfo("Asia/Kolkata"))
+            saved = _read("nse-history.json") or {}
+            if now.weekday() < 5 and now.hour >= 16 and now.date().isoformat() not in saved:
+                _india(_nse_indices, today=now.date())
+        except Exception as exc:  # noqa: BLE001 - never kill the thread
+            logger.info("Valuation snapshot failed: %s", exc)
+        time.sleep(check_seconds)

marketmyna/services/mf_holdings.py

@@ -0,0 +1,199 @@
+"""Mutual fund holdings in the web portfolio: CAS import and live NAVs.
+
+A fund holding is an ordinary portfolio holding with market "mf" and the AMFI
+scheme code as its symbol, so the existing transaction ledger, gains and
+delete/replay logic all apply. NAV, name and category come from mfapi.in.
+
+CAS import uses casparser (CAMS / KFintech / MFCentral statements, usually
+locked with the investor's PAN). The PDF and password stay in memory only, and
+nothing personal (PAN, email, address) is returned or stored.
+"""
+
+from __future__ import annotations
+
+import io
+import threading
+import time
+from datetime import date, datetime
+
+import requests
+
+MARKET = "mf"
+_TTL = 15 * 60
+_cache: dict[int, tuple[float, dict | None]] = {}
+_lock = threading.Lock()
+
+
+def latest(code: int | str) -> dict | None:
+    """Latest NAV plus scheme name / AMC / category, cached 15 minutes."""
+    try:
+        code = int(code)
+    except (TypeError, ValueError):
+        return None
+    with _lock:
+        hit = _cache.get(code)
+    if hit and time.monotonic() - hit[0] < _TTL:
+        return hit[1]
+    info = None
+    try:
+        r = requests.get(f"https://api.mfapi.in/mf/{code}/latest", timeout=10)
+        r.raise_for_status()
+        data = r.json()
+        meta, rows = data.get("meta") or {}, data.get("data") or []
+        if rows:
+            info = {
+                "name": meta.get("scheme_name") or str(code),
+                "amc": meta.get("fund_house") or "",
+                "category": meta.get("scheme_category") or "",
+                "nav": float(rows[0]["nav"]),
+                "nav_date": datetime.strptime(rows[0]["date"], "%d-%m-%Y").date().isoformat(),
+            }
+    except Exception:
+        if hit:  # keep serving the stale value rather than zeroing the holding
+            return hit[1]
+    with _lock:
+        _cache[code] = (time.monotonic(), info)
+    return info
+
+
+_HIST_TTL = 6 * 3600
+_hist: dict[int, tuple[float, list[tuple[date, float]]]] = {}
+
+
+def nav_on(code: int | str, day: date) -> dict | None:
+    """Official NAV of a scheme on `day`, or the last one published before it
+    (weekends and holidays have none). History cached a few hours per scheme."""
+    try:
+        code = int(code)
+    except (TypeError, ValueError):
+        return None
+    with _lock:
+        hit = _hist.get(code)
+    if hit and time.monotonic() - hit[0] < _HIST_TTL:
+        rows = hit[1]
+    else:
+        try:
+            r = requests.get(f"https://api.mfapi.in/mf/{code}", timeout=15)
+            r.raise_for_status()
+            rows = sorted(
+                (datetime.strptime(x["date"], "%d-%m-%Y").date(), float(x["nav"]))
+                for x in (r.json().get("data") or [])
+            )
+        except Exception:
+            if not hit:
+                return None
+            rows = hit[1]
+        with _lock:
+            _hist[code] = (time.monotonic(), rows)
+    before = [(d, nav) for d, nav in rows if d <= day]
+    if not before:
+        return None
+    d, nav = before[-1]
+    return {"nav": nav, "nav_date": d.isoformat()}
+
+
+class CASError(ValueError):
+    """The statement could not be read (wrong password, not a CAS, ...)."""
+
+
+def parse_cas(pdf: bytes, password: str) -> dict:
+    """Read a CAS PDF into per-scheme holdings (aggregated across folios)."""
+    import casparser
+    from casparser.exceptions import IncorrectPasswordError, ParserException
+
+    try:
+        cas = casparser.read_cas_pdf(io.BytesIO(pdf), password)
+    except IncorrectPasswordError as exc:
+        raise CASError("Wrong password. CAS statements are usually locked with your PAN in capitals.") from exc
+    except ParserException as exc:
+        raise CASError(f"Could not read this statement: {exc}") from exc
+    except Exception as exc:  # malformed PDF etc
+        raise CASError(f"Could not read this file as a CAS statement ({type(exc).__name__}).") from exc
+    if not hasattr(cas, "folios"):
+        raise CASError("Depository (NSDL / CDSL) statements are not supported yet. "
+                       "Use the CAMS / KFintech CAS from camsonline.com or MFCentral.")
+    return summarise(cas)
+
+
+def summarise(cas) -> dict:
+    """CASData -> holdings rows; split out so tests can feed a fake statement."""
+    rows: dict[str, dict] = {}
+    skipped: list[dict] = []
+    for folio in cas.folios:
+        for s in folio.schemes:
+            units = max(float(s.close or 0), 0.0)  # 0 = fully redeemed, still syncs a sale
+            if not s.amfi:
+                if units <= 0:
+                    continue
+                skipped.append({"scheme": s.scheme, "reason": "No AMFI code, cannot price it"})
+                continue
+            v = s.valuation
+            cost = float(v.cost) if v.cost is not None else None
+            row = rows.setdefault(s.amfi, {
+                "amfi": str(s.amfi), "isin": s.isin, "scheme": s.scheme, "amc": folio.amc,
+                "folios": 0, "units": 0.0, "cost": 0.0, "cost_known": True,
+                "value": 0.0, "nav": float(v.nav), "date": _iso(v.date),
+            })
+            row["folios"] += 1
+            row["units"] += units
+            row["value"] += float(v.value)
+            if cost is None:
+                row["cost_known"] = False
+            else:
+                row["cost"] += cost
+    holdings = sorted((r for r in rows.values()), key=lambda r: (r["units"] <= 0, -r["value"]))
+    for r in holdings:
+        if not r["cost_known"]:
+            r["cost"] = None
+    period = getattr(cas, "statement_period", None)
+    return {
+        "statement_to": getattr(period, "to", None),
+        "cas_type": str(getattr(cas, "cas_type", "") or ""),
+        "holdings": holdings,
+        "skipped": skipped,
+        "total_value": sum(r["value"] for r in holdings),
+        "warnings": list(getattr(cas, "parse_warnings", []) or []),
+    }
+
+
+def plan_import(held: dict[str, float], rows: list[dict]) -> list[dict]:
+    """Transactions that bring each scheme's units to the statement's balance.
+
+    New scheme: one BUY at the statement cost (or NAV when cost is unknown).
+    Already held: BUY / SELL only the unit difference at the statement NAV, so a
+    monthly re-import stays in sync without double counting.
+    """
+    out = []
+    for r in rows:
+        code, units = str(r["amfi"]), float(r["units"])
+        have = float(held.get(code, 0.0))
+        diff = -have if units <= 1e-6 else units - have  # full exit sells exactly what is held
+        if abs(diff) < 1e-3:
+            continue
+        when = r.get("date") or date.today().isoformat()
+        if have <= 0 and r.get("cost") and units > 0:
+            price = float(r["cost"]) / units
+        else:
+            price = float(r["nav"])
+        out.append({
+            "symbol": code,
+            "type": "buy" if diff > 0 else "sell",
+            "quantity": round(diff, 6) if diff > 0 else min(-diff, have),
+            "price": price,
+            "date": when,
+            "notes": f"CAS import: {r.get('scheme', code)}"[:200],
+        })
+    return out
+
+
+def _iso(d) -> str | None:
+    if d is None:
+        return None
+    if isinstance(d, (date, datetime)):
+        return d.isoformat()[:10]
+    for fmt in ("%d-%b-%Y", "%Y-%m-%d", "%d-%m-%Y"):
+        try:
+            return datetime.strptime(str(d), fmt).date().isoformat()
+        except ValueError:
+            continue
+    return None

marketmyna/services/mf_insights.py

@@ -0,0 +1,132 @@
+"""Health check for the mutual funds in a portfolio.
+
+Works without factsheet PDFs: fund stock lists have no free source, so overlap is estimated
+from how closely the funds' daily NAV returns move together, plus funds that
+sit in the same SEBI category. Category labels come from mfapi.in, e.g.
+"Equity Scheme - Mid Cap Fund".
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+# Indian equity funds mostly correlate 0.85-0.93 with each other, so only near-identical
+# daily moves count as overlap; the related band is listed but not penalised.
+SIMILAR = 0.95
+RELATED = 0.85
+
+
+_DEBT_WORDS = ("gilt", "bond", "sdl", "debt", "g-sec", "gsec", "liquid", "money market", "psu", "crisil ibx", "t-bill")
+
+
+def asset_class(category: str, name: str = "") -> str:
+    c, n = (category or "").lower(), (name or "").lower()
+    if c.startswith("equity"):
+        return "Equity"
+    if c.startswith("debt"):
+        return "Debt"
+    if c.startswith("hybrid"):
+        return "Hybrid"
+    if "index" in c or "etf" in c:  # SEBI files index funds under "Other"; the name tells equity from debt
+        if any(w in n for w in _DEBT_WORDS):
+            return "Debt"
+        if "gold" in n or "silver" in n:
+            return "Commodity"
+        return "Equity"
+    if "fof" in c or "fund of funds" in c:
+        return "Fund of funds"
+    if c.startswith("solution"):
+        return "Solution oriented"
+    return "Other"
+
+
+def sub_category(category: str) -> str:
+    """'Equity Scheme - Mid Cap Fund' -> 'Mid Cap Fund'."""
+    return (category or "").split(" - ", 1)[-1].strip() or "Unknown"
+
+
+def insights(funds: list[dict], navs: dict[str, pd.Series]) -> dict:
+    """funds: [{symbol, name, category, market_value}]; navs: symbol -> NAV series."""
+    funds = [f for f in funds if (f.get("market_value") or 0) > 0]
+    total = sum(float(f["market_value"]) for f in funds)
+    if not funds or total <= 0:
+        return {"funds": 0}
+
+    def short(f: dict) -> str:
+        return (f.get("name") or f["symbol"]).split(" - ")[0]
+
+    by_class: dict[str, float] = {}
+    by_sub: dict[str, list[dict]] = {}
+    for f in funds:
+        w = float(f["market_value"]) / total
+        cls = asset_class(f.get("category", ""), f.get("name", ""))
+        by_class[cls] = by_class.get(cls, 0) + w
+        by_sub.setdefault(sub_category(f.get("category", "")), []).append(f)
+
+    issues: list[str] = []
+    kinds: list[str] = []  # parallel to issues, for the portfolio review
+    score = 100
+
+    top_class, top_w = max(by_class.items(), key=lambda kv: kv[1])
+    if len(funds) >= 3 and top_w > 0.9 and top_class not in ("Hybrid", "Fund of funds"):
+        score -= 10
+        kinds.append("one_asset_class")
+        issues.append(f"{top_w:.0%} is in {top_class} funds. Fine for a long horizon, "
+                      "but there is no cushion from debt or hybrid funds.")
+
+    same = [(sub, fs) for sub, fs in by_sub.items() if len(fs) > 1 and sub != "Unknown"]
+    for sub, fs in same:
+        score -= 8
+        kinds.append("same_category")
+        issues.append(f"{len(fs)} funds in the same category ({sub}): {', '.join(short(f) for f in fs)}. "
+                      "They likely hold similar stocks; one may be enough.")
+
+    pairs = []
+    rets = pd.DataFrame({s: v.pct_change() for s, v in navs.items() if s in {f["symbol"] for f in funds}}).dropna()
+    if rets.shape[1] >= 2 and len(rets) >= 60:
+        corr = rets.corr()
+        names = {f["symbol"]: short(f) for f in funds}
+        cols = list(corr.columns)
+        for i, a in enumerate(cols):
+            for b in cols[i + 1:]:
+                c = float(corr.loc[a, b])
+                if c >= RELATED:
+                    pairs.append({"a": a, "b": b, "a_name": names[a], "b_name": names[b], "correlation": c})
+        pairs.sort(key=lambda p: -p["correlation"])
+        similar = [p for p in pairs if p["correlation"] >= SIMILAR]
+        if similar:
+            score -= min(24, 6 * len(similar))
+            p = similar[0]
+            kinds.append("lockstep")
+            issues.append(f"{len(similar)} fund pair{'s move' if len(similar) > 1 else ' moves'} almost in lockstep "
+                          f"(e.g. {p['a_name']} and {p['b_name']}, correlation {p['correlation']:.2f}), "
+                          "so they add little diversification.")
+
+    regular = [f for f in funds if "regular" in (f.get("name") or "").lower()]
+    if regular:
+        score -= 5 * len(regular)
+        kinds.append("regular_plan")
+        issues.append(f"{len(regular)} Regular-plan fund{'s' if len(regular) > 1 else ''} "
+                      f"({', '.join(short(f) for f in regular)}). Direct plans of the same fund "
+                      "charge less (often 0.5-1% a year), which compounds.")
+
+    idcw = [f for f in funds if any(k in (f.get("name") or "").lower() for k in ("idcw", "dividend"))]
+    if idcw:
+        kinds.append("idcw")
+        issues.append(f"{len(idcw)} IDCW / dividend option fund{'s' if len(idcw) > 1 else ''}: payouts are taxed "
+                      "at your slab rate; Growth options usually suit long-term investing better.")
+
+    score = max(0, min(100, score))
+    return {
+        "funds": len(funds),
+        "total_value": total,
+        "score": score,
+        "grade": "Good" if score >= 80 else "Fair" if score >= 60 else "Needs attention",
+        "asset_classes": [{"name": k, "weight": v} for k, v in sorted(by_class.items(), key=lambda kv: -kv[1])],
+        "categories": [{"name": k, "weight": sum(float(f["market_value"]) for f in fs) / total, "count": len(fs)}
+                       for k, fs in sorted(by_sub.items(), key=lambda kv: -sum(float(f["market_value"]) for f in kv[1]))],
+        "similar_pairs": pairs[:10],
+        "issues": issues,
+        "issue_kinds": kinds,
+        "method": "Overlap is estimated from 1 year of daily NAV moves and shared categories, not from fund stock lists.",
+    }