marketmyna 0.2.0__py3-none-any.whl

marketmyna/services/india_flows.py

@@ -0,0 +1,263 @@
+"""India institutional flows: FII / DII cash-market buying and selling, and
+how each participant group is positioned in index derivatives.
+
+Cash flows: NSE's daily provisional FII/FPI and DII figures (buy, sell, net,
+in ₹ crore; NSE publishes only the latest session, so every session fetched
+is saved under ~/.marketmyna/india/fii_dii.json and the history builds up
+from there - a background thread saves each evening's figures even with no
+visitors; a failed fetch serves the saved copy, marked).
+
+Positioning: NSE's participant-wise open interest file (one CSV per session
+in nsearchives, so the last 30 sessions are fetched directly and each file
+is kept on disk for good). Shows FII / DII / client / pro long and short
+index futures and index options in contracts, and the FII index-futures
+long share, the number Indian traders watch most.
+
+Figures as published; a reading of positioning, not a forecast.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import logging
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor
+from datetime import date, datetime, timedelta
+from pathlib import Path
+from typing import Callable, Optional
+
+import requests
+
+from .india_rates import _dir  # same ~/.marketmyna/india folder
+
+logger = logging.getLogger(__name__)
+
+NSE = "https://www.nseindia.com"
+ARCHIVE = "https://nsearchives.nseindia.com/content/nsccl/fao_participant_oi_{d:%d%m%Y}.csv"
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36",
+    "Accept": "application/json, text/plain, */*",
+    "Referer": "https://www.nseindia.com/reports/fii-dii",
+}
+SESSIONS = 30
+CACHE_SECONDS = 30 * 60
+GROUPS = ("FII", "DII", "Client", "Pro")
+
+_cache: dict[str, tuple[float, dict]] = {}
+_lock = threading.Lock()
+
+
+def _num(x) -> Optional[float]:
+    try:
+        return float(str(x).replace(",", "").strip())
+    except (TypeError, ValueError):
+        return None
+
+
+# ------------------------------------------------------------------ cash flows
+
+def _fetch_cash() -> list[dict]:
+    s = requests.Session()
+    s.headers.update(HEADERS)
+    s.get(f"{NSE}/reports/fii-dii", timeout=15)  # cookies the API wants
+    r = s.get(f"{NSE}/api/fiidiiTradeReact", timeout=15)
+    r.raise_for_status()
+    return r.json()
+
+
+def parse_cash(rows: list[dict]) -> Optional[dict]:
+    """NSE rows -> {"date": "YYYY-MM-DD", "fii": {...}, "dii": {...}} in ₹ crore."""
+    out: dict = {}
+    for r in rows or []:
+        cat = (r.get("category") or "").upper()
+        key = "fii" if cat.startswith("FII") else "dii" if cat.startswith("DII") else None
+        if not key:
+            continue
+        out[key] = {"buy": _num(r.get("buyValue")), "sell": _num(r.get("sellValue")), "net": _num(r.get("netValue"))}
+        try:
+            out["date"] = datetime.strptime(r.get("date", ""), "%d-%b-%Y").date().isoformat()
+        except ValueError:
+            pass
+    return out if "date" in out and ("fii" in out or "dii" in out) else None
+
+
+def _history_file() -> Path:
+    return _dir() / "fii_dii.json"
+
+
+def load_history() -> dict[str, dict]:
+    try:
+        return json.loads(_history_file().read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {}
+
+
+def save_session(day: dict) -> dict[str, dict]:
+    hist = load_history()
+    hist[day["date"]] = {"fii": day.get("fii"), "dii": day.get("dii")}
+    path = _history_file()
+    tmp = path.with_suffix(".tmp")
+    tmp.write_text(json.dumps(dict(sorted(hist.items())), indent=0), encoding="utf-8")
+    tmp.replace(path)
+    return hist
+
+
+def cash(fetch: Callable[[], list[dict]] = _fetch_cash) -> dict:
+    stale = None
+    try:
+        day = parse_cash(fetch())
+        if not day:
+            raise ValueError("NSE returned no FII/DII rows")
+        hist = save_session(day)
+    except Exception as exc:
+        logger.warning("FII/DII fetch failed: %s", exc)
+        hist = load_history()
+        stale = f"Live fetch failed ({type(exc).__name__}); showing the saved figures"
+    dates = sorted(hist)
+    series = [{"date": d, "fii": (hist[d].get("fii") or {}).get("net"), "dii": (hist[d].get("dii") or {}).get("net")} for d in dates]
+
+    def total(key: str, rows: list[dict]) -> Optional[float]:
+        vals = [r[key] for r in rows if r[key] is not None]
+        return round(sum(vals), 2) if vals else None
+
+    latest = dates[-1] if dates else None
+    month = latest[:7] if latest else None
+    mtd = [r for r in series if month and r["date"].startswith(month)]
+    last5 = series[-5:]
+    return {
+        "latest": {"date": latest, **hist[latest]} if latest else None,
+        "series": series[-120:],
+        "sessions_saved": len(dates),
+        "five_day": {"fii": total("fii", last5), "dii": total("dii", last5), "sessions": len(last5)},
+        "month_to_date": {"fii": total("fii", mtd), "dii": total("dii", mtd), "sessions": len(mtd)},
+        "stale": stale,
+    }
+
+
+# ------------------------------------------------------------------ positioning
+
+def _archive_file(d: date) -> Path:
+    path = _dir() / "participant_oi"
+    path.mkdir(parents=True, exist_ok=True)
+    return path / f"{d:%Y%m%d}.csv"
+
+
+def _fetch_archive(d: date) -> Optional[str]:
+    """One session's CSV, from disk if fetched before; None on holidays / not yet out."""
+    path = _archive_file(d)
+    if path.exists():
+        return path.read_text(encoding="utf-8") or None  # empty file = no session that day (holiday)
+    r = requests.get(ARCHIVE.format(d=d), headers={"User-Agent": HEADERS["User-Agent"]}, timeout=20)
+    if r.status_code == 404 or "Participant" not in r.text[:200]:
+        if r.status_code == 404 and d < date.today() - timedelta(days=3):
+            path.write_text("", encoding="utf-8")
+        return None
+    r.raise_for_status()
+    path.write_text(r.text, encoding="utf-8")
+    return r.text
+
+
+def parse_participants(text: str) -> dict[str, dict]:
+    rows = list(csv.reader(io.StringIO(text)))
+    head_i = next(i for i, r in enumerate(rows) if r and r[0].strip().lower() == "client type")
+    head = [h.strip() for h in rows[head_i]]
+    out = {}
+    for r in rows[head_i + 1:]:
+        if not r or not r[0].strip():
+            continue
+        vals = {h: _num(v) for h, v in zip(head[1:], r[1:])}
+        out[r[0].strip()] = vals
+    return out
+
+
+def _summarise(groups: dict[str, dict]) -> dict:
+    out = {}
+    for g in GROUPS:
+        v = groups.get(g)
+        if not v:
+            continue
+        fl, fs = v.get("Future Index Long") or 0, v.get("Future Index Short") or 0
+        cl, cs = v.get("Option Index Call Long") or 0, v.get("Option Index Call Short") or 0
+        pl, ps = v.get("Option Index Put Long") or 0, v.get("Option Index Put Short") or 0
+        out[g] = {
+            "index_futures_long": fl, "index_futures_short": fs, "index_futures_net": fl - fs,
+            "index_futures_long_pct": round(fl / (fl + fs) * 100, 1) if fl + fs else None,
+            # bullish option exposure: calls bought + puts sold, minus puts bought + calls sold
+            "index_options_net": (cl - cs) - (pl - ps),
+            "stock_futures_net": (v.get("Future Stock Long") or 0) - (v.get("Future Stock Short") or 0),
+        }
+    return out
+
+
+def positioning(fetch: Callable[[date], Optional[str]] = _fetch_archive, today: Optional[date] = None,
+                sessions: int = SESSIONS) -> Optional[dict]:
+    today = today or date.today()
+    days = [today - timedelta(days=i) for i in range(int(sessions * 1.6) + 5)]
+    days = [d for d in days if d.weekday() < 5]
+
+    def one(d):
+        try:
+            text = fetch(d)
+            return (d, _summarise(parse_participants(text))) if text else None
+        except Exception as exc:
+            logger.info("Participant OI %s unavailable: %s", d, exc)
+            return None
+
+    with ThreadPoolExecutor(max_workers=6) as pool:
+        got = [x for x in pool.map(one, days) if x and x[1].get("FII")]
+    got.sort(key=lambda x: x[0])
+    got = got[-sessions:]
+    if not got:
+        return None
+    latest_day, latest = got[-1]
+    prev = got[-2][1] if len(got) > 1 else None
+    fii_pct = [x[1]["FII"]["index_futures_long_pct"] for x in got]
+    lows_highs = [v for v in fii_pct if v is not None]
+    return {
+        "date": latest_day.isoformat(),
+        "groups": latest,
+        "previous": prev,
+        "fii_long_pct_series": [{"date": d.isoformat(), "pct": g["FII"]["index_futures_long_pct"],
+                                 "net": g["FII"]["index_futures_net"]} for d, g in got],
+        "fii_long_pct_range": [min(lows_highs), max(lows_highs)] if lows_highs else None,
+        "sessions": len(got),
+    }
+
+
+def snapshot_loop(check_seconds: int = 3600) -> None:
+    """Background thread: save each session's FII/DII figures in the evening
+    (NSE posts them around 18:00-19:30 IST) and fetch the day's participant
+    file, so the history builds without visitors."""
+    from zoneinfo import ZoneInfo
+
+    time.sleep(240)  # let start-up traffic settle
+    while True:
+        try:
+            now = datetime.now(ZoneInfo("Asia/Kolkata"))
+            if now.weekday() < 5 and now.hour >= 19 and now.date().isoformat() not in load_history():
+                cash()
+                positioning()
+        except Exception as exc:  # noqa: BLE001 - never kill the thread
+            logger.info("FII/DII snapshot failed: %s", exc)
+        time.sleep(check_seconds)
+
+
+def build(use_cache: bool = True) -> dict:
+    if use_cache:
+        with _lock:
+            hit = _cache.get("all")
+        if hit and time.time() - hit[0] < CACHE_SECONDS:
+            return hit[1]
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        c = pool.submit(cash)
+        p = pool.submit(positioning)
+        out = {"cash": c.result(), "positioning": p.result(),
+               "source": "NSE provisional FII/DII cash-market figures (₹ crore) and participant-wise open interest "
+                         "in equity derivatives (contracts)"}
+    if use_cache and (out["positioning"] or (out["cash"] and out["cash"]["latest"])):
+        with _lock:
+            _cache["all"] = (time.time(), out)
+    return out

marketmyna/services/india_rates.py

@@ -0,0 +1,258 @@
+"""India rates and bullion: RBI policy / money-market rates, T-bill and G-sec
+yields (an India yield curve), and IBJA benchmark gold / silver prices.
+
+Sources, both free and public, scraped from their HTML:
+- rbi.org.in home page "Current Rates" box: policy rates, CRR / SLR, FBIL
+  reference FX, call money, T-bill cut-offs and benchmark G-sec yields.
+- ibjarates.com: India Bullion and Jewellers Association daily AM / PM rates
+  (Rs per 10 g gold, per kg silver, before GST and making charges), the
+  benchmark most Indian jewellers and gold loans quote. MCX itself blocks
+  scripted access (HTTP 403), so it is not used.
+
+Either site may block cloud IPs or change layout. Every good fetch is saved
+under ~/.marketmyna/india/, and a failed fetch serves that last copy marked
+stale, so the card degrades instead of disappearing.
+"""
+
+from __future__ import annotations
+
+import html as htmllib
+import json
+import logging
+import re
+import threading
+import time
+from datetime import date, datetime
+from pathlib import Path
+from typing import Callable, Optional
+
+from marketmyna.core.home import app_home
+
+logger = logging.getLogger(__name__)
+
+RBI_URL = "https://www.rbi.org.in/"
+IBJA_URL = "https://ibjarates.com/"
+CACHE_SECONDS = 3600
+UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36"
+
+Fetch = Callable[[str], str]
+
+_cache: dict[str, tuple[float, dict]] = {}
+_lock = threading.Lock()
+
+
+def _http(url: str) -> str:
+    import requests
+
+    r = requests.get(url, headers={"User-Agent": UA, "Accept-Language": "en-IN,en;q=0.9"}, timeout=20)
+    r.raise_for_status()
+    return r.text
+
+
+def _dir() -> Path:
+    path = app_home() / "india"
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def _pct(text: str) -> Optional[float]:
+    m = re.search(r"-?\d+(?:\.\d+)?", text.replace(",", ""))
+    return float(m.group(0)) if m else None
+
+
+def _text(fragment: str) -> str:
+    return re.sub(r"\s+", " ", htmllib.unescape(re.sub(r"<[^>]+>", " ", fragment))).strip()
+
+
+# ------------------------------------------------------------------ RBI
+
+POLICY = {
+    "Policy Repo Rate": ("repo", "Repo rate", "What banks pay to borrow from RBI; the main policy rate"),
+    "Standing Deposit Facility Rate": ("sdf", "SDF rate", "Standing deposit facility: floor of the overnight corridor"),
+    "Marginal Standing Facility Rate": ("msf", "MSF rate", "Marginal standing facility: ceiling of the overnight corridor"),
+    "Bank Rate": ("bank_rate", "Bank rate", "Long-term lending rate to banks"),
+    "CRR": ("crr", "CRR", "Cash reserve ratio: deposits banks must park with RBI, unpaid"),
+    "SLR": ("slr", "SLR", "Statutory liquidity ratio: deposits banks must hold in government bonds"),
+}
+
+
+def parse_rbi(page: str) -> dict:
+    """Pull the rate table rows from the RBI home page."""
+    rows = [(_text(k), _text(v).lstrip(": ").strip())
+            for k, v in re.findall(r"<th[^>]*>(.*?)</th>\s*<td[^>]*>(.*?)</td>", page, re.S)]
+    out: dict = {"policy": [], "tbills": [], "gsecs": [], "fx": [], "lending": [], "call_rate": None}
+    for key, val in rows:
+        if key in POLICY:
+            k, label, note = POLICY[key]
+            out["policy"].append({"key": k, "label": label, "value": _pct(val), "note": note})
+        elif m := re.match(r"(\d+) day T-bills", key):
+            out["tbills"].append({"days": int(m.group(1)), "years": int(m.group(1)) / 365, "yield": _pct(val)})
+        elif m := re.match(r"(\d+(?:\.\d+)?)% GS (\d{4})", key):
+            out["gsecs"].append({"name": key, "coupon": float(m.group(1)), "maturity_year": int(m.group(2)),
+                                 "yield": _pct(val)})
+        elif m := re.match(r"INR / (\d+) (\w+)", key):
+            out["fx"].append({"pair": f"{m.group(2)}/INR", "per": int(m.group(1)), "value": _pct(val)})
+        elif key in ("Base Rate", "MCLR (Overnight)", "Savings Deposit Rate", "Term Deposit Rate > 1 Year"):
+            out["lending"].append({"label": key, "value": val})
+    if m := re.search(r"Call Rates\s*</[^>]+>.*?:\s*([\d.]+)%\s*-\s*([\d.]+)%", page, re.S):
+        out["call_rate"] = {"low": float(m.group(1)), "high": float(m.group(2))}
+    if m := re.search(r"As at[^)]*?of\s+([A-Za-z]+ \d{1,2}, \d{4})", _text(page)):
+        out["fx_as_of"] = m.group(1)
+    if m := re.search(r"Government Securities Market.*?as on\s*([A-Za-z]+ \d{1,2}, \d{4})", _text(page)):
+        out["gsec_as_of"] = m.group(1)
+    if not out["policy"]:
+        raise ValueError("RBI page layout not recognised (no policy rates found)")
+    out["curve"] = _curve(out["tbills"], out["gsecs"])
+    return out
+
+
+def _curve(tbills: list[dict], gsecs: list[dict]) -> list[dict]:
+    """T-bills + benchmark G-secs as one curve, x = years to maturity (G-secs approximate,
+    mid-year maturity assumed since the page gives only the year)."""
+    today = date.today()
+    pts = [{"label": f"{t['days']}D", "years": round(t["years"], 3), "yield": t["yield"]} for t in tbills]
+    for g in gsecs:
+        yrs = max(g["maturity_year"] + 0.5 - (today.year + today.timetuple().tm_yday / 365), 0.1)
+        pts.append({"label": str(g["maturity_year"]), "years": round(yrs, 2), "yield": g["yield"]})
+    return sorted((p for p in pts if p["yield"] is not None), key=lambda p: p["years"])
+
+
+# ------------------------------------------------------------------ IBJA
+
+IBJA_FIELDS = {
+    "Gold999": ("gold_999", "Gold 999 (24K)", "₹/10g"),
+    "Gold995": ("gold_995", "Gold 995", "₹/10g"),
+    "Gold916": ("gold_916", "Gold 916 (22K)", "₹/10g"),
+    "Gold750": ("gold_750", "Gold 750 (18K)", "₹/10g"),
+    "Silver999": ("silver_999", "Silver 999", "₹/kg"),
+    "Platinum999": ("platinum_999", "Platinum 999", "₹/10g"),
+}
+
+
+def parse_ibja(page: str) -> dict:
+    def label(id_):
+        m = re.search(rf'id="{id_}"[^>]*>([^<]*)<', page)
+        return _pct(m.group(1)) if m and m.group(1).strip() else None
+
+    items = []
+    for field, (key, name, unit) in IBJA_FIELDS.items():
+        am, pm = label(f"lbl{field}_AM"), label(f"lbl{field}_PM")
+        if am is None and pm is None:
+            continue
+        items.append({"key": key, "label": name, "unit": unit, "am": am, "pm": pm,
+                      "latest": pm if pm is not None else am, "session": "PM" if pm is not None else "AM"})
+    # The headline table holds today's rates; dated tables below it are past days.
+    as_of = _ist_today()
+
+    def series(hidden: str, value_key: str) -> list[dict]:
+        m = re.search(rf'id="{hidden}"[^>]*value="([^"]*)"', page) or re.search(rf'value="([^"]*)"[^>]*id="{hidden}"', page)
+        if not m:
+            return []
+        try:
+            data = json.loads(htmllib.unescape(m.group(1)))
+            return [{"date": datetime.strptime(d, "%d/%m/%Y").date().isoformat(), "value": float(v)}
+                    for d, v in zip(data.get("labels", []), data.get(value_key, [])) if v]
+        except (ValueError, TypeError):
+            return []
+
+    hist = {"gold_999": series("HdnGold", "purity999"), "silver_999": series("HdnSilver", "silverRate")}
+    if not items:  # before the morning fix is published: fall back to the last PM close
+        for key, (name, unit) in {"gold_999": ("Gold 999 (24K)", "₹/10g"), "silver_999": ("Silver 999", "₹/kg")}.items():
+            if hist[key]:
+                last = hist[key][-1]
+                items.append({"key": key, "label": name, "unit": unit, "am": None, "pm": last["value"],
+                              "latest": last["value"], "session": "PM"})
+                as_of = last["date"]
+    if not items:
+        raise ValueError("IBJA page layout not recognised (no rates found)")
+    for i in items:  # change vs the previous day's PM close
+        prev = [h for h in hist.get(i["key"], []) if h["date"] < as_of]
+        i["change_pct"] = (i["latest"] / prev[-1]["value"] - 1) * 100 if prev and i["latest"] else None
+    return {"items": items, "as_of": as_of, "history": hist}
+
+
+def _ist_today() -> str:
+    from datetime import timedelta, timezone
+
+    return datetime.now(timezone(timedelta(hours=5, minutes=30))).date().isoformat()
+
+
+# ------------------------------------------------------------------ fetch with fallback
+
+
+def _load(name: str, fetch: Fetch, url: str, parser) -> dict:
+    path = _dir() / f"{name}.json"
+    try:
+        data = parser(fetch(url))
+        data["fetched_at"] = datetime.now().isoformat(timespec="seconds")
+        data["stale"] = False
+        try:
+            path.write_text(json.dumps(data), encoding="utf-8")
+            _append_history(name, data)
+        except OSError as exc:
+            logger.info("Could not save %s snapshot: %s", name, exc)
+        return data
+    except Exception as exc:
+        logger.warning("%s fetch failed: %s", name.upper(), exc)
+        if path.exists():
+            data = json.loads(path.read_text(encoding="utf-8"))
+            data["stale"] = True
+            data["error"] = f"Live fetch failed; showing the copy from {data.get('fetched_at', 'earlier')}."
+            return data
+        return {"error": f"{name.upper()} unavailable right now", "stale": True}
+
+
+def _append_history(name: str, data: dict) -> None:
+    """One snapshot per day of the headline numbers, so trends build up over time."""
+    path = _dir() / f"{name}_history.json"
+    hist = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
+    day = date.today().isoformat()
+    if name == "rbi":
+        snap = {p["key"]: p["value"] for p in data.get("policy", [])}
+        snap.update({f"tbill_{t['days']}": t["yield"] for t in data.get("tbills", [])})
+        snap.update({f"gs_{g['maturity_year']}": g["yield"] for g in data.get("gsecs", [])})
+    else:
+        snap = {i["key"]: i["latest"] for i in data.get("items", [])}
+    hist[day] = snap
+    path.write_text(json.dumps(dict(sorted(hist.items())[-1500:])), encoding="utf-8")
+
+
+def history(name: str) -> dict:
+    path = _dir() / f"{name}_history.json"
+    return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
+
+
+def build(fetch: Fetch = _http, use_cache: bool = True) -> dict:
+    if use_cache:
+        with _lock:
+            hit = _cache.get("all")
+        if hit and time.time() - hit[0] < CACHE_SECONDS:
+            return hit[1]
+    from concurrent.futures import ThreadPoolExecutor
+
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        rbi = pool.submit(_load, "rbi", fetch, RBI_URL, parse_rbi)
+        ibja = pool.submit(_load, "ibja", fetch, IBJA_URL, parse_ibja)
+        out = {"rbi": rbi.result(), "ibja": ibja.result()}
+    out["sources"] = {"rbi": RBI_URL, "ibja": IBJA_URL}
+    if use_cache and not (out["rbi"].get("stale") and out["ibja"].get("stale")):
+        with _lock:
+            _cache["all"] = (time.time(), out)
+    return out
+
+
+def context_lines(data: Optional[dict] = None) -> list[str]:
+    """Short lines for LLM prompts."""
+    data = data or build()
+    lines = []
+    rbi = data.get("rbi", {})
+    for p in rbi.get("policy", []):
+        if p["key"] in ("repo", "crr") and p["value"] is not None:
+            lines.append(f"RBI {p['label']}: {p['value']:.2f}%")
+    for t in rbi.get("tbills", []):
+        if t["days"] == 364 and t["yield"] is not None:
+            lines.append(f"India 364-day T-bill: {t['yield']:.2f}%")
+    for i in data.get("ibja", {}).get("items", []):
+        if i["key"] in ("gold_999", "silver_999") and i["latest"]:
+            lines.append(f"IBJA {i['label']}: Rs {i['latest']:,.0f} {i['unit'].replace('₹', '')}")
+    return lines