oq-data 0.1.0__py3-none-any.whl

oq_data/__init__.py

@@ -0,0 +1,94 @@
+"""oq-data — NSE/BSE data pipeline for OpenQuant India.
+
+Top-level convenience imports mirror the most-used public API:
+
+    >>> from oq_data import prices, universe, wide_prices
+"""
+
+from __future__ import annotations
+
+from oq_data.announcements import (
+    download_announcements,
+    parse_announcements_blob,
+    read_announcements,
+    write_announcements,
+)
+from oq_data.api import list_symbols, prices, resolve_symbol, universe, wide_prices
+from oq_data.bhavcopy import build_url, download_bhavcopy, parse_bhavcopy_blob, sync_range
+from oq_data.config import DataPaths, default_root, get_paths
+from oq_data.corporate_actions import CorporateAction, add_actions, adjust_prices, load_actions
+from oq_data.delivery import (
+    download_delivery,
+    parse_delivery_blob,
+    read_delivery,
+    write_delivery,
+)
+from oq_data.flows import (
+    download_flows,
+    parse_flows_blob,
+    read_flows,
+    write_flows,
+)
+from oq_data.fno import (
+    download_fno,
+    parse_fno_blob,
+)
+from oq_data.storage import (
+    coverage,
+    query,
+    read_fno,
+    read_prices,
+    write_eod,
+    write_fno,
+)
+from oq_data.symbols import SymbolMaster, add_mapping, load_master
+from oq_data.universes import UniverseEntry, add_entries, load_universes, members_as_of
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "CorporateAction",
+    "DataPaths",
+    "SymbolMaster",
+    "UniverseEntry",
+    "__version__",
+    "add_actions",
+    "add_entries",
+    "add_mapping",
+    "adjust_prices",
+    "build_url",
+    "coverage",
+    "default_root",
+    "download_announcements",
+    "download_bhavcopy",
+    "download_delivery",
+    "download_flows",
+    "download_fno",
+    "get_paths",
+    "list_symbols",
+    "load_actions",
+    "load_master",
+    "load_universes",
+    "members_as_of",
+    "parse_announcements_blob",
+    "parse_bhavcopy_blob",
+    "parse_delivery_blob",
+    "parse_flows_blob",
+    "parse_fno_blob",
+    "prices",
+    "query",
+    "read_announcements",
+    "read_delivery",
+    "read_flows",
+    "read_fno",
+    "read_prices",
+    "resolve_symbol",
+    "sync_range",
+    "universe",
+    "wide_prices",
+    "write_announcements",
+    "write_delivery",
+    "write_eod",
+    "write_flows",
+    "write_fno",
+]

oq_data/announcements.py

@@ -0,0 +1,176 @@
+"""Corporate-announcements feed ingestion.
+
+NSE publishes a rolling JSON feed of corporate announcements at
+``https://www.nseindia.com/api/corporate-announcements?index=equities``.
+Each row carries the announcement timestamp, symbol, broad category, a
+short subject line, and an attachment URL.
+
+The canonical schema we persist is::
+
+    date, symbol, category, subject, attachment
+
+``date`` is the announcement business date (``date``-typed), suitable
+for the same year-partitioned storage layout used by the EOD writers.
+
+Network calls go through the same injectable ``Fetcher`` as the rest of
+the pipeline so the suite stays offline.
+"""
+
+from __future__ import annotations
+
+import io
+import json
+import logging
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import date, timedelta
+
+import pandas as pd
+
+from oq_data.bhavcopy import Fetcher, _default_fetcher
+from oq_data.config import DataPaths, get_paths
+from oq_data.storage import write_partitioned
+
+logger = logging.getLogger(__name__)
+
+_NORMALISED_COLUMNS = ["date", "symbol", "category", "subject", "attachment"]
+
+
+@dataclass(frozen=True, slots=True)
+class AnnouncementsSource:
+    when: date
+    url: str
+    filename: str
+
+
+def build_url(when: date) -> AnnouncementsSource:
+    fname = f"announcements_{when:%Y%m%d}.json"
+    url = (
+        "https://www.nseindia.com/api/corporate-announcements"
+        f"?index=equities&from_date={when:%d-%m-%Y}&to_date={when:%d-%m-%Y}"
+    )
+    return AnnouncementsSource(when=when, url=url, filename=fname)
+
+
+def _pick(row: dict, *keys: str) -> object:
+    for k in keys:
+        if k in row and row[k] not in (None, ""):
+            return row[k]
+    return ""
+
+
+def parse_announcements_blob(blob: bytes, when: date) -> pd.DataFrame:
+    text = blob.decode("utf-8-sig", errors="ignore").lstrip()
+    if text.startswith("[") or text.startswith("{"):
+        data = json.loads(text)
+        rows = data if isinstance(data, list) else data.get("data", data.get("rows", []))
+    else:
+        rows = pd.read_csv(io.BytesIO(blob)).to_dict("records")
+    if not rows:
+        return pd.DataFrame(columns=_NORMALISED_COLUMNS)
+    df = pd.DataFrame(
+        {
+            "date": pd.to_datetime(when),
+            "symbol": [str(_pick(r, "symbol", "Symbol", "SYMBOL")).strip() for r in rows],
+            "category": [
+                str(_pick(r, "category", "Category", "broadcastsubject")).strip() for r in rows
+            ],
+            "subject": [
+                str(_pick(r, "subject", "Subject", "desc", "Description")).strip() for r in rows
+            ],
+            "attachment": [
+                str(_pick(r, "attchmntFile", "attachment", "attachmentUrl")).strip() for r in rows
+            ],
+        }
+    )
+    df = df[df["symbol"] != ""].reset_index(drop=True)
+    return df[_NORMALISED_COLUMNS]
+
+
+def _cache_dir(paths: DataPaths):
+    p = paths.raw / "announcements"
+    p.mkdir(parents=True, exist_ok=True)
+    return p
+
+
+def download_announcements(
+    when: date,
+    paths: DataPaths | None = None,
+    fetcher: Fetcher | None = None,
+    use_cache: bool = True,
+) -> pd.DataFrame:
+    paths = paths or get_paths()
+    paths.ensure()
+    src = build_url(when)
+    cache_path = _cache_dir(paths) / src.filename
+    fetch = fetcher or _default_fetcher
+    if use_cache and cache_path.exists():
+        blob = cache_path.read_bytes()
+    else:
+        blob = fetch(src.url)
+        cache_path.write_bytes(blob)
+    return parse_announcements_blob(blob, when)
+
+
+def write_announcements(df: pd.DataFrame, paths: DataPaths | None = None) -> int:
+    paths = paths or get_paths()
+    paths.ensure()
+    keys = ["date", "symbol", "subject"]
+    return write_partitioned(df, paths.announcements, keys)
+
+
+def read_announcements(
+    symbols: str | Iterable[str] | None = None,
+    start: date | str | None = None,
+    end: date | str | None = None,
+    paths: DataPaths | None = None,
+) -> pd.DataFrame:
+    paths = paths or get_paths()
+    parts = sorted(paths.announcements.glob("year=*/data.parquet"))
+    if not parts:
+        return pd.DataFrame(columns=_NORMALISED_COLUMNS)
+    df = pd.concat([pd.read_parquet(p) for p in parts], ignore_index=True)
+    if symbols is not None:
+        syms = {symbols} if isinstance(symbols, str) else set(symbols)
+        df = df[df["symbol"].isin(syms)]
+    if start is not None:
+        df = df[df["date"] >= pd.to_datetime(start)]
+    if end is not None:
+        df = df[df["date"] <= pd.to_datetime(end)]
+    return df.sort_values(["date", "symbol"]).reset_index(drop=True)
+
+
+def sync_range(
+    start: date,
+    end: date,
+    paths: DataPaths | None = None,
+    fetcher: Fetcher | None = None,
+    on_missing: str = "skip",
+) -> Iterable[date]:
+    if end < start:
+        raise ValueError("end must be >= start")
+    paths = paths or get_paths()
+    paths.ensure()
+    cur = start
+    one_day = timedelta(days=1)
+    while cur <= end:
+        if cur.weekday() < 5:
+            try:
+                download_announcements(cur, paths=paths, fetcher=fetcher)
+                yield cur
+            except Exception as exc:
+                if on_missing == "raise":
+                    raise
+                logger.info("announcements feed unavailable for %s: %s", cur, exc)
+        cur += one_day
+
+
+__all__ = [
+    "AnnouncementsSource",
+    "build_url",
+    "download_announcements",
+    "parse_announcements_blob",
+    "read_announcements",
+    "sync_range",
+    "write_announcements",
+]

oq_data/api.py

@@ -0,0 +1,103 @@
+"""High-level Python API for downstream packages.
+
+This is the surface most users will touch: :func:`prices` for a clean
+adjusted price series, :func:`universe` for a point-in-time membership
+set, and :func:`wide_prices` for a date-indexed wide frame that drops
+straight into :func:`oq_backtest.backtest`.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from datetime import date
+
+import pandas as pd
+
+from oq_data import corporate_actions, storage, symbols, universes
+from oq_data.config import DataPaths, get_paths
+
+
+def prices(
+    symbol: str | Iterable[str],
+    start: date | str | None = None,
+    end: date | str | None = None,
+    adjusted: bool = True,
+    paths: DataPaths | None = None,
+) -> pd.DataFrame:
+    """Read a long-form OHLCV frame for one or many symbols.
+
+    With ``adjusted=True`` (default), splits, bonuses, and dividends are
+    back-adjusted so the returned series is a continuous total-return
+    proxy suitable for backtesting.
+    """
+    paths = paths or get_paths()
+    df = storage.read_prices(symbols=symbol, start=start, end=end, paths=paths)
+    if df.empty or not adjusted:
+        return df
+    actions = corporate_actions.load_actions(paths=paths)
+    if actions.empty:
+        return df
+    return corporate_actions.adjust_prices(df, actions)
+
+
+def wide_prices(
+    universe_symbols: Iterable[str],
+    start: date | str | None = None,
+    end: date | str | None = None,
+    field: str = "close",
+    adjusted: bool = True,
+    paths: DataPaths | None = None,
+) -> pd.DataFrame:
+    """Return a date-indexed wide DataFrame ready for the backtester.
+
+    The output is what :func:`oq_backtest.backtest` consumes as
+    ``prices``: rows are trading dates, columns are symbols, values are
+    the requested field (default ``close``).
+    """
+    syms = list(universe_symbols)
+    if not syms:
+        raise ValueError("universe_symbols must be non-empty")
+    long_df = prices(syms, start=start, end=end, adjusted=adjusted, paths=paths)
+    if long_df.empty:
+        return pd.DataFrame()
+    if field not in long_df.columns:
+        raise KeyError(f"field {field!r} not in {sorted(long_df.columns)}")
+    wide = long_df.pivot_table(index="date", columns="symbol", values=field, aggfunc="last")
+    wide = wide.sort_index()
+    wide.index = pd.DatetimeIndex(wide.index)
+    return wide
+
+
+def universe(
+    index_name: str,
+    as_of: date | str,
+    paths: DataPaths | None = None,
+) -> list[str]:
+    """List the symbols that made up ``index_name`` on ``as_of``."""
+    paths = paths or get_paths()
+    when = pd.to_datetime(as_of).date()
+    members = universes.members_as_of(index_name, when, paths=paths)
+    return members["symbol"].tolist()
+
+
+def resolve_symbol(symbol: str, when: date | str, paths: DataPaths | None = None) -> str:
+    """Translate a current ticker to the symbol used on ``when``."""
+    paths = paths or get_paths()
+    master = symbols.load_master(paths=paths)
+    return master.resolve_as_of(symbol, pd.to_datetime(when).date())
+
+
+def list_symbols(paths: DataPaths | None = None) -> list[str]:
+    """All distinct ``symbol`` values present in the EOD dataset."""
+    paths = paths or get_paths()
+    df = storage.query("SELECT DISTINCT symbol FROM eod ORDER BY symbol", paths=paths)
+    return df["symbol"].tolist() if not df.empty else []
+
+
+__all__ = [
+    "list_symbols",
+    "prices",
+    "resolve_symbol",
+    "universe",
+    "wide_prices",
+]

oq_data/bhavcopy.py

@@ -0,0 +1,272 @@
+"""NSE equity Bhavcopy ingestion.
+
+NSE publishes one zipped CSV per trading day with all listed cash-market
+instruments and their OHLCV. Two formats are in active circulation:
+
+* **Legacy** (until ~July 2020): ``cm{DDMMMYYYY}bhav.csv.zip`` with the
+  classic ``SYMBOL,SERIES,OPEN,HIGH,LOW,CLOSE,LAST,PREVCLOSE,...`` schema.
+* **UDiFF** (from ~July 2020 onward): ``BhavCopy_NSE_CM_0_0_0_{YYYYMMDD}
+  _F_0000.csv.zip`` with the longer ``TradDt,BizDt,Sgmt,Src,FinInstrmTp,
+  FinInstrmId,ISIN,TckrSymb,SctySrs,...`` schema.
+
+This module exposes URL builders, format-detecting parsers, and a single
+:func:`download_bhavcopy` entry point that retries, resumes from cache,
+and returns a normalised :class:`pandas.DataFrame`.
+
+Network calls are isolated behind an injectable ``fetcher`` callable so
+tests run fully offline against the fixtures under ``tests/fixtures``.
+"""
+
+from __future__ import annotations
+
+import io
+import logging
+import time
+import zipfile
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from datetime import date, datetime, timedelta
+from pathlib import Path
+
+import httpx
+import pandas as pd
+
+from oq_data.config import DataPaths, get_paths
+
+logger = logging.getLogger(__name__)
+
+UDIFF_CUTOVER = date(2020, 7, 11)
+USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/124.0 Safari/537.36"
+)
+DEFAULT_HEADERS = {
+    "User-Agent": USER_AGENT,
+    "Accept": "*/*",
+    "Accept-Language": "en-US,en;q=0.9",
+}
+
+Fetcher = Callable[[str], bytes]
+
+_NORMALISED_COLUMNS = [
+    "date",
+    "symbol",
+    "isin",
+    "series",
+    "open",
+    "high",
+    "low",
+    "close",
+    "prev_close",
+    "volume",
+    "value",
+    "trades",
+]
+
+
+@dataclass(frozen=True, slots=True)
+class BhavcopySource:
+    """The full address of a bhavcopy for a given date."""
+
+    when: date
+    url: str
+    filename: str
+    is_udiff: bool
+
+
+def is_udiff_date(when: date) -> bool:
+    """Return True if NSE was publishing the UDiFF schema on ``when``."""
+    return when >= UDIFF_CUTOVER
+
+
+def build_url(when: date) -> BhavcopySource:
+    """Build the canonical NSE archive URL for the bhavcopy on ``when``."""
+    if is_udiff_date(when):
+        fname = f"BhavCopy_NSE_CM_0_0_0_{when:%Y%m%d}_F_0000.csv.zip"
+        url = f"https://nsearchives.nseindia.com/content/cm/{fname}"
+        return BhavcopySource(when=when, url=url, filename=fname, is_udiff=True)
+    fname = f"cm{when.strftime('%d%b%Y').upper()}bhav.csv.zip"
+    url = (
+        "https://nsearchives.nseindia.com/content/historical/EQUITIES/"
+        f"{when:%Y}/{when.strftime('%b').upper()}/{fname}"
+    )
+    return BhavcopySource(when=when, url=url, filename=fname, is_udiff=False)
+
+
+def _default_fetcher(url: str, timeout: float = 30.0, retries: int = 3) -> bytes:
+    """HTTP GET with retry and NSE-friendly headers. Raises on final failure."""
+    last_exc: Exception | None = None
+    for attempt in range(1, retries + 1):
+        try:
+            with httpx.Client(
+                headers=DEFAULT_HEADERS, follow_redirects=True, timeout=timeout
+            ) as client:
+                resp = client.get(url)
+                resp.raise_for_status()
+                return resp.content
+        except httpx.HTTPError as exc:
+            last_exc = exc
+            logger.warning("bhavcopy fetch failed (attempt %d/%d): %s", attempt, retries, exc)
+            time.sleep(min(2**attempt, 8))
+    raise RuntimeError(f"failed to fetch {url} after {retries} attempts") from last_exc
+
+
+def _read_csv_from_zip(blob: bytes) -> pd.DataFrame:
+    with zipfile.ZipFile(io.BytesIO(blob)) as zf:
+        names = [n for n in zf.namelist() if n.lower().endswith(".csv")]
+        if not names:
+            raise ValueError("zip archive contains no .csv member")
+        with zf.open(names[0]) as fh:
+            return pd.read_csv(fh)
+
+
+def _normalise_legacy(raw: pd.DataFrame, when: date) -> pd.DataFrame:
+    raw = raw.rename(columns=lambda c: c.strip().upper())
+    df = pd.DataFrame(
+        {
+            "date": pd.to_datetime(when),
+            "symbol": raw["SYMBOL"].astype(str).str.strip(),
+            "isin": raw.get("ISIN", pd.Series([pd.NA] * len(raw))).astype("string").str.strip(),
+            "series": raw["SERIES"].astype(str).str.strip(),
+            "open": pd.to_numeric(raw["OPEN"], errors="coerce"),
+            "high": pd.to_numeric(raw["HIGH"], errors="coerce"),
+            "low": pd.to_numeric(raw["LOW"], errors="coerce"),
+            "close": pd.to_numeric(raw["CLOSE"], errors="coerce"),
+            "prev_close": pd.to_numeric(raw["PREVCLOSE"], errors="coerce"),
+            "volume": pd.to_numeric(raw["TOTTRDQTY"], errors="coerce").astype("Int64"),
+            "value": pd.to_numeric(raw["TOTTRDVAL"], errors="coerce"),
+            "trades": pd.to_numeric(raw.get("TOTALTRADES", pd.NA), errors="coerce").astype("Int64"),
+        }
+    )
+    return df
+
+
+def _normalise_udiff(raw: pd.DataFrame, when: date) -> pd.DataFrame:
+    raw = raw.rename(columns=lambda c: c.strip())
+    df = pd.DataFrame(
+        {
+            "date": pd.to_datetime(when),
+            "symbol": raw["TckrSymb"].astype(str).str.strip(),
+            "isin": raw["ISIN"].astype("string").str.strip(),
+            "series": raw["SctySrs"].astype(str).str.strip(),
+            "open": pd.to_numeric(raw["OpnPric"], errors="coerce"),
+            "high": pd.to_numeric(raw["HghPric"], errors="coerce"),
+            "low": pd.to_numeric(raw["LwPric"], errors="coerce"),
+            "close": pd.to_numeric(raw["ClsPric"], errors="coerce"),
+            "prev_close": pd.to_numeric(raw["PrvsClsgPric"], errors="coerce"),
+            "volume": pd.to_numeric(raw["TtlTradgVol"], errors="coerce").astype("Int64"),
+            "value": pd.to_numeric(raw["TtlTrfVal"], errors="coerce"),
+            "trades": pd.to_numeric(raw["TtlNbOfTxsExctd"], errors="coerce").astype("Int64"),
+        }
+    )
+    fininstrm = raw.get("FinInstrmTp")
+    if fininstrm is not None:
+        df = df[fininstrm.astype(str).str.upper().isin({"STK", "EQ"})].reset_index(drop=True)
+    return df
+
+
+def parse_bhavcopy_blob(blob: bytes, when: date) -> pd.DataFrame:
+    """Parse a downloaded bhavcopy zip (or raw csv) into the canonical schema.
+
+    The schema returned, regardless of input format, is exactly:
+    ``date, symbol, isin, series, open, high, low, close, prev_close,
+    volume, value, trades``.
+    """
+    raw = _read_csv_from_zip(blob) if blob[:2] == b"PK" else pd.read_csv(io.BytesIO(blob))
+    upper_cols = {c.strip().upper() for c in raw.columns}
+    is_udiff = "TCKRSYMB" in upper_cols
+    df = _normalise_udiff(raw, when) if is_udiff else _normalise_legacy(raw, when)
+    return df[_NORMALISED_COLUMNS]
+
+
+def download_bhavcopy(
+    when: date,
+    paths: DataPaths | None = None,
+    fetcher: Fetcher | None = None,
+    use_cache: bool = True,
+) -> pd.DataFrame:
+    """Download and parse the NSE equity bhavcopy for ``when``.
+
+    The raw zip is cached under ``paths.bhavcopy`` so reruns are offline.
+    Pass ``fetcher`` to substitute a callable (used by tests) or to swap
+    in your own retry/auth wrapper.
+    """
+    paths = paths or get_paths()
+    paths.ensure()
+    src = build_url(when)
+    cache_path = paths.bhavcopy / src.filename
+    fetch = fetcher or _default_fetcher
+
+    if use_cache and cache_path.exists():
+        blob = cache_path.read_bytes()
+    else:
+        blob = fetch(src.url)
+        cache_path.write_bytes(blob)
+    return parse_bhavcopy_blob(blob, when)
+
+
+def sync_range(
+    start: date,
+    end: date,
+    paths: DataPaths | None = None,
+    fetcher: Fetcher | None = None,
+    on_missing: str = "skip",
+) -> Iterable[date]:
+    """Download every available bhavcopy in ``[start, end]`` inclusive.
+
+    Weekends are always skipped. NSE holidays show up as 404s; with
+    ``on_missing='skip'`` (default) they are silently passed over. With
+    ``on_missing='raise'`` the first 404 aborts the run.
+    Yields the dates that were successfully ingested.
+    """
+    if end < start:
+        raise ValueError("end must be >= start")
+    paths = paths or get_paths()
+    paths.ensure()
+    cur = start
+    one_day = timedelta(days=1)
+    while cur <= end:
+        if cur.weekday() < 5:
+            try:
+                download_bhavcopy(cur, paths=paths, fetcher=fetcher)
+                yield cur
+            except Exception as exc:
+                if on_missing == "raise":
+                    raise
+                logger.info("bhavcopy unavailable for %s: %s", cur, exc)
+        cur += one_day
+
+
+def parse_filename_date(filename: str) -> date:
+    """Recover the trading date from a cached bhavcopy filename."""
+    if filename.startswith("BhavCopy_NSE_CM_"):
+        token = filename.split("_")[6]
+        return datetime.strptime(token, "%Y%m%d").date()
+    if filename.startswith("cm") and filename.endswith("bhav.csv.zip"):
+        token = filename[2:-12]
+        return datetime.strptime(token, "%d%b%Y").date()
+    raise ValueError(f"unrecognised bhavcopy filename: {filename}")
+
+
+def iter_cached(paths: DataPaths) -> Iterable[Path]:
+    """Yield every cached bhavcopy archive under ``paths.bhavcopy``."""
+    if not paths.bhavcopy.exists():
+        return
+    for entry in sorted(paths.bhavcopy.iterdir()):
+        if entry.is_file() and entry.suffix == ".zip":
+            yield entry
+
+
+__all__ = [
+    "DEFAULT_HEADERS",
+    "UDIFF_CUTOVER",
+    "BhavcopySource",
+    "Fetcher",
+    "build_url",
+    "download_bhavcopy",
+    "is_udiff_date",
+    "iter_cached",
+    "parse_bhavcopy_blob",
+    "parse_filename_date",
+    "sync_range",
+]