tradekart-backtester-data 0.1.0__tar.gz

tradekart_backtester_data-0.1.0/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: tradekart-backtester-data
+Version: 0.1.0
+Summary: Daily NSE price data fetcher and storage for TradeKart's backtester
+Author-email: Vikalp Varshney <vikalp@tradekart.gmail.com>
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: tradekart-cloud
+Requires-Dist: yfinance>=0.2.40
+Requires-Dist: pandas>=2.2.0
+Requires-Dist: pyarrow>=16.0.0
+Requires-Dist: requests>=2.31.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: moto[s3]>=5.0.0; extra == "dev"
+Requires-Dist: ruff>=0.6.0; extra == "dev"
+
+# README #
+
+This README would normally document whatever steps are necessary to get your application up and running.
+
+### What is this repository for? ###
+
+* Quick summary
+* Version
+* [Learn Markdown](https://bitbucket.org/tutorials/markdowndemo)
+
+### How do I get set up? ###
+
+* Summary of set up
+* Configuration
+* Dependencies
+* Database configuration
+* How to run tests
+* Deployment instructions
+
+### Contribution guidelines ###
+
+* Writing tests
+* Code review
+* Other guidelines
+
+### Who do I talk to? ###
+
+* Repo owner or admin
+* Other community or team contact

tradekart_backtester_data-0.1.0/README.md

@@ -0,0 +1,29 @@
+# README #
+
+This README would normally document whatever steps are necessary to get your application up and running.
+
+### What is this repository for? ###
+
+* Quick summary
+* Version
+* [Learn Markdown](https://bitbucket.org/tutorials/markdowndemo)
+
+### How do I get set up? ###
+
+* Summary of set up
+* Configuration
+* Dependencies
+* Database configuration
+* How to run tests
+* Deployment instructions
+
+### Contribution guidelines ###
+
+* Writing tests
+* Code review
+* Other guidelines
+
+### Who do I talk to? ###
+
+* Repo owner or admin
+* Other community or team contact

tradekart_backtester_data-0.1.0/data/backfill.py

@@ -0,0 +1,33 @@
+from datetime import date, timedelta
+
+import yfinance as yf
+from cloud import AWS
+
+from .storage import S3ParquetStore
+from .symbol_cache import get_nifty100_symbols
+
+YEARS_OF_HISTORY = 3
+
+
+def backfill():
+    aws = AWS()
+    store = S3ParquetStore(aws=aws)
+    symbols = get_nifty100_symbols(aws)
+
+    start = date.today() - timedelta(days=365 * YEARS_OF_HISTORY)
+    print(f"Backfilling {len(symbols)} symbols from {start} to today...")
+
+    for symbol in symbols:
+        data = yf.download(symbol, start=start, end=date.today(), progress=False)
+        if data.empty:
+            print(f"  {symbol}: no data, skipping")
+            continue
+        data.index.name = "date"
+        store.save(symbol, data[["Open", "High", "Low", "Close", "Volume"]])
+        print(f"  {symbol}: saved {len(data)} rows")
+
+    print("Backfill complete.")
+
+
+if __name__ == "__main__":
+    backfill()

tradekart_backtester_data-0.1.0/data/constants.py

@@ -0,0 +1,2 @@
+NIFTY100_URL = "https://archives.nseindia.com/content/indices/ind_nifty100list.csv"
+BUCKET = "tradekart-strategies-data"

tradekart_backtester_data-0.1.0/data/fetch.py

@@ -0,0 +1,18 @@
+from datetime import date, timedelta
+
+import pandas as pd
+import yfinance as yf
+
+
+class YahooFinanceSource:
+    def fetch_latest_candle(self, symbol: str, for_date: date) -> pd.DataFrame | None:
+        data = yf.download(
+            symbol,
+            start=for_date,
+            end=for_date + timedelta(days=1),
+            progress=False,
+        )
+        if data.empty:
+            return None
+        data.index.name = "date"
+        return data[["Open", "High", "Low", "Close", "Volume"]]

tradekart_backtester_data-0.1.0/data/interfaces.py

@@ -0,0 +1,21 @@
+from datetime import date
+from typing import Protocol
+
+import pandas as pd
+
+
+class PriceStore(Protocol):
+    def load(self, symbol: str) -> pd.DataFrame | None:
+        """
+        Return the symbol's full stored history, or None if nothing stored yet.
+        """
+
+    def save(self, symbol: str, prices: pd.DataFrame) -> None:
+        """
+        Overwrite the symbol's stored history with this DataFrame
+        """
+
+
+class DataSource(Protocol):
+    def fetch_latest_candle(self, symbol: str, for_date: date) -> pd.DataFrame | None:
+        """Return one day's OHLCV for this symbol, or None if unavailable."""

tradekart_backtester_data-0.1.0/data/lambda_handler.py

@@ -0,0 +1,11 @@
+import json
+
+from .pipeline import run
+
+
+def handler(event, context):
+    result = run()
+    return {
+        "statusCode": 200,
+        "body": json.dumps(result),
+    }

tradekart_backtester_data-0.1.0/data/loader.py

@@ -0,0 +1,19 @@
+import pandas as pd
+from cloud import AWS
+
+from .interfaces import PriceStore
+from .storage import S3ParquetStore
+from .symbol_cache import get_nifty100_symbols
+
+
+def load_all_prices(store: PriceStore | None = None, aws: AWS | None = None):
+    aws = aws or AWS()
+    store = store or S3ParquetStore(aws)
+    symbols = get_nifty100_symbols(aws)
+
+    closes: dict[str, pd.Series] = {}
+    for symbol in symbols:
+        data = store.load(symbol)
+        if data is not None:
+            closes[symbol] = data["Close"]
+    return pd.DataFrame(closes).sort_index()

tradekart_backtester_data-0.1.0/data/pipeline.py

@@ -0,0 +1,40 @@
+from datetime import date, timedelta
+
+import pandas as pd
+from cloud import AWS
+
+from .fetch import YahooFinanceSource
+from .interfaces import DataSource, PriceStore
+from .storage import S3ParquetStore
+from .symbol_cache import get_nifty100_symbols
+
+
+def run(
+    store: PriceStore | None = None,
+    data_source: DataSource | None = None,
+    aws: AWS | None = None,
+    for_date: date | None = None,
+) -> dict:
+    aws = aws or AWS()
+    store = store or S3ParquetStore(aws)
+    data_source = data_source or YahooFinanceSource()
+    target_date = for_date or date.today() - timedelta(days=1)
+    symbols = get_nifty100_symbols(aws)
+    fetched, skipped = 0, 0
+
+    for symbol in symbols:
+        candle = data_source.fetch_latest_candle(symbol, target_date)
+        if candle is None:
+            skipped += 1
+            continue
+        existing = store.load(symbol)
+        combined = pd.concat([existing, candle]) if existing is not None else candle
+        combined = combined[~combined.index.duplicated(keep="last")].sort_index()
+        store.save(symbol, combined)
+        fetched += 1
+    return {
+        "fetched": fetched,
+        "skipped": skipped,
+        "total": len(symbols),
+        "date": str(target_date),
+    }

tradekart_backtester_data-0.1.0/data/storage.py

@@ -0,0 +1,44 @@
+import io
+from pathlib import Path
+
+import pandas as pd
+from cloud import AWS
+from cloud.exceptions import TradeKartCloudError
+from cloud.storage.s3 import DownloadFileRequest, UploadBytesRequest
+
+from .constants import BUCKET
+
+
+class S3ParquetStore:
+    def __init__(self, aws: AWS | None = None, bucket: str = BUCKET):
+        self._aws = aws or AWS()
+        self._bucket = bucket
+
+    def _object_key(self, symbol: str) -> str:
+        return f"prices/{symbol}.parquet"
+
+    def load(self, symbol: str) -> pd.DataFrame | None:
+        tmp_path = Path(f"/tmp/{symbol}.parquet")
+        try:
+            self._aws.s3.download_file(
+                DownloadFileRequest(
+                    bucket_name=self._bucket,
+                    object_key=self._object_key(symbol),
+                    destination_path=tmp_path,
+                )
+            )
+        except TradeKartCloudError:
+            return None
+        return pd.read_parquet(tmp_path)
+
+    def save(self, symbol: str, prices: pd.DataFrame) -> None:
+        buffer = io.BytesIO()
+        prices.to_parquet(buffer)
+        self._aws.s3.upload_bytes(
+            UploadBytesRequest(
+                bucket_name=self._bucket,
+                object_key=self._object_key(symbol),
+                body=buffer.getvalue(),
+                content_type="application/octet-stream",
+            )
+        )

tradekart_backtester_data-0.1.0/data/symbol_cache.py

@@ -0,0 +1,57 @@
+import json
+from datetime import date, timedelta
+from pathlib import Path
+
+from cloud import AWS
+from cloud.exceptions import TradeKartCloudError
+from cloud.storage.s3 import DownloadFileRequest, UploadBytesRequest
+
+from .constants import BUCKET
+from .symbols import fetch_nifty100_symbols
+
+CACHE_KEY = "symbols/nifty100.json"
+MAX_AGE_DAYS = 180
+
+
+def _load_cache(aws: AWS) -> dict | None:
+    tmp_path = Path("/tmp/nifty100_cache.json")
+    try:
+        aws.s3.download_file(
+            DownloadFileRequest(
+                bucket_name=BUCKET, object_key=CACHE_KEY, destination_path=tmp_path
+            )
+        )
+    except TradeKartCloudError:
+        return None
+    return json.loads(tmp_path.read_text())
+
+
+def _save_cache(aws: AWS, symbols: list[str]) -> None:
+    payload = json.dumps(
+        {"fetched_on": date.today().isoformat(), "symbols": symbols}
+    ).encode()
+    aws.s3.upload_bytes(
+        UploadBytesRequest(
+            bucket_name=BUCKET,
+            object_key=CACHE_KEY,
+            body=payload,
+            content_type="application/json",
+        )
+    )
+
+
+def get_nifty100_symbols(aws: AWS) -> list[str]:
+    cached = _load_cache(aws)
+    if cached and date.today() - date.fromisoformat(cached["fetched_on"]) < timedelta(
+        days=MAX_AGE_DAYS
+    ):
+        return cached["symbols"]
+
+    try:
+        symbols = fetch_nifty100_symbols()
+    except Exception:
+        if cached:
+            return cached["symbols"]
+        raise
+    _save_cache(aws, symbols)
+    return symbols

tradekart_backtester_data-0.1.0/data/symbols.py

@@ -0,0 +1,14 @@
+import csv
+import io
+
+import requests
+
+from data.constants import NIFTY100_URL
+
+
+def fetch_nifty100_symbols() -> list[str]:
+    headers = {"User-Agent": "Mozilla/5.0"}
+    response = requests.get(NIFTY100_URL, headers=headers, timeout=10)
+    response.raise_for_status()
+    reader = csv.DictReader(io.StringIO(response.text))
+    return [f"{row['Symbol'].strip()}.NS" for row in reader]

tradekart_backtester_data-0.1.0/data/version.py

@@ -0,0 +1,4 @@
+"""Version info."""
+
+__version_info__ = (0, 1, 0)
+__version__ = ".".join(str(v) for v in __version_info__)

tradekart_backtester_data-0.1.0/pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+requires = ["setuptools>=69.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tradekart-backtester-data"
+dynamic = ["version"]
+authors = [{ name = "Vikalp Varshney", email = "vikalp@tradekart.gmail.com" }]
+description = "Daily NSE price data fetcher and storage for TradeKart's backtester"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "tradekart-cloud",
+    "yfinance>=0.2.40",
+    "pandas>=2.2.0",
+    "pyarrow>=16.0.0",
+    "requests>=2.31.0",
+]
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["data*"]
+namespaces = false
+
+[tool.setuptools.dynamic]
+version = { attr = "data.version.__version__" }
+
+[project.optional-dependencies]
+dev = ["pytest>=8.0.0", "moto[s3]>=5.0.0", "ruff>=0.6.0"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff.lint]
+ignore = [
+    "DTZ011", # date.today() is fine for day-level cache/batch checks - no precise moment-in-time matters here
+]

tradekart_backtester_data-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tradekart_backtester_data-0.1.0/tests/test_loader.py

@@ -0,0 +1,31 @@
+import pandas as pd
+
+from data.loader import load_all_prices
+
+
+class FakeStore:
+    def __init__(self, data: dict[str, pd.DataFrame]):
+        self._data = data
+
+    def load(self, symbol):
+        return self._data.get(symbol)
+
+    def save(self, symbol, prices):
+        self._data[symbol] = prices
+
+
+def test_load_all_prices_combines_symbols(monkeypatch):
+    fake_data = {
+        "RELIANCE.NS": pd.DataFrame(
+            {"Close": [100.0]}, index=pd.to_datetime(["2024-01-01"])
+        ),
+        "TCS.NS": pd.DataFrame(
+            {"Close": [200.0]}, index=pd.to_datetime(["2024-01-01"])
+        ),
+    }
+    monkeypatch.setattr(
+        "data.loader.get_nifty100_symbols", lambda aws: list(fake_data.keys())
+    )
+    wide = load_all_prices(store=FakeStore(fake_data))
+    assert list(wide.columns) == list(fake_data.keys())
+    assert wide.loc["2024-01-01", "RELIANCE.NS"] == 100.0

tradekart_backtester_data-0.1.0/tests/test_pipeline.py

@@ -0,0 +1,31 @@
+from datetime import date
+
+import pandas as pd
+
+from data.pipeline import run
+
+
+class FakeStore:
+    def __init__(self):
+        self._data: dict[str, pd.DataFrame] = {}
+
+    def load(self, symbol):
+        return self._data.get(symbol)
+
+    def save(self, symbol, prices):
+        self._data[symbol] = prices
+
+
+class FakeSource:
+    def fetch_latest_candle(self, symbol, for_date):
+        return pd.DataFrame({"Close": [100.0]}, index=pd.to_datetime([for_date]))
+
+
+def test_run_fetches_and_saves_each_symbol(monkeypatch):
+    monkeypatch.setattr(
+        "data.pipeline.get_nifty100_symbols", lambda aws: ["RELIANCE.NS", "TCS.NS"]
+    )
+    store = FakeStore()
+    result = run(store=store, data_source=FakeSource(), for_date=date(2024, 1, 2))
+    assert result["fetched"] == 2
+    assert result["skipped"] == 0

tradekart_backtester_data-0.1.0/tests/test_storage.py

@@ -0,0 +1,15 @@
+import pandas as pd
+
+
+def test_save_then_load_round_trip(s3_store):
+    prices = pd.DataFrame(
+        {"Close": [100.0, 101.5]}, index=pd.date_range(start="2026-10-07", periods=2)
+    )
+    s3_store.save("RELIANCE.NS", prices)
+    loaded = s3_store.load("RELIANCE.NS")
+    assert loaded is not None
+    assert loaded["Close"].tolist() == [100.0, 101.5]
+
+
+def test_load_missing_symbol_returns_none(s3_store):
+    assert s3_store.load("DOES_NOT_EXIST.NS") is None

tradekart_backtester_data-0.1.0/tests/test_symbol_cache.py

@@ -0,0 +1,51 @@
+import pytest
+
+from data import symbol_cache
+
+
+def test_fetches_live_when_no_cache(monkeypatch, aws, bucket):
+    monkeypatch.setattr(symbol_cache, "BUCKET", bucket)
+    monkeypatch.setattr(
+        symbol_cache,
+        "fetch_nifty100_symbols",
+        lambda: ["RELIANCE.NS", "TCS.NS", "INFY.NS"],
+    )
+    assert symbol_cache.get_nifty100_symbols(aws) == [
+        "RELIANCE.NS",
+        "TCS.NS",
+        "INFY.NS",
+    ]
+
+
+def test_uses_cache_when_fresh(monkeypatch, aws, bucket):
+    monkeypatch.setattr(symbol_cache, "BUCKET", bucket)
+    symbol_cache._save_cache(aws, ["CACHED.NS"])
+
+    def fail_if_called():
+        raise AssertionError("fetch_nifty100_symbols should not be called")
+
+    monkeypatch.setattr(symbol_cache, "fetch_nifty100_symbols", fail_if_called)
+    assert symbol_cache.get_nifty100_symbols(aws) == ["CACHED.NS"]
+
+
+def test_falls_back_to_stale_cache_if_nse_fails(monkeypatch, aws, bucket):
+    monkeypatch.setattr(symbol_cache, "BUCKET", bucket)
+    symbol_cache._save_cache(aws, ["STALE.NS"])
+    monkeypatch.setattr(symbol_cache, "MAX_AGE_DAYS", -1)
+    monkeypatch.setattr(
+        symbol_cache,
+        "fetch_nifty100_symbols",
+        lambda: (_ for _ in ()).throw(RuntimeError("NSE down")),
+    )
+    assert symbol_cache.get_nifty100_symbols(aws) == ["STALE.NS"]
+
+
+def test_raises_if_no_cache_and_nse_fails(monkeypatch, aws, bucket):
+    monkeypatch.setattr(symbol_cache, "BUCKET", bucket)
+    monkeypatch.setattr(
+        symbol_cache,
+        "fetch_nifty100_symbols",
+        lambda: (_ for _ in ()).throw(RuntimeError("NSE down")),
+    )
+    with pytest.raises(RuntimeError):
+        symbol_cache.get_nifty100_symbols(aws)

tradekart_backtester_data-0.1.0/tradekart_backtester_data.egg-info/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: tradekart-backtester-data
+Version: 0.1.0
+Summary: Daily NSE price data fetcher and storage for TradeKart's backtester
+Author-email: Vikalp Varshney <vikalp@tradekart.gmail.com>
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: tradekart-cloud
+Requires-Dist: yfinance>=0.2.40
+Requires-Dist: pandas>=2.2.0
+Requires-Dist: pyarrow>=16.0.0
+Requires-Dist: requests>=2.31.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: moto[s3]>=5.0.0; extra == "dev"
+Requires-Dist: ruff>=0.6.0; extra == "dev"
+
+# README #
+
+This README would normally document whatever steps are necessary to get your application up and running.
+
+### What is this repository for? ###
+
+* Quick summary
+* Version
+* [Learn Markdown](https://bitbucket.org/tutorials/markdowndemo)
+
+### How do I get set up? ###
+
+* Summary of set up
+* Configuration
+* Dependencies
+* Database configuration
+* How to run tests
+* Deployment instructions
+
+### Contribution guidelines ###
+
+* Writing tests
+* Code review
+* Other guidelines
+
+### Who do I talk to? ###
+
+* Repo owner or admin
+* Other community or team contact

tradekart_backtester_data-0.1.0/tradekart_backtester_data.egg-info/SOURCES.txt

@@ -0,0 +1,23 @@
+README.md
+pyproject.toml
+data/__init__.py
+data/backfill.py
+data/constants.py
+data/fetch.py
+data/interfaces.py
+data/lambda_handler.py
+data/loader.py
+data/pipeline.py
+data/storage.py
+data/symbol_cache.py
+data/symbols.py
+data/version.py
+tests/test_loader.py
+tests/test_pipeline.py
+tests/test_storage.py
+tests/test_symbol_cache.py
+tradekart_backtester_data.egg-info/PKG-INFO
+tradekart_backtester_data.egg-info/SOURCES.txt
+tradekart_backtester_data.egg-info/dependency_links.txt
+tradekart_backtester_data.egg-info/requires.txt
+tradekart_backtester_data.egg-info/top_level.txt

tradekart_backtester_data-0.1.0/tradekart_backtester_data.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

tradekart_backtester_data-0.1.0/tradekart_backtester_data.egg-info/requires.txt

@@ -0,0 +1,10 @@
+tradekart-cloud
+yfinance>=0.2.40
+pandas>=2.2.0
+pyarrow>=16.0.0
+requests>=2.31.0
+
+[dev]
+pytest>=8.0.0
+moto[s3]>=5.0.0
+ruff>=0.6.0

tradekart_backtester_data-0.1.0/tradekart_backtester_data.egg-info/top_level.txt

@@ -0,0 +1 @@
+data