narf 0.1.0__tar.gz

narf-0.1.0/MANIFEST.in

@@ -0,0 +1,3 @@
+include README.md
+recursive-include narf *.pyi
+

narf-0.1.0/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.4
+Name: narf
+Version: 0.1.0
+Summary: Market data loader library for cryptocurrency exchanges
+Project-URL: Homepage, https://github.com/numan-narf/narf-market-data
+Project-URL: Repository, https://github.com/numan-narf/narf-market-data
+Project-URL: Issues, https://github.com/numan-narf/narf-market-data/issues
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: build>=1.3.0
+Requires-Dist: matplotlib>=3.10.7
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: pandas-stubs>=2.3.2.250926
+Requires-Dist: requests>=2.32.5
+Requires-Dist: twine>=6.2.0
+Requires-Dist: types-requests>=2.32.4.20250913

narf-0.1.0/narf/__init__.py

@@ -0,0 +1,4 @@
+"""Narf - Market data loader library for cryptocurrency exchanges."""
+
+__version__ = "0.1.0"
+

narf-0.1.0/narf/data/__init__.py

@@ -0,0 +1,4 @@
+from .loaders.binance_vision import binance
+
+
+__all__ = ["binance"]

narf-0.1.0/narf/data/generate.py

@@ -0,0 +1,141 @@
+import ast
+import os
+from dataclasses import dataclass
+from typing import Optional, Dict
+
+
+@dataclass
+class NodeSpec:
+    children: Optional[Dict[str, "NodeSpec"]] = None
+    loader: Optional[type] = None
+
+
+def generate_pyi(
+    spec: NodeSpec,
+    root_class_name: str,
+    root_var_name: str,
+    package_name: str,
+    loader_import: str,
+    loader_class_name: str = "LoaderType",
+):
+    """
+    Generates:
+
+        narf/data/loaders/<package_name>/__init__.pyi
+
+    The loader class name is taken from `loader_class_name`.
+    """
+
+    module = ast.Module(body=[], type_ignores=[])
+
+    # ------------------------------------------------------------
+    # Import loader dynamically under unified name LoaderType
+    # ------------------------------------------------------------
+    # Example:
+    # from narf.data.loaders.binance_vision.loader import BinanceVisionLoader as LoaderType
+    import_stmt = f"from {loader_import} import {loader_class_name} as LoaderType"
+    module.body.extend(ast.parse(import_stmt).body)
+
+    class_defs: list[ast.ClassDef] = []
+
+    # ------------------------------------------------------------
+    # Class builder
+    # ------------------------------------------------------------
+    def make_class(name: str, fields: Dict[str, str], bases: list[str]):
+        return ast.ClassDef(
+            name=name,
+            bases=[ast.Name(id=b, ctx=ast.Load()) for b in bases],
+            keywords=[],
+            decorator_list=[],
+            body=[
+                ast.AnnAssign(
+                    target=ast.Name(id=field, ctx=ast.Store()),
+                    annotation=ast.Name(id=typ, ctx=ast.Load()),
+                    value=None,
+                    simple=1,
+                )
+                for field, typ in fields.items()
+            ] or [ast.Pass()],
+        )
+
+    # ------------------------------------------------------------
+    # DFS
+    # ------------------------------------------------------------
+    def visit(node: NodeSpec, name: str):
+        bases = []
+
+        # This node also behaves as a loader
+        if node.loader:
+            bases.append("LoaderType")
+
+        fields: Dict[str, str] = {}
+
+        if node.children:
+            for ch_name, ch_spec in node.children.items():
+
+                # child namespace OR non-leaf loader
+                if ch_spec.children or ch_spec.loader:
+                    child_class_name = name + ch_name.capitalize()
+                    fields[ch_name] = child_class_name
+                    visit(ch_spec, child_class_name)
+
+                else:
+                    # pure leaf loader
+                    fields[ch_name] = "LoaderType"
+
+        # Always create a class for this node
+        class_defs.append(make_class(name, fields, bases))
+
+    # Start from root class
+    visit(spec, root_class_name)
+
+    # ------------------------------------------------------------
+    # Add classes
+    # ------------------------------------------------------------
+    module.body.extend(class_defs)
+
+    # ------------------------------------------------------------
+    # Add final variable: <root_var_name>: <root_class_name>
+    # ------------------------------------------------------------
+    module.body.extend([
+        ast.AnnAssign(
+            target=ast.Name(id=root_var_name, ctx=ast.Store()),
+            annotation=ast.Name(id=root_class_name, ctx=ast.Load()),
+            value=None,
+            simple=1,
+        ),
+        # Keep your dummy function (unchanged)
+        ast.FunctionDef(
+            name="generate_pyi",
+            args=ast.arguments(
+                posonlyargs=[],
+                args=[],
+                vararg=None,
+                kwonlyargs=[],
+                kw_defaults=[],
+                kwarg=None,
+                defaults=[],
+            ),
+            body=[ast.Pass()],
+            decorator_list=[],
+            returns=None,
+        )
+    ])
+
+    # ------------------------------------------------------------
+    # Write file to:
+    #     narf/data/loaders/<package_name>/__init__.pyi
+    # ------------------------------------------------------------
+    ast.fix_missing_locations(module)
+    source = ast.unparse(module)
+
+    this_file = os.path.abspath(__file__)
+    this_dir = os.path.dirname(this_file)
+    out_path = os.path.join(this_dir, "loaders", package_name, "__init__.pyi")
+
+    os.makedirs(os.path.dirname(out_path), exist_ok=True)
+
+    with open(out_path, "w", encoding="utf-8") as f:
+        f.write(source)
+
+    return out_path

narf-0.1.0/narf/data/loaders/base_loader.py

@@ -0,0 +1,94 @@
+import abc
+from datetime import datetime
+from typing import Optional
+from narf.data.loaders.base_spec import NodeSpec
+
+class Loader(abc.ABC):
+    BASE_URL: str
+    URL_PATTERNS: list[tuple[tuple[str, ...], str]]
+
+    def __init__(self, path: tuple[str, ...]):
+        self.__path = path
+
+    def get_parsed_path(self):
+        from narf.data.loaders.binance_vision.spec import binance_vision_spec
+        return extract_semantic_path(binance_vision_spec, self.__path)
+
+    def _build_url(self, **kwargs):
+        from narf.data.loaders.binance_vision.spec import binance_vision_spec
+        template, semantic = select_pattern(self.URL_PATTERNS, self.__path, binance_vision_spec)
+
+        ctx = {
+            "base": self.BASE_URL,
+            **kwargs,
+            **semantic,
+        }
+
+        return template.format(**ctx)
+
+    def load(self, date: str, start: datetime, end: Optional[datetime] = None, interval: str = "1m") -> None:
+        print("Loading", self.__path, date, interval, start, end)
+
+
+def extract_semantic_path(spec: NodeSpec, path: tuple[str, ...]):
+    """
+    Returns a dict mapping semantic key -> actual fragment.
+    Example:
+        ("futures","um","klines") →
+        {"market":"futures","margination":"um","datatype":"klines"}
+    """
+    mapping = {}
+    node = spec
+    i = 0
+
+    while i < len(path):
+        fragment = path[i]
+        if not node.children or fragment not in node.children:
+            raise ValueError("Invalid path fragment: " + fragment)
+
+        child = node.children[fragment]
+
+        # assign semantic key
+        mapping[node.key] = fragment
+
+        node = child
+        i += 1
+
+    return mapping
+
+
+def match_pattern(pattern, path, semantic_keys):
+    """
+    pattern: ("futures","<margination>","<datatype>")
+    path: ("futures","um","klines")
+    semantic_keys: ["market","margination","datatype"]
+    """
+
+    if len(pattern) != len(path):
+        return False
+
+    for element, frag, key in zip(pattern, path, semantic_keys):
+
+        if element.startswith("<") and element.endswith(">"):
+            # semantic placeholder — must match the same semantic key
+            expected_key = element[1:-1]
+            if expected_key != key:
+                return False
+
+        else:
+            # literal element — must match fragment exactly
+            if element != frag:
+                return False
+            
+    return True
+
+
+def select_pattern(patterns: list[tuple[tuple[str, ...], str]], path: tuple[str, ...], spec: NodeSpec) -> tuple[str, dict[str, str]]:
+    semantic = extract_semantic_path(spec, path)
+    semantic_keys = list(semantic.keys())  # ["market","margination","datatype"]
+
+    for pattern, template in patterns:
+        if match_pattern(pattern, path, semantic_keys):
+            return template, semantic
+
+    raise ValueError(f"No pattern matches path: {path}")

narf-0.1.0/narf/data/loaders/base_spec.py

@@ -0,0 +1,44 @@
+from dataclasses import dataclass
+from typing import Optional
+
+@dataclass
+class NodeSpec:
+    key: str = ""
+    children: Optional[dict[str, "NodeSpec"]] = None
+    loader: Optional[type] = None
+
+
+class Namespace:
+    def __init__(self, **children):
+        for name, val in children.items():
+            setattr(self, name, val)
+
+
+def build(spec: NodeSpec, path: tuple[str, ...] = ()) -> object:
+    # Case A: namespace
+    if spec.children:
+        # build children
+        attrs = {
+            name: build(sub, path + (name,))
+            for name, sub in spec.children.items()
+        }
+
+        # create namespace object
+        ns = Namespace()
+        for k, v in attrs.items():
+            setattr(ns, k, v)
+
+        # also attach loader behavior
+        if spec.loader:
+            loader = spec.loader(path)
+            # copy loader methods/attrs into namespace
+            ns.load = loader.load
+            ns.__loader__ = loader  # optional: keep reference to real loader
+
+        return ns
+
+    # Case B: pure leaf
+    if spec.loader:
+        return spec.loader(path)
+
+    raise ValueError("NodeSpec must have children or leaf")

narf-0.1.0/narf/data/loaders/binance/__init__.pyi

@@ -0,0 +1,47 @@
+from narf.data.loaders.binance_vision.loader import BinanceVisionLoader as LoaderType
+
+class BinanceVisionFuturesUmKlines(LoaderType):
+    pass
+
+class BinanceVisionFuturesUmTrades(LoaderType):
+    pass
+
+class BinanceVisionFuturesUm:
+    klines: BinanceVisionFuturesUmKlines
+    trades: BinanceVisionFuturesUmTrades
+
+class BinanceVisionFuturesCmKlines(LoaderType):
+    pass
+
+class BinanceVisionFuturesCmTrades(LoaderType):
+    pass
+
+class BinanceVisionFuturesCm:
+    klines: BinanceVisionFuturesCmKlines
+    trades: BinanceVisionFuturesCmTrades
+
+class BinanceVisionFutures(LoaderType):
+    um: BinanceVisionFuturesUm
+    cm: BinanceVisionFuturesCm
+
+class BinanceVisionSpotKlines(LoaderType):
+    pass
+
+class BinanceVisionSpotTrades(LoaderType):
+    pass
+
+class BinanceVisionSpotAggtrades(LoaderType):
+    pass
+
+class BinanceVisionSpot(LoaderType):
+    klines: BinanceVisionSpotKlines
+    trades: BinanceVisionSpotTrades
+    aggTrades: BinanceVisionSpotAggtrades
+
+class BinanceVision:
+    futures: BinanceVisionFutures
+    spot: BinanceVisionSpot
+binance: BinanceVision
+
+def generate_pyi():
+    pass

narf-0.1.0/narf/data/loaders/binance_vision/__init__.py

@@ -0,0 +1,21 @@
+from narf.data.loaders.base_spec import build
+from .spec import binance_vision_spec
+
+
+binance = build(binance_vision_spec)
+
+ 
+def generate_pyi():
+    from narf.data.generate import generate_pyi
+
+    generate_pyi(
+        spec=binance_vision_spec,
+        root_class_name="BinanceVision",
+        root_var_name="binance",
+        package_name="binance_vision",
+        loader_import="narf.data.loaders.binance_vision.loader",
+        loader_class_name="BinanceVisionLoader",
+    )
+
+
+__all__ = ["binance", "generate_pyi"]

narf-0.1.0/narf/data/loaders/binance_vision/__init__.pyi

@@ -0,0 +1,47 @@
+from narf.data.loaders.binance_vision.loader import BinanceVisionLoader as LoaderType
+
+class BinanceVisionFuturesUmKlines(LoaderType):
+    pass
+
+class BinanceVisionFuturesUmTrades(LoaderType):
+    pass
+
+class BinanceVisionFuturesUm:
+    klines: BinanceVisionFuturesUmKlines
+    trades: BinanceVisionFuturesUmTrades
+
+class BinanceVisionFuturesCmKlines(LoaderType):
+    pass
+
+class BinanceVisionFuturesCmTrades(LoaderType):
+    pass
+
+class BinanceVisionFuturesCm:
+    klines: BinanceVisionFuturesCmKlines
+    trades: BinanceVisionFuturesCmTrades
+
+class BinanceVisionFutures(LoaderType):
+    um: BinanceVisionFuturesUm
+    cm: BinanceVisionFuturesCm
+
+class BinanceVisionSpotKlines(LoaderType):
+    pass
+
+class BinanceVisionSpotTrades(LoaderType):
+    pass
+
+class BinanceVisionSpotAggtrades(LoaderType):
+    pass
+
+class BinanceVisionSpot(LoaderType):
+    klines: BinanceVisionSpotKlines
+    trades: BinanceVisionSpotTrades
+    aggTrades: BinanceVisionSpotAggtrades
+
+class BinanceVision:
+    futures: BinanceVisionFutures
+    spot: BinanceVisionSpot
+binance: BinanceVision
+
+def generate_pyi():
+    pass

narf-0.1.0/narf/data/loaders/binance_vision/loader.py

@@ -0,0 +1,106 @@
+import hashlib
+import io
+import os
+from typing import List, Optional
+from datetime import datetime
+import zipfile
+from matplotlib.dates import relativedelta
+import requests
+import pandas as pd
+
+from narf.data.loaders.base_loader import Loader, extract_semantic_path
+
+
+URL_PATTERNS = [
+    (
+        ("futures",),
+        "{base}/{market}/um/monthly/klines/{symbol}/{interval}/{symbol}-{interval}-{year}-{month:02d}.zip",
+    ),
+    (
+        ("futures", "<margination>", "<datatype>"),
+        "{base}/{market}/{margination}/monthly/{datatype}/{symbol}/{interval}/{symbol}-{interval}-{year}-{month:02d}.zip",
+    ),
+
+    (
+        ("spot",),
+        "{base}/{market}/monthly/{datatype}/{symbol}/{interval}/{symbol}-{interval}-{year}-{month:02d}.zip",
+    ),
+    (
+        ("spot", "aggTrades"),
+        "{base}/{market}/monthly/{datatype}/{symbol}/{symbol}-{datatype}-{year}-{month:02d}.zip",
+    ),
+    (
+        ("spot", "<datatype>"),
+        "{base}/{market}/monthly/{datatype}/{symbol}/{interval}/{symbol}-{interval}-{year}-{month:02d}.zip",
+    ),
+]
+
+
+INDEX_COLUMN = {
+    "klines": "open_time",
+    "trades": "timestamp",
+    "aggTrades": "timestamp",
+}
+
+
+def cache_to_file(func):
+    def wrapper(url: str, *args, **kwargs):
+        url_hash = hashlib.sha256(url.encode()).hexdigest()
+        cache_file = f"cache/{url_hash}.csv"
+        if not os.path.exists(cache_file):
+            df = func(url, *args, **kwargs)
+            df.to_csv(cache_file)
+        return pd.read_csv(cache_file)
+    return wrapper
+
+
+@cache_to_file
+def load_zipped_csv(url: str, header: Optional[List[str]] = None) -> pd.DataFrame:
+    r = requests.get(url)
+    r.raise_for_status()
+
+    zf = zipfile.ZipFile(io.BytesIO(r.content))
+    name = zf.namelist()[0]
+
+    with zf.open(name) as f:
+        df = pd.read_csv(f)#, header=None if header else 0, names=header, index_col=False)
+
+    return df
+
+
+class BinanceVisionLoader(Loader):
+    BASE_URL = "https://data.binance.vision/data"
+    URL_PATTERNS = URL_PATTERNS
+
+    def _load_month(self, symbol: str, interval: str, year: int, month: int) -> pd.DataFrame:
+        url = self._build_url(**{
+            "symbol": symbol,
+            "interval": interval,
+            "year": year,
+            "month": month,
+        })
+        print('Loading', url)
+        df = load_zipped_csv(url)
+
+        path = self.get_parsed_path()
+        idx_col = INDEX_COLUMN[path["datatype"]]
+
+        df[idx_col] = pd.to_datetime(df[idx_col] * 1000000)
+        df.set_index(idx_col, inplace=True)
+
+        return df
+    
+    def load(self, symbol: str, start: datetime, end: Optional[datetime] = None, interval: str = "1m") -> pd.DataFrame:
+        if end is None:
+            end = datetime.now()
+
+        cursor = start
+        df = pd.DataFrame()
+        while cursor <= end:
+            df = pd.concat([
+                df, 
+                self._load_month(symbol, interval, cursor.year, cursor.month),
+            ])
+            cursor += relativedelta(months=1)
+        
+        return df

narf-0.1.0/narf/data/loaders/binance_vision/spec.py

@@ -0,0 +1,21 @@
+from narf.data.loaders.base_spec import NodeSpec
+from narf.data.loaders.binance_vision.loader import BinanceVisionLoader
+
+
+binance_vision_spec = NodeSpec("market", {
+    "futures": NodeSpec("margination", {
+        "um": NodeSpec("datatype", {
+            "klines": NodeSpec(loader=BinanceVisionLoader),
+            "trades": NodeSpec(loader=BinanceVisionLoader),
+        }),
+        "cm": NodeSpec("datatype", {
+            "klines": NodeSpec(loader=BinanceVisionLoader),
+            "trades": NodeSpec(loader=BinanceVisionLoader),
+        }),
+    }, loader=BinanceVisionLoader),
+    "spot": NodeSpec("datatype", {
+        "klines": NodeSpec(loader=BinanceVisionLoader),
+        "trades": NodeSpec(loader=BinanceVisionLoader),
+        "aggTrades": NodeSpec(loader=BinanceVisionLoader),
+    }, loader=BinanceVisionLoader),
+})

narf-0.1.0/narf.egg-info/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.4
+Name: narf
+Version: 0.1.0
+Summary: Market data loader library for cryptocurrency exchanges
+Project-URL: Homepage, https://github.com/numan-narf/narf-market-data
+Project-URL: Repository, https://github.com/numan-narf/narf-market-data
+Project-URL: Issues, https://github.com/numan-narf/narf-market-data/issues
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: build>=1.3.0
+Requires-Dist: matplotlib>=3.10.7
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: pandas-stubs>=2.3.2.250926
+Requires-Dist: requests>=2.32.5
+Requires-Dist: twine>=6.2.0
+Requires-Dist: types-requests>=2.32.4.20250913

narf-0.1.0/narf.egg-info/SOURCES.txt

@@ -0,0 +1,19 @@
+MANIFEST.in
+README.md
+pyproject.toml
+narf/__init__.py
+narf.egg-info/PKG-INFO
+narf.egg-info/SOURCES.txt
+narf.egg-info/dependency_links.txt
+narf.egg-info/requires.txt
+narf.egg-info/top_level.txt
+narf/data/__init__.py
+narf/data/generate.py
+narf/data/loaders/__init__.py
+narf/data/loaders/base_loader.py
+narf/data/loaders/base_spec.py
+narf/data/loaders/binance/__init__.pyi
+narf/data/loaders/binance_vision/__init__.py
+narf/data/loaders/binance_vision/__init__.pyi
+narf/data/loaders/binance_vision/loader.py
+narf/data/loaders/binance_vision/spec.py

narf-0.1.0/narf.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

narf-0.1.0/narf.egg-info/requires.txt

@@ -0,0 +1,7 @@
+build>=1.3.0
+matplotlib>=3.10.7
+pandas>=2.3.3
+pandas-stubs>=2.3.2.250926
+requests>=2.32.5
+twine>=6.2.0
+types-requests>=2.32.4.20250913

narf-0.1.0/narf.egg-info/top_level.txt

@@ -0,0 +1,3 @@
+cache
+dist
+narf

narf-0.1.0/pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "narf"
+version = "0.1.0"
+description = "Market data loader library for cryptocurrency exchanges"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "build>=1.3.0",
+    "matplotlib>=3.10.7",
+    "pandas>=2.3.3",
+    "pandas-stubs>=2.3.2.250926",
+    "requests>=2.32.5",
+    "twine>=6.2.0",
+    "types-requests>=2.32.4.20250913",
+]
+
+[project.urls]
+Homepage = "https://github.com/numan-narf/narf-market-data"
+Repository = "https://github.com/numan-narf/narf-market-data"
+Issues = "https://github.com/numan-narf/narf-market-data/issues"
+
+[tool.setuptools]
+packages = {find = {}}
+
+[tool.setuptools.package-data]
+"*" = ["*.pyi"]

narf-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+