rm-bdd 0.1.0__tar.gz

rm_bdd-0.1.0/PKG-INFO

@@ -0,0 +1,180 @@
+Metadata-Version: 2.4
+Name: rm-bdd
+Version: 0.1.0
+Summary: Binance Data Downloader
+Author-email: Your Name <you@example.com>
+License-Expression: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp==3.13.3
+Requires-Dist: pyarrow==23.0.1
+Requires-Dist: pandas==3.0.1
+Requires-Dist: xmltodict==1.0.4
+
+# rm-binance-data-downloader
+
+A Python library for downloading, extracting and formatting historical
+Binance market data.
+
+The package provides a pipeline that:
+
+1.  Downloads data from Binance Vision
+2.  Extracts compressed archives
+3.  Formats data into a structured dataset
+
+It is designed for fast data preparation for quantitative trading,
+backtesting and data analysis.
+
+------------------------------------------------------------------------
+
+# Installation
+
+``` bash
+pip install rm-bdd
+```
+
+------------------------------------------------------------------------
+
+# Features
+
+-   Download historical Binance data
+-   Automatic archive extraction
+-   Data formatting pipeline
+-   Metadata management
+-   Async architecture
+-   Easy integration into trading systems
+
+------------------------------------------------------------------------
+
+# Quick Example
+
+``` python
+import asyncio
+import time
+
+from rm_bdd.data_downloader import DataDownloader
+from rm_bdd.data_extractor import DataExtractor
+from rm_bdd.data_formatter import DataFormatter
+from rm_bdd.binance_metadata_manager import BinanceMetadataManager
+
+
+class DataManager:
+
+    def __init__(self, downloader, extractor, formatter):
+        self._downloader = downloader
+        self._extractor = extractor
+        self._formatter = formatter
+
+    async def download_and_save(self, symbol, timeframe, date_from=None, date_to=None):
+
+        start = time.time()
+        await self._downloader.download(symbol, timeframe, date_from=date_from, date_to=date_to)
+        print("download time:", time.time() - start)
+
+        start = time.time()
+        await self._extractor.extract(symbol, timeframe, date_from=date_from, date_to=date_to)
+        print("extract time:", time.time() - start)
+
+        start = time.time()
+        await self._formatter.format(symbol, timeframe, date_from=date_from, date_to=date_to)
+        print("format time:", time.time() - start)
+
+
+async def main():
+
+    downloader = DataDownloader(
+        "downloads/",
+        BinanceMetadataManager("downloads/metadata.json")
+    )
+
+    extractor = DataExtractor(
+        "downloads/",
+        "extracts/",
+        BinanceMetadataManager("extracts/metadata.json")
+    )
+
+    formatter = DataFormatter(
+        "extracts/",
+        "data/",
+        BinanceMetadataManager("data/metadata.json")
+    )
+
+    manager = DataManager(downloader, extractor, formatter)
+
+    await manager.download_and_save("BTCUSDT", "1m")
+
+
+asyncio.run(main())
+```
+
+------------------------------------------------------------------------
+
+# Result Folder Structure
+
+After execution the folders will look like:
+
+    downloads/
+        BTCUSDT/
+        metadata.json
+
+    extracts/
+        BTCUSDT/
+        metadata.json
+
+    data/
+        BTCUSDT/
+        metadata.json
+
+------------------------------------------------------------------------
+
+# Pipeline Overview
+
+The processing pipeline consists of three stages:
+
+### Downloader
+
+Downloads historical data archives from Binance Vision.
+
+### Extractor
+
+Extracts downloaded archives.
+
+### Formatter
+
+Formats extracted CSV data into a structured dataset ready for analysis.
+
+------------------------------------------------------------------------
+
+# Metadata Manager
+
+The library uses a metadata system to track downloaded, extracted and
+formatted data.
+
+This prevents duplicate downloads and processing.
+
+------------------------------------------------------------------------
+
+# Example Use Case
+
+Typical workflow:
+
+    download → extract → format → analyze
+
+Used for:
+
+-   algorithmic trading
+-   backtesting
+-   machine learning datasets
+-   market research
+
+------------------------------------------------------------------------
+
+# Requirements
+
+Python 3.10+
+
+------------------------------------------------------------------------
+
+# License
+
+MIT

rm_bdd-0.1.0/README.md

@@ -0,0 +1,167 @@
+# rm-binance-data-downloader
+
+A Python library for downloading, extracting and formatting historical
+Binance market data.
+
+The package provides a pipeline that:
+
+1.  Downloads data from Binance Vision
+2.  Extracts compressed archives
+3.  Formats data into a structured dataset
+
+It is designed for fast data preparation for quantitative trading,
+backtesting and data analysis.
+
+------------------------------------------------------------------------
+
+# Installation
+
+``` bash
+pip install rm-bdd
+```
+
+------------------------------------------------------------------------
+
+# Features
+
+-   Download historical Binance data
+-   Automatic archive extraction
+-   Data formatting pipeline
+-   Metadata management
+-   Async architecture
+-   Easy integration into trading systems
+
+------------------------------------------------------------------------
+
+# Quick Example
+
+``` python
+import asyncio
+import time
+
+from rm_bdd.data_downloader import DataDownloader
+from rm_bdd.data_extractor import DataExtractor
+from rm_bdd.data_formatter import DataFormatter
+from rm_bdd.binance_metadata_manager import BinanceMetadataManager
+
+
+class DataManager:
+
+    def __init__(self, downloader, extractor, formatter):
+        self._downloader = downloader
+        self._extractor = extractor
+        self._formatter = formatter
+
+    async def download_and_save(self, symbol, timeframe, date_from=None, date_to=None):
+
+        start = time.time()
+        await self._downloader.download(symbol, timeframe, date_from=date_from, date_to=date_to)
+        print("download time:", time.time() - start)
+
+        start = time.time()
+        await self._extractor.extract(symbol, timeframe, date_from=date_from, date_to=date_to)
+        print("extract time:", time.time() - start)
+
+        start = time.time()
+        await self._formatter.format(symbol, timeframe, date_from=date_from, date_to=date_to)
+        print("format time:", time.time() - start)
+
+
+async def main():
+
+    downloader = DataDownloader(
+        "downloads/",
+        BinanceMetadataManager("downloads/metadata.json")
+    )
+
+    extractor = DataExtractor(
+        "downloads/",
+        "extracts/",
+        BinanceMetadataManager("extracts/metadata.json")
+    )
+
+    formatter = DataFormatter(
+        "extracts/",
+        "data/",
+        BinanceMetadataManager("data/metadata.json")
+    )
+
+    manager = DataManager(downloader, extractor, formatter)
+
+    await manager.download_and_save("BTCUSDT", "1m")
+
+
+asyncio.run(main())
+```
+
+------------------------------------------------------------------------
+
+# Result Folder Structure
+
+After execution the folders will look like:
+
+    downloads/
+        BTCUSDT/
+        metadata.json
+
+    extracts/
+        BTCUSDT/
+        metadata.json
+
+    data/
+        BTCUSDT/
+        metadata.json
+
+------------------------------------------------------------------------
+
+# Pipeline Overview
+
+The processing pipeline consists of three stages:
+
+### Downloader
+
+Downloads historical data archives from Binance Vision.
+
+### Extractor
+
+Extracts downloaded archives.
+
+### Formatter
+
+Formats extracted CSV data into a structured dataset ready for analysis.
+
+------------------------------------------------------------------------
+
+# Metadata Manager
+
+The library uses a metadata system to track downloaded, extracted and
+formatted data.
+
+This prevents duplicate downloads and processing.
+
+------------------------------------------------------------------------
+
+# Example Use Case
+
+Typical workflow:
+
+    download → extract → format → analyze
+
+Used for:
+
+-   algorithmic trading
+-   backtesting
+-   machine learning datasets
+-   market research
+
+------------------------------------------------------------------------
+
+# Requirements
+
+Python 3.10+
+
+------------------------------------------------------------------------
+
+# License
+
+MIT

rm_bdd-0.1.0/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "rm-bdd"
+version = "0.1.0"
+description = "Binance Data Downloader"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [
+    { name = "Your Name", email = "you@example.com" }
+]
+dependencies = [
+    "aiohttp==3.13.3",
+    "pyarrow==23.0.1",
+    "pandas==3.0.1",
+    "xmltodict==1.0.4",
+]
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["rm_bdd*"]

rm_bdd-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

rm_bdd-0.1.0/src/rm_bdd/__init__.py

@@ -0,0 +1,2 @@
+from src.rm_bdd import data_downloader, data_formatter, data_manager
+from src.rm_bdd import data_extractor

rm_bdd-0.1.0/src/rm_bdd/binance_metadata_manager.py

@@ -0,0 +1,23 @@
+from src.rm_bdd.metadata_manager import MetadataManager
+
+
+def _get_data_from_filename(filename) -> tuple[str, str, str]:
+    filename = filename.split(".", 1)[0]
+    symbol, timeframe, date = filename.split("-", 2)
+    return symbol, timeframe, date
+
+
+class BinanceMetadataManager:
+    def __init__(self, file_path):
+        self._metadata_manager = MetadataManager(file_path)
+
+    def update(self, filename):
+        symbol, timeframe, date = _get_data_from_filename(filename.rsplit("/", 1)[1])
+        self._metadata_manager.update_metadata(symbol, timeframe, date)
+
+    def check(self, filename):
+        symbol, timeframe, date = _get_data_from_filename(filename.rsplit("/", 1)[1])
+
+        return self._metadata_manager.check_date_not_in_metadata(
+            symbol, timeframe, date
+        )

rm_bdd-0.1.0/src/rm_bdd/csvs_to_parquet.py

@@ -0,0 +1,66 @@
+import os
+
+import pandas as pd
+
+
+def csvs_to_parquets(files):
+    for file in files:
+        # Read the CSV file
+        df = pd.read_csv(file, parse_dates=["timestamp"], infer_datetime_format=True)
+
+        # Ensure the 'timestamp' column is in datetime format
+        df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms")
+
+        # Set the 'timestamp' column as the index
+        df.set_index("timestamp", inplace=True)
+
+        # Create the output directory if it doesn't exist
+        output_dir = "parquets"
+        os.makedirs(output_dir, exist_ok=True)
+
+        # Define the output parquet file path
+        parquet_file = os.path.join(
+            output_dir, os.path.basename(file).replace(".csv", ".parquet")
+        )
+
+        # Write the DataFrame to a Parquet file
+        df.to_parquet(parquet_file, engine="pyarrow")
+
+
+def normalize_time(series):
+    """
+    Визначає, чи open_time у мілісекундах чи наносекундах,
+    і перетворює їх в datetime (UTC).
+    """
+    series[series > 10 ** 13] = series[series > 10 ** 13] // 1000
+    return series
+
+
+async def csv_to_partitioned_parquet(files, symbol, timeframe, path="data"):
+    columns = [
+        "open_time",
+        "open",
+        "high",
+        "low",
+        "close",
+        "volume",
+        "close_time",
+        "quote_asset_volume",
+        "number_of_trades",
+        "taker_buy_base_asset_volume",
+        "taker_buy_quote_asset_volume",
+        "ignore",
+    ]
+    df = pd.concat(
+        [pd.read_csv(file, names=columns) for file in files], ignore_index=True
+    )
+    df["open_time"] = normalize_time(df["open_time"])
+    df["close_time"] = normalize_time(df["close_time"])
+    df["year"] = pd.to_datetime(df["open_time"], unit="ms", utc=True).dt.year
+    df["month"] = pd.to_datetime(df["open_time"], unit="ms", utc=True).dt.month
+    df["day"] = pd.to_datetime(df["open_time"], unit="ms", utc=True).dt.day
+    df["symbol"] = symbol
+    df["timeframe"] = timeframe
+    df.to_parquet(path, partition_cols=["symbol", "timeframe", "year", "month", "day"])
+
+

rm_bdd-0.1.0/src/rm_bdd/data_downloader.py

@@ -0,0 +1,73 @@
+import asyncio
+import logging
+import os
+
+import aiohttp
+
+from src.rm_bdd.binance_metadata_manager import BinanceMetadataManager
+from src.rm_bdd.files_parser import get_all_available_filenames_in_daterange
+
+logger = logging.getLogger(__name__)
+BASE_VISION_URL = "https://data.binance.vision/"
+
+
+async def download_file(key, folder) -> str | None:
+    logger.info(f"Downloading {key} to {folder}")
+    url = BASE_VISION_URL + key
+
+    os.makedirs(folder, exist_ok=True)
+    try:
+        local_filename = os.path.join(folder, url.split("/")[-1])
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as r:
+                r.raise_for_status()
+                with open(local_filename, "wb") as f:
+                    async for chunk in r.content.iter_chunked(8192):
+                        chunk: bytes
+                        f.write(chunk)
+        logger.info(f"Downloaded {key} to {local_filename}")
+        return str(local_filename)
+    except Exception as e:
+        logger.info(f"Failed to download {key}: {e}")
+        logger.error(e)
+        return None
+
+
+class DataDownloader:
+    def __init__(self, download_folder, metadata_manager):
+        self._download_folder = download_folder
+        self._metadata_manager = metadata_manager
+
+    async def download(self, symbol, timeframe, date_from=None, date_to=None):
+        filenames = await get_all_available_filenames_in_daterange(
+            symbol, timeframe, date_from, date_to
+        )
+
+        # get day from filename and check in metadata manager
+        filenames = list(
+            filter(self._metadata_manager.check, filenames)
+        )
+
+        logger.info(
+            f"Downloading {timeframe} data for {symbol} from {date_from} to {date_to}"
+        )
+
+        results = await asyncio.gather(
+            *(download_file(filename, self._download_folder) for filename in filenames)
+        )
+        logger.info(f"Finished downloading {timeframe} data for {symbol}")
+        logger.debug(f"Downloaded {len(results)} results for {symbol}")
+
+        for filename in results:
+            self._metadata_manager.update(filename)
+
+        return results
+
+
+if __name__ == '__main__':
+    async def main():
+        downloader = DataDownloader("downloads/", BinanceMetadataManager("downloads/metadata.json"))
+        await downloader.download("BTCUSDT", '1m')
+
+
+    asyncio.run(main())

rm_bdd-0.1.0/src/rm_bdd/data_extractor.py

@@ -0,0 +1,55 @@
+import os
+from functools import partial
+
+import zipfile
+
+from src.rm_bdd.binance_metadata_manager import BinanceMetadataManager
+from src.rm_bdd.files_parser import _is_filename_in_date_range
+
+
+class DataExtractor:
+    def __init__(self, download_folder, extract_folder, metadata_manager):
+        self._download_folder = download_folder
+        self._extract_folder = extract_folder
+        self._metadata_manager = metadata_manager
+
+    async def extract(self, symbol, timeframe, date_from=None, date_to=None):
+        downloaded_files = self._downloaded_files
+        files_to_extract = filter(lambda a: a.startswith(f'{symbol.upper()}-{timeframe}'), downloaded_files)
+        files_to_extract = list(filter(
+            partial(_is_filename_in_date_range, date_from=date_from, date_to=date_to),
+            files_to_extract,
+        ))
+        retval = await self.extract_files(files_to_extract)
+
+        return retval
+
+    async def extract_files(self, filenames):
+        filenames = [self._download_folder + filename for filename in filenames]
+        filenames = list(
+            filter(self._metadata_manager.check, filenames)
+        )
+        filenames = list(filter(lambda a: a.count("-") == 1 or f'{a.rsplit("-", 1)[0]}.zip' not in filenames, filenames))
+        retval = []
+        for filename in filenames:
+            with zipfile.ZipFile(filename) as zip_file:
+                zip_file.extractall(self._extract_folder)
+            self._metadata_manager.update(filename)
+            retval.append(filename)
+        return retval
+
+    @property
+    def _downloaded_files(self) -> list[str]:
+        return list(filter(lambda a: a.endswith(".zip"), os.listdir(self._download_folder)))
+
+
+if __name__ == '__main__':
+    import asyncio
+
+
+    async def main():
+        extractor = DataExtractor("downloads/", "extracts/", BinanceMetadataManager("extracts/metadata.json"))
+        await extractor.extract("BTCUSDT", '1m')
+
+
+    asyncio.run(main())

rm_bdd-0.1.0/src/rm_bdd/data_formatter.py

@@ -0,0 +1,57 @@
+import os
+from functools import partial
+
+from src.rm_bdd.csvs_to_parquet import csv_to_partitioned_parquet
+from src.rm_bdd.files_parser import _is_filename_in_date_range
+
+
+class DataFormatter:
+    def __init__(self, extract_folder, data_folder, metadata_manager):
+        self._extract_folder = extract_folder
+        self._data_folder = data_folder
+        self._metadata_manager = metadata_manager
+
+    async def format(self, symbol, timeframe, date_from=None, date_to=None):
+        extracted_files = self._extracted_files
+        files_to_format = filter(lambda a: a.startswith(f'{symbol.upper()}-{timeframe}'), extracted_files)
+        files_to_format = list(filter(
+            partial(_is_filename_in_date_range, date_from=date_from, date_to=date_to),
+            files_to_format,
+        ))
+
+        await self.format_files(files_to_format)
+
+    async def format_files(self, filenames):
+        os.makedirs(self._data_folder, exist_ok=True)
+        filenames = [self._extract_folder + filename for filename in filenames]
+
+        filenames = list(
+            filter(lambda a: a.count("-") == 1 or f'{a.rsplit("-", 1)[0]}.csv' not in filenames, filenames))
+        retval = []
+        symbol = filenames[0].rsplit("/", 1)[-1].split("-", 1)[0]
+        timeframe = filenames[0].rsplit("/", 1)[-1].split("-", 2)[1]
+        filenames = list(
+            filter(self._metadata_manager.check, filenames)
+        )
+        print("files to format: ", filenames)
+        if filenames:
+            await csv_to_partitioned_parquet(filenames, symbol, timeframe, self._data_folder)
+
+            for filename in filenames:
+                self._metadata_manager.update(filename)
+                retval.append(filename)
+        return retval
+
+    @property
+    def _extracted_files(self) -> list[str]:
+        return list(filter(lambda a: a.endswith(".csv"), os.listdir(self._extract_folder)))
+
+
+if __name__ == '__main__':
+    def main():
+        from src.rm_bdd.binance_metadata_manager import BinanceMetadataManager
+        formatter = DataFormatter("extracts/", "data/", BinanceMetadataManager("data/metadata.json"))
+        formatter.format("BTCUSDT", '1m')
+
+
+    main()

rm_bdd-0.1.0/src/rm_bdd/data_manager.py

@@ -0,0 +1,45 @@
+import time
+
+
+class DataManager:
+    def __init__(self, downloader, extractor, formatter):
+        self._downloader = downloader
+        self._extractor = extractor
+        self._formatter = formatter
+
+    async def download_and_save(self, symbol, timeframe, date_from=None, date_to=None):
+        start = time.time()
+        await self._downloader.download(symbol, timeframe, date_from=date_from, date_to=date_to)
+        print("download time: ", time.time() - start)
+
+        start = time.time()
+        await self._extractor.extract(symbol, timeframe, date_from=date_from, date_to=date_to)
+        print("extract time: ", time.time() - start)
+
+        start = time.time()
+        await self._formatter.format(symbol, timeframe, date_from=date_from, date_to=date_to)
+        print("format time: ", time.time() - start)
+
+
+if __name__ == '__main__':
+    import asyncio
+    from src.rm_bdd.data_downloader import DataDownloader
+    from src.rm_bdd.data_extractor import DataExtractor
+    from src.rm_bdd.data_formatter import DataFormatter
+    from src.rm_bdd.binance_metadata_manager import BinanceMetadataManager
+
+    downloader = DataDownloader("downloads/", BinanceMetadataManager("downloads/metadata.json"))
+    extractor = DataExtractor("downloads/", "extracts/", BinanceMetadataManager("extracts/metadata.json"))
+    formatter = DataFormatter("extracts/", "data/", BinanceMetadataManager("data/metadata.json"))
+
+    manager = DataManager(downloader, extractor, formatter)
+
+
+    async def main():
+        await manager.download_and_save("BTCUSDT", '1m')
+
+
+    start = time.time()
+    asyncio.run(main())
+
+    print(time.time() - start)

rm_bdd-0.1.0/src/rm_bdd/files_parser.py

@@ -0,0 +1,76 @@
+import asyncio
+import datetime
+import re
+from functools import partial
+
+from src.rm_bdd.binance_metadata_manager import _get_data_from_filename
+from src.rm_bdd.path_loader import get_all_path_responses
+
+DAY_DATE_REGEX = re.compile(r"^\d{4}-\d{2}-\d{2}$")
+async def _get_all_available_filenames(symbol, timeframe) -> list[str]:
+    # get all monthly data
+    # get all daily data for last month
+    path = f"data/spot/monthly/klines/{symbol}/{timeframe}"
+    monthly = await get_all_path_responses(f"data/spot/monthly/klines/{symbol}/{timeframe}")
+
+    filenames = [item['Key'] for item in monthly[0]['ListBucketResult'].get('Contents', [])]
+    file_path, year, month = filenames[-1].rsplit(".", 2)[0].rsplit("-", 2)
+
+    file_path = file_path.replace("monthly", "daily")
+    if month == '12':
+        year = "%02i"%(int(year) + 1)
+    else:
+        month = "%02i"%(int(month) + 1)
+
+    file_path = "-".join([file_path, year, month])
+
+    path = path.replace("monthly", "daily")
+
+    daily = await get_all_path_responses(path, marker=file_path)
+
+    filenames.extend([item['Key'] for item in daily[0]['ListBucketResult'].get('Contents', [])])
+    return filenames
+
+
+def str_to_datetime(date: str) -> datetime.datetime:
+    if DAY_DATE_REGEX.match(date):
+        return datetime.datetime.strptime(date, "%Y-%m-%d")
+    return datetime.datetime.strptime(date, "%Y-%m")
+
+
+def _is_filename_in_date_range(filename: str, date_from: str, date_to: str) -> bool:
+    *_, date = _get_data_from_filename(filename)
+    date = str_to_datetime(date)
+    retval = True
+    if date_from:
+        date_from = str_to_datetime(date_from)
+        retval = retval and date >= date_from
+
+    if date_to:
+        date_to = str_to_datetime(date_to)
+        retval =  retval and date <= date_to
+    return retval
+
+
+async def get_all_available_filenames_in_daterange(
+        symbol, timeframe, date_from=None, date_to=None
+) -> list[str]:
+    filenames = await _get_all_available_filenames(symbol, timeframe)
+    if date_from or date_to:
+        filenames = list(filter(
+            partial(_is_filename_in_date_range, date_from=date_from, date_to=date_to),
+            filenames,
+        ))
+    return filenames
+
+
+if __name__ == '__main__':
+    async def main():
+        # await _get_all_available_filenames("BTCUSDT", timeframe="1m")
+        # result = await get_all_available_filenames_in_daterange("BTCUSDT", '1m', '2024-03', '2024-05')
+        # result = await get_all_available_filenames_in_daterange("BTCUSDT", '1m')
+        result = await get_all_available_filenames_in_daterange("BTCUSDT", '1m', date_from="2025-09")
+        print(result)
+        ...
+
+    asyncio.run(main())

rm_bdd-0.1.0/src/rm_bdd/metadata_manager.py

@@ -0,0 +1,48 @@
+import json
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+class MetadataManager:
+    def __init__(self, file_path):
+        self._data = None
+        self._file_path = file_path
+
+    def _load_metadata(self):
+        if os.path.isfile(self._file_path):
+            with open(self._file_path, "r") as f:
+                self._data = json.load(f)
+        else:
+            self._data = {}
+
+    @property
+    def data(self):
+        if self._data is None:
+            self._load_metadata()
+        return self._data
+
+    def _save_metadata(self):
+        with open(self._file_path, mode="w", encoding="utf-8") as f:
+            json.dump(self.data, f, indent=4)
+
+    def update_metadata(self, symbol: str, timeframe: str, date: str):
+        if symbol not in self.data:
+            self._data[symbol] = {}
+
+        if timeframe not in self.data[symbol]:
+            self._data[symbol][timeframe] = []
+
+        if not self._data[symbol][timeframe] or self._data[symbol][timeframe][-1] != date:
+            self._data[symbol][timeframe].append(date)
+            self._save_metadata()
+
+    def check_date_not_in_metadata(self, symbol: str, timeframe: str, date: str):
+        metadata = self.data.get(symbol, {}).get(timeframe, [])
+        date_not_in_metadata = date not in metadata
+        date_month_not_in_metadata = date.count("-") != 2 or date.rsplit("-", 1)[0] not in metadata
+        result = date_not_in_metadata and date_month_not_in_metadata
+        if not result:
+            logger.info(f"Data for {symbol} {timeframe} {date} already exists. Skip.")
+        return result

rm_bdd-0.1.0/src/rm_bdd/path_loader.py

@@ -0,0 +1,62 @@
+import asyncio
+
+import aiohttp
+import xmltodict
+
+URL_TEMPLATE = "https://s3-ap-northeast-1.amazonaws.com/data.binance.vision"
+SEMAPHORE = asyncio.Semaphore(300)
+
+
+async def get_all_paths(prefix: str, delimiter="/"):
+    responses = await get_all_path_responses(prefix, delimiter)
+    retval = []
+    for response in responses:
+        common_prefixes = response.get("ListBucketResult", {}).get("CommonPrefixes", [])
+        if isinstance(common_prefixes, list):
+            retval.extend([i["Prefix"] for i in common_prefixes])
+        else:
+            retval.append(common_prefixes["Prefix"])
+    return retval
+
+
+async def get_all_path_responses(prefix: str, delimiter="/", marker=None, **params):
+    is_truncated = True
+    responses = []
+    while is_truncated:
+        response = await get_paths_response_json(prefix, delimiter, marker, **params)
+        is_truncated = (
+            response.get("ListBucketResult", {}).get("IsTruncated", "false") == "true"
+        )
+        responses.append(response)
+        marker = response.get("ListBucketResult", {}).get("NextMarker", None)
+
+    return responses
+
+
+async def get_paths_response_json(prefix: str, delimiter="/", marker=None, **params):
+    if not prefix.endswith("/"):
+        prefix += "/"
+    params = {"prefix": prefix, "delimiter": delimiter, "marker": marker, **params}
+    params = {k: v for k, v in params.items() if v is not None}
+    if prefix.endswith("/"):
+        prefix = prefix[:-1]
+    url = URL_TEMPLATE.format(prefix=prefix)
+    async with SEMAPHORE:
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url, params=params) as response:
+                text = await response.text()
+                return xmltodict.parse(text)
+
+
+if __name__ == "__main__":
+
+    async def main():
+        responses = await get_all_path_responses("data/spot/monthly/klines/SOLUSDT/1m")
+        print(
+            [
+                i.get("Key")
+                for response in responses for i in response.get("ListBucketResult", {}).get("Contents", [])
+            ]
+        )
+
+    asyncio.run(main())

rm_bdd-0.1.0/src/rm_bdd.egg-info/PKG-INFO

@@ -0,0 +1,180 @@
+Metadata-Version: 2.4
+Name: rm-bdd
+Version: 0.1.0
+Summary: Binance Data Downloader
+Author-email: Your Name <you@example.com>
+License-Expression: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp==3.13.3
+Requires-Dist: pyarrow==23.0.1
+Requires-Dist: pandas==3.0.1
+Requires-Dist: xmltodict==1.0.4
+
+# rm-binance-data-downloader
+
+A Python library for downloading, extracting and formatting historical
+Binance market data.
+
+The package provides a pipeline that:
+
+1.  Downloads data from Binance Vision
+2.  Extracts compressed archives
+3.  Formats data into a structured dataset
+
+It is designed for fast data preparation for quantitative trading,
+backtesting and data analysis.
+
+------------------------------------------------------------------------
+
+# Installation
+
+``` bash
+pip install rm-bdd
+```
+
+------------------------------------------------------------------------
+
+# Features
+
+-   Download historical Binance data
+-   Automatic archive extraction
+-   Data formatting pipeline
+-   Metadata management
+-   Async architecture
+-   Easy integration into trading systems
+
+------------------------------------------------------------------------
+
+# Quick Example
+
+``` python
+import asyncio
+import time
+
+from rm_bdd.data_downloader import DataDownloader
+from rm_bdd.data_extractor import DataExtractor
+from rm_bdd.data_formatter import DataFormatter
+from rm_bdd.binance_metadata_manager import BinanceMetadataManager
+
+
+class DataManager:
+
+    def __init__(self, downloader, extractor, formatter):
+        self._downloader = downloader
+        self._extractor = extractor
+        self._formatter = formatter
+
+    async def download_and_save(self, symbol, timeframe, date_from=None, date_to=None):
+
+        start = time.time()
+        await self._downloader.download(symbol, timeframe, date_from=date_from, date_to=date_to)
+        print("download time:", time.time() - start)
+
+        start = time.time()
+        await self._extractor.extract(symbol, timeframe, date_from=date_from, date_to=date_to)
+        print("extract time:", time.time() - start)
+
+        start = time.time()
+        await self._formatter.format(symbol, timeframe, date_from=date_from, date_to=date_to)
+        print("format time:", time.time() - start)
+
+
+async def main():
+
+    downloader = DataDownloader(
+        "downloads/",
+        BinanceMetadataManager("downloads/metadata.json")
+    )
+
+    extractor = DataExtractor(
+        "downloads/",
+        "extracts/",
+        BinanceMetadataManager("extracts/metadata.json")
+    )
+
+    formatter = DataFormatter(
+        "extracts/",
+        "data/",
+        BinanceMetadataManager("data/metadata.json")
+    )
+
+    manager = DataManager(downloader, extractor, formatter)
+
+    await manager.download_and_save("BTCUSDT", "1m")
+
+
+asyncio.run(main())
+```
+
+------------------------------------------------------------------------
+
+# Result Folder Structure
+
+After execution the folders will look like:
+
+    downloads/
+        BTCUSDT/
+        metadata.json
+
+    extracts/
+        BTCUSDT/
+        metadata.json
+
+    data/
+        BTCUSDT/
+        metadata.json
+
+------------------------------------------------------------------------
+
+# Pipeline Overview
+
+The processing pipeline consists of three stages:
+
+### Downloader
+
+Downloads historical data archives from Binance Vision.
+
+### Extractor
+
+Extracts downloaded archives.
+
+### Formatter
+
+Formats extracted CSV data into a structured dataset ready for analysis.
+
+------------------------------------------------------------------------
+
+# Metadata Manager
+
+The library uses a metadata system to track downloaded, extracted and
+formatted data.
+
+This prevents duplicate downloads and processing.
+
+------------------------------------------------------------------------
+
+# Example Use Case
+
+Typical workflow:
+
+    download → extract → format → analyze
+
+Used for:
+
+-   algorithmic trading
+-   backtesting
+-   machine learning datasets
+-   market research
+
+------------------------------------------------------------------------
+
+# Requirements
+
+Python 3.10+
+
+------------------------------------------------------------------------
+
+# License
+
+MIT

rm_bdd-0.1.0/src/rm_bdd.egg-info/SOURCES.txt

@@ -0,0 +1,17 @@
+README.md
+pyproject.toml
+src/rm_bdd/__init__.py
+src/rm_bdd/binance_metadata_manager.py
+src/rm_bdd/csvs_to_parquet.py
+src/rm_bdd/data_downloader.py
+src/rm_bdd/data_extractor.py
+src/rm_bdd/data_formatter.py
+src/rm_bdd/data_manager.py
+src/rm_bdd/files_parser.py
+src/rm_bdd/metadata_manager.py
+src/rm_bdd/path_loader.py
+src/rm_bdd.egg-info/PKG-INFO
+src/rm_bdd.egg-info/SOURCES.txt
+src/rm_bdd.egg-info/dependency_links.txt
+src/rm_bdd.egg-info/requires.txt
+src/rm_bdd.egg-info/top_level.txt

rm_bdd-0.1.0/src/rm_bdd.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

rm_bdd-0.1.0/src/rm_bdd.egg-info/requires.txt

@@ -0,0 +1,4 @@
+aiohttp==3.13.3
+pyarrow==23.0.1
+pandas==3.0.1
+xmltodict==1.0.4

rm_bdd-0.1.0/src/rm_bdd.egg-info/top_level.txt

@@ -0,0 +1 @@
+rm_bdd