pfeed 0.0.1.dev3__tar.gz → 0.0.1.dev5__tar.gz

pfeed-0.0.1.dev5/PKG-INFO

@@ -0,0 +1,281 @@
+Metadata-Version: 2.1
+Name: pfeed
+Version: 0.0.1.dev5
+Summary: Data pipeline for algo-trading, getting and storing both real-time and historical data made easy.
+Home-page: https://pfund.ai
+License: Apache-2.0
+Keywords: trading,algo-trading,data pipeline,ETL,data lake,data warehouse,data integration,historical data,live data,data streaming
+Author: Stephen Yau
+Author-email: softwareentrepreneer+pfeed@gmail.com
+Requires-Python: >=3.10,<3.12
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0.0)
+Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: connectorx (>=0.3.2,<0.4.0)
+Requires-Dist: minio (>=7.2.5,<8.0.0)
+Requires-Dist: pandas (>=2.2.0,<3.0.0)
+Requires-Dist: pfund (>=0.0.1.dev4,<0.0.2)
+Requires-Dist: platformdirs (>=4.2.0,<5.0.0)
+Requires-Dist: polars (>=0.20.16,<0.21.0)
+Requires-Dist: pyarrow (>=15.0.0,<16.0.0)
+Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
+Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
+Requires-Dist: ray (>=2.10.0,<3.0.0)
+Requires-Dist: requests (>=2.31.0,<3.0.0)
+Requires-Dist: rich (>=13.7.0,<14.0.0)
+Requires-Dist: s3fs (>=2024.3.1,<2025.0.0)
+Requires-Dist: tqdm (>=4.66.2,<5.0.0)
+Requires-Dist: yfinance (>=0.2.37,<0.3.0)
+Project-URL: Documentation, https://pfeed-docs.pfund.ai
+Project-URL: Repository, https://github.com/PFund-Software-Ltd/pfeed
+Description-Content-Type: text/markdown
+
+# PFeed: Data Pipeline for Algo-Trading, Getting and Storing Real-Time and Historical Data Made Easy.
+
+![GitHub stars](https://img.shields.io/github/stars/PFund-Software-Ltd/pfeed?style=social)
+![PyPI downloads](https://img.shields.io/pypi/dm/pfeed?label=downloads)
+[![PyPI](https://img.shields.io/pypi/v/pfeed.svg)](https://pypi.org/project/pfeed)
+![PyPI - Support Python Versions](https://img.shields.io/pypi/pyversions/pfeed)
+[![Jupyter Book Badge](https://raw.githubusercontent.com/PFund-Software-Ltd/pfeed/main/docs/images/jupyterbook.svg)](https://jupyterbook.org)
+[![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
+
+PFeed (/piː fiːd/) is a data integration library tailored for algorithmic trading, 
+serving as an ETL (Extract, Transform, Load) data pipeline between raw data sources and traders,
+helping them in creating a **local data lake for quantitative research**.
+
+PFeed allows traders to download historical, paper, and live data from various data sources, both free and paid,
+and stores them into a local data lake using [MinIO](https://min.io/).
+
+It is designed to be used alongside [PFund](https://github.com/PFund-Software-Ltd/pfund) — A Complete Algo-Trading Framework for Machine Learning, TradFi, CeFi and DeFi ready. Supports Vectorized and Event-Driven Backtesting, Paper and Live Trading, or as a standalone package.
+
+<details>
+<summary>Table of Contents</summary>
+
+- [Project Status](#project-status)
+- [Mission](#mission)
+- [Core Features](#core-features)
+- [Installation](#installation)
+- [Quick Start](#quick-start)
+    - [Main Usage: Data Feed](#main-usage-data-feed)
+    - [Download Historical Data on Command Line](#download-historical-data-on-command-line)
+    - [Download Historical Data in Python](#download-historical-data-in-python)
+    - [List Current Config](#list-current-config)
+    - [Run PFeed's docker-compose.yml](#run-pfeeds-docker-composeyml)
+- [Supported Data Sources](#supported-data-sources)
+- [Related Projects](#related-projects)
+
+</details>
+
+
+## Project Status
+**_Caution: PFeed is at a VERY EARLY stage, use it at your own risk._**
+
+PFeed is currently under active development, the framework design will be prioritized first over
+stability and scalability. 
+
+Please note that the available version is a *dev* version, not a *stable* one. \
+You are encouraged to play with the *dev* version, but only use it when a *stable* version is released.
+
+> PFeed for the time being only supports [Bybit](https://bybit.com/) and Yahoo Finance for testing purpose.
+
+## Mission
+Algo-trading has always been a complicated task due to the multitude of components and procedures involved. \
+Data collection and processing is probably the most mundane and yet critical part of it, as all results and findings 
+are derived from the data.
+
+However, preparing this data for use is not quick and easy. For example, sometimes even when the data is available (e.g. [Bybit data](https://public.bybit.com/trading/)), it is often in raw form and requires some cleaning.
+
+> PFeed's mission is to **_free traders from the tedious data work_** by providing cleaned data in a standard format that is ready for use, making them significantly faster to get to the analysis and strategy development phase.
+
+
+## Core Features
+- [x] Unified approach for interacting with various data sources, obtaining historical and real-time data
+- [x] ETL data pipline for transforming raw data and storing it in [MinIO](https://min.io/) (optional)
+- [x] Utilizes [Ray](https://github.com/ray-project/ray) for parallel data downloading
+- [x] Supports Pandas, [Polars](https://github.com/pola-rs/polars) as data tools
+- [ ] Integrates with [Prefect](https://www.prefect.io) to control data flows
+- [ ] Listens to PFund's trade engine and adds trade history to a local database [Timescaledb](https://www.timescale.com/) (optional)
+
+
+## Installation
+### Using [Poetry](https://python-poetry.org) (Recommended)
+```bash
+poetry add pfeed
+```
+
+### Using Pip
+```bash
+pip install pfeed
+```
+
+
+## Quick Start
+### Main Usage: Data Feed
+1. Download bybit raw data on the fly if not stored locally
+
+    ```python
+    import pfeed as pe
+
+    feed = pe.BybitFeed()
+
+    # df is a dataframe or a lazyframe (lazily loaded dataframe)
+    df = feed.get_historical_data(
+        'BTC_USDT_PERP',
+        resolution='raw',
+        start_date='2024-03-01',
+        end_date='2024-03-01',
+        data_tool='polars',  # or 'pandas'
+    )
+    ```
+
+    > By using pfeed, you are just one line of code away from playing with e.g. bybit data, how convenient!
+
+    Printing the first few rows of `df`:
+    |    | ts                            | symbol   |   side |   volume |   price | tickDirection   | trdMatchID                           |   grossValue |   homeNotional |   foreignNotional |
+    |---:|:------------------------------|:---------|-------:|---------:|--------:|:----------------|:-------------------------------------|-------------:|---------------:|------------------:|
+    |  0 | 2024-03-01 00:00:00.097599983 | BTCUSDT  |      1 |    0.003 | 61184.1 | ZeroMinusTick   | 79ac9a21-0249-5985-b042-906ec7604794 |  1.83552e+10 |          0.003 |           183.552 |
+    |  1 | 2024-03-01 00:00:00.098299980 | BTCUSDT  |      1 |    0.078 | 61184.9 | PlusTick        | 2af4e516-8ff4-5955-bb9c-38aa385b7b44 |  4.77242e+11 |          0.078 |          4772.42  |
+
+2. Get dataframe with different resolution, e.g. 1-minute data
+    ```python
+    import pfeed as pe
+
+    feed = pe.BybitFeed()
+
+    # df is a dataframe or a lazyframe (lazily loaded dataframe)
+    df = feed.get_historical_data(
+        'BTC_USDT_PERP',
+        resolution='1minute',  # or '1tick'/'1t', '2second'/'2s', '3minute'/'3m' etc.
+        start_date='2024-03-01',
+        end_date='2024-03-01',
+        data_tool='polars',
+    )
+    ```
+    > If you will be interacting with the data frequently, you should consider downloading it to your local machine.
+
+    Printing the first few rows of `df`:
+    |    | ts                  | product       | resolution   |    open |    high |     low |   close |   volume |
+    |---:|:--------------------|:--------------|:-------------|--------:|--------:|--------:|--------:|---------:|
+    |  0 | 2024-03-01 00:00:00 | BTC_USDT_PERP | 1m           | 61184.1 | 61244.5 | 61175.8 | 61244.5 |  159.142 |
+    |  1 | 2024-03-01 00:01:00 | BTC_USDT_PERP | 1m           | 61245.3 | 61276.5 | 61200.7 | 61232.2 |  227.242 |
+    |  2 | 2024-03-01 00:02:00 | BTC_USDT_PERP | 1m           | 61232.2 | 61249   | 61180   | 61184.2 |   91.446 |
+
+
+3. pfeed also supports simple wrapping of [yfinance](https://github.com/ranaroussi/yfinance)
+    ```python
+    import pfeed as pe
+
+    feed = pe.YahooFinanceFeed()
+
+    # you can still use any kwargs supported by yfinance's ticker.history(...)
+    # e.g. 'prepost', 'auto_adjust' etc.
+    yfinance_kwargs = {}
+
+    df = feed.get_historical_data(
+        'AAPL',
+        resolution='1d',
+        start_date='2024-03-01',
+        end_date='2024-03-20',
+        **yfinance_kwargs
+    )
+    ```
+    > Note that YahooFinanceFeed doesn't support the kwarg `data_tool`, e.g. polars
+    
+    Printing the first few rows of `df`:
+    | ts                  | symbol   | resolution   |   open |   high |    low |   close |   volume |   dividends |   stock_splits |
+    |:--------------------|:---------|:-------------|-------:|-------:|-------:|--------:|---------:|------------:|---------------:|
+    | 2024-03-01 05:00:00 | AAPL     | 1d           | 179.55 | 180.53 | 177.38 |  179.66 | 73488000 |           0 |              0 |
+    | 2024-03-04 05:00:00 | AAPL     | 1d           | 176.15 | 176.9  | 173.79 |  175.1  | 81510100 |           0 |              0 |
+    | 2024-03-05 05:00:00 | AAPL     | 1d           | 170.76 | 172.04 | 169.62 |  170.12 | 95132400 |           0 |              0 |
+
+
+
+### Download Historical Data on the Command Line Interface (CLI)
+```bash
+# download data, default data type (dtype) is 'raw' data
+pfeed download -d BYBIT -p BTC_USDT_PERP --start-date 2024-03-01 --end-date 2024-03-08
+
+# download multiple products BTC_USDT_PERP and ETH_USDT_PERP and minute data
+pfeed download -d BYBIT -p BTC_USDT_PERP -p ETH_USDT_PERP --dtype minute
+
+# download all perpetuals data from bybit
+pfeed download -d BYBIT --ptype PERP
+
+# download all the data from bybit (CAUTION: your local machine probably won't have enough space for this!)
+pfeed download -d BYBIT
+
+# store data into MinIO (need to start MinIO by running `pfeed docker-compose up -d` first)
+pfeed download -d BYBIT -p BTC_USDT_PERP --use-minio
+
+# enable debug mode and turn off using Ray
+pfeed download -d BYBIT -p BTC_USDT_PERP --debug --no-ray
+```
+
+### Download Historical Data in Python
+```python
+import pfeed as pe
+
+# compared to the CLI approach, this is more convenient for downloading multiple products
+pe.bybit.download(
+    pdts=[
+        'BTC_USDT_PERP',
+        'ETH_USDT_PERP',
+        'BCH_USDT_PERP',
+    ],
+    dtypes=['raw'],  # data types, e.g. 'raw', 'tick', 'second', 'minute' etc.
+    start_date='2024-03-01',
+    end_date='2024-03-08',
+    use_minio=False,
+)
+```
+
+### List Current Config
+```bash
+# list the current config:
+pfeed config --list
+
+# change the data storage location to your local project's 'data' folder:
+pfeed config --data-path ./data
+
+# for more commands:
+pfeed --help
+```
+
+### Run PFeed's docker-compose.yml
+```bash
+# same as 'docker-compose', only difference is it has pointed to pfeed's docker-compose.yml file
+pfeed docker-compose [COMMAND]
+
+# e.g. start services
+pfeed docker-compose up -d
+
+# e.g. stop services
+pfeed docker-compose down
+```
+
+
+## Supported Data Sources
+| Data Source               | Get Historical Data | Download Historical Data | Get Live/Paper Data | Stream Live/Paper Data |
+| ------------------------- | ------------------- | ------------------------ | ------------------- | ---------------------- |
+| Yahoo Finance             | 🟢                  | ⚪                       | ⚪                  | ⚪                     |
+| Bybit                     | 🟢                  | 🟢                       | 🟡                  | 🔴                     |
+| *Interactive Brokers (IB) | 🔴                  | ⚪                       | 🔴                  | 🔴                     |
+| *[FirstRate Data]         | 🔴                  | 🔴                       | ⚪                  | ⚪                     |
+| Binance                   | 🔴                  | 🔴                       | 🔴                  | 🔴                     |
+| OKX                       | 🔴                  | 🔴                       | 🔴                  | 🔴                     |
+
+[FirstRate Data]: https://firstratedata.com
+
+🟢 = finished \
+🟡 = in progress \
+🔴 = todo \
+⚪ = not applicable \
+\* = paid data
+
+
+## Related Projects
+- [PFund](https://github.com/PFund-Software-Ltd/pfund) — A Complete Algo-Trading Framework for Machine Learning, TradFi, CeFi and DeFi ready. Supports Vectorized and Event-Driven Backtesting, Paper and Live Trading
+- [PyTrade.org](https://pytrade.org) - A curated list of Python libraries and resources for algorithmic trading.

pfeed-0.0.1.dev5/README.md

@@ -0,0 +1,246 @@
+# PFeed: Data Pipeline for Algo-Trading, Getting and Storing Real-Time and Historical Data Made Easy.
+
+![GitHub stars](https://img.shields.io/github/stars/PFund-Software-Ltd/pfeed?style=social)
+![PyPI downloads](https://img.shields.io/pypi/dm/pfeed?label=downloads)
+[![PyPI](https://img.shields.io/pypi/v/pfeed.svg)](https://pypi.org/project/pfeed)
+![PyPI - Support Python Versions](https://img.shields.io/pypi/pyversions/pfeed)
+[![Jupyter Book Badge](https://raw.githubusercontent.com/PFund-Software-Ltd/pfeed/main/docs/images/jupyterbook.svg)](https://jupyterbook.org)
+[![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
+
+PFeed (/piː fiːd/) is a data integration library tailored for algorithmic trading, 
+serving as an ETL (Extract, Transform, Load) data pipeline between raw data sources and traders,
+helping them in creating a **local data lake for quantitative research**.
+
+PFeed allows traders to download historical, paper, and live data from various data sources, both free and paid,
+and stores them into a local data lake using [MinIO](https://min.io/).
+
+It is designed to be used alongside [PFund](https://github.com/PFund-Software-Ltd/pfund) — A Complete Algo-Trading Framework for Machine Learning, TradFi, CeFi and DeFi ready. Supports Vectorized and Event-Driven Backtesting, Paper and Live Trading, or as a standalone package.
+
+<details>
+<summary>Table of Contents</summary>
+
+- [Project Status](#project-status)
+- [Mission](#mission)
+- [Core Features](#core-features)
+- [Installation](#installation)
+- [Quick Start](#quick-start)
+    - [Main Usage: Data Feed](#main-usage-data-feed)
+    - [Download Historical Data on Command Line](#download-historical-data-on-command-line)
+    - [Download Historical Data in Python](#download-historical-data-in-python)
+    - [List Current Config](#list-current-config)
+    - [Run PFeed's docker-compose.yml](#run-pfeeds-docker-composeyml)
+- [Supported Data Sources](#supported-data-sources)
+- [Related Projects](#related-projects)
+
+</details>
+
+
+## Project Status
+**_Caution: PFeed is at a VERY EARLY stage, use it at your own risk._**
+
+PFeed is currently under active development, the framework design will be prioritized first over
+stability and scalability. 
+
+Please note that the available version is a *dev* version, not a *stable* one. \
+You are encouraged to play with the *dev* version, but only use it when a *stable* version is released.
+
+> PFeed for the time being only supports [Bybit](https://bybit.com/) and Yahoo Finance for testing purpose.
+
+## Mission
+Algo-trading has always been a complicated task due to the multitude of components and procedures involved. \
+Data collection and processing is probably the most mundane and yet critical part of it, as all results and findings 
+are derived from the data.
+
+However, preparing this data for use is not quick and easy. For example, sometimes even when the data is available (e.g. [Bybit data](https://public.bybit.com/trading/)), it is often in raw form and requires some cleaning.
+
+> PFeed's mission is to **_free traders from the tedious data work_** by providing cleaned data in a standard format that is ready for use, making them significantly faster to get to the analysis and strategy development phase.
+
+
+## Core Features
+- [x] Unified approach for interacting with various data sources, obtaining historical and real-time data
+- [x] ETL data pipline for transforming raw data and storing it in [MinIO](https://min.io/) (optional)
+- [x] Utilizes [Ray](https://github.com/ray-project/ray) for parallel data downloading
+- [x] Supports Pandas, [Polars](https://github.com/pola-rs/polars) as data tools
+- [ ] Integrates with [Prefect](https://www.prefect.io) to control data flows
+- [ ] Listens to PFund's trade engine and adds trade history to a local database [Timescaledb](https://www.timescale.com/) (optional)
+
+
+## Installation
+### Using [Poetry](https://python-poetry.org) (Recommended)
+```bash
+poetry add pfeed
+```
+
+### Using Pip
+```bash
+pip install pfeed
+```
+
+
+## Quick Start
+### Main Usage: Data Feed
+1. Download bybit raw data on the fly if not stored locally
+
+    ```python
+    import pfeed as pe
+
+    feed = pe.BybitFeed()
+
+    # df is a dataframe or a lazyframe (lazily loaded dataframe)
+    df = feed.get_historical_data(
+        'BTC_USDT_PERP',
+        resolution='raw',
+        start_date='2024-03-01',
+        end_date='2024-03-01',
+        data_tool='polars',  # or 'pandas'
+    )
+    ```
+
+    > By using pfeed, you are just one line of code away from playing with e.g. bybit data, how convenient!
+
+    Printing the first few rows of `df`:
+    |    | ts                            | symbol   |   side |   volume |   price | tickDirection   | trdMatchID                           |   grossValue |   homeNotional |   foreignNotional |
+    |---:|:------------------------------|:---------|-------:|---------:|--------:|:----------------|:-------------------------------------|-------------:|---------------:|------------------:|
+    |  0 | 2024-03-01 00:00:00.097599983 | BTCUSDT  |      1 |    0.003 | 61184.1 | ZeroMinusTick   | 79ac9a21-0249-5985-b042-906ec7604794 |  1.83552e+10 |          0.003 |           183.552 |
+    |  1 | 2024-03-01 00:00:00.098299980 | BTCUSDT  |      1 |    0.078 | 61184.9 | PlusTick        | 2af4e516-8ff4-5955-bb9c-38aa385b7b44 |  4.77242e+11 |          0.078 |          4772.42  |
+
+2. Get dataframe with different resolution, e.g. 1-minute data
+    ```python
+    import pfeed as pe
+
+    feed = pe.BybitFeed()
+
+    # df is a dataframe or a lazyframe (lazily loaded dataframe)
+    df = feed.get_historical_data(
+        'BTC_USDT_PERP',
+        resolution='1minute',  # or '1tick'/'1t', '2second'/'2s', '3minute'/'3m' etc.
+        start_date='2024-03-01',
+        end_date='2024-03-01',
+        data_tool='polars',
+    )
+    ```
+    > If you will be interacting with the data frequently, you should consider downloading it to your local machine.
+
+    Printing the first few rows of `df`:
+    |    | ts                  | product       | resolution   |    open |    high |     low |   close |   volume |
+    |---:|:--------------------|:--------------|:-------------|--------:|--------:|--------:|--------:|---------:|
+    |  0 | 2024-03-01 00:00:00 | BTC_USDT_PERP | 1m           | 61184.1 | 61244.5 | 61175.8 | 61244.5 |  159.142 |
+    |  1 | 2024-03-01 00:01:00 | BTC_USDT_PERP | 1m           | 61245.3 | 61276.5 | 61200.7 | 61232.2 |  227.242 |
+    |  2 | 2024-03-01 00:02:00 | BTC_USDT_PERP | 1m           | 61232.2 | 61249   | 61180   | 61184.2 |   91.446 |
+
+
+3. pfeed also supports simple wrapping of [yfinance](https://github.com/ranaroussi/yfinance)
+    ```python
+    import pfeed as pe
+
+    feed = pe.YahooFinanceFeed()
+
+    # you can still use any kwargs supported by yfinance's ticker.history(...)
+    # e.g. 'prepost', 'auto_adjust' etc.
+    yfinance_kwargs = {}
+
+    df = feed.get_historical_data(
+        'AAPL',
+        resolution='1d',
+        start_date='2024-03-01',
+        end_date='2024-03-20',
+        **yfinance_kwargs
+    )
+    ```
+    > Note that YahooFinanceFeed doesn't support the kwarg `data_tool`, e.g. polars
+    
+    Printing the first few rows of `df`:
+    | ts                  | symbol   | resolution   |   open |   high |    low |   close |   volume |   dividends |   stock_splits |
+    |:--------------------|:---------|:-------------|-------:|-------:|-------:|--------:|---------:|------------:|---------------:|
+    | 2024-03-01 05:00:00 | AAPL     | 1d           | 179.55 | 180.53 | 177.38 |  179.66 | 73488000 |           0 |              0 |
+    | 2024-03-04 05:00:00 | AAPL     | 1d           | 176.15 | 176.9  | 173.79 |  175.1  | 81510100 |           0 |              0 |
+    | 2024-03-05 05:00:00 | AAPL     | 1d           | 170.76 | 172.04 | 169.62 |  170.12 | 95132400 |           0 |              0 |
+
+
+
+### Download Historical Data on the Command Line Interface (CLI)
+```bash
+# download data, default data type (dtype) is 'raw' data
+pfeed download -d BYBIT -p BTC_USDT_PERP --start-date 2024-03-01 --end-date 2024-03-08
+
+# download multiple products BTC_USDT_PERP and ETH_USDT_PERP and minute data
+pfeed download -d BYBIT -p BTC_USDT_PERP -p ETH_USDT_PERP --dtype minute
+
+# download all perpetuals data from bybit
+pfeed download -d BYBIT --ptype PERP
+
+# download all the data from bybit (CAUTION: your local machine probably won't have enough space for this!)
+pfeed download -d BYBIT
+
+# store data into MinIO (need to start MinIO by running `pfeed docker-compose up -d` first)
+pfeed download -d BYBIT -p BTC_USDT_PERP --use-minio
+
+# enable debug mode and turn off using Ray
+pfeed download -d BYBIT -p BTC_USDT_PERP --debug --no-ray
+```
+
+### Download Historical Data in Python
+```python
+import pfeed as pe
+
+# compared to the CLI approach, this is more convenient for downloading multiple products
+pe.bybit.download(
+    pdts=[
+        'BTC_USDT_PERP',
+        'ETH_USDT_PERP',
+        'BCH_USDT_PERP',
+    ],
+    dtypes=['raw'],  # data types, e.g. 'raw', 'tick', 'second', 'minute' etc.
+    start_date='2024-03-01',
+    end_date='2024-03-08',
+    use_minio=False,
+)
+```
+
+### List Current Config
+```bash
+# list the current config:
+pfeed config --list
+
+# change the data storage location to your local project's 'data' folder:
+pfeed config --data-path ./data
+
+# for more commands:
+pfeed --help
+```
+
+### Run PFeed's docker-compose.yml
+```bash
+# same as 'docker-compose', only difference is it has pointed to pfeed's docker-compose.yml file
+pfeed docker-compose [COMMAND]
+
+# e.g. start services
+pfeed docker-compose up -d
+
+# e.g. stop services
+pfeed docker-compose down
+```
+
+
+## Supported Data Sources
+| Data Source               | Get Historical Data | Download Historical Data | Get Live/Paper Data | Stream Live/Paper Data |
+| ------------------------- | ------------------- | ------------------------ | ------------------- | ---------------------- |
+| Yahoo Finance             | 🟢                  | ⚪                       | ⚪                  | ⚪                     |
+| Bybit                     | 🟢                  | 🟢                       | 🟡                  | 🔴                     |
+| *Interactive Brokers (IB) | 🔴                  | ⚪                       | 🔴                  | 🔴                     |
+| *[FirstRate Data]         | 🔴                  | 🔴                       | ⚪                  | ⚪                     |
+| Binance                   | 🔴                  | 🔴                       | 🔴                  | 🔴                     |
+| OKX                       | 🔴                  | 🔴                       | 🔴                  | 🔴                     |
+
+[FirstRate Data]: https://firstratedata.com
+
+🟢 = finished \
+🟡 = in progress \
+🔴 = todo \
+⚪ = not applicable \
+\* = paid data
+
+
+## Related Projects
+- [PFund](https://github.com/PFund-Software-Ltd/pfund) — A Complete Algo-Trading Framework for Machine Learning, TradFi, CeFi and DeFi ready. Supports Vectorized and Event-Driven Backtesting, Paper and Live Trading
+- [PyTrade.org](https://pytrade.org) - A curated list of Python libraries and resources for algorithmic trading.

{pfeed-0.0.1.dev3 → pfeed-0.0.1.dev5}/pfeed/__init__.py

@@ -2,6 +2,7 @@ from pfeed.config_handler import configure
 from pfeed.sources import bybit
 from pfeed.feeds import YahooFinanceFeed, BybitFeed
 from importlib.metadata import version
+from pfeed import etl
 
 
 __version__ = version('pfeed')

{pfeed-0.0.1.dev3 → pfeed-0.0.1.dev5}/pfeed/cli/commands/config.py

@@ -9,15 +9,10 @@ from pfeed.const.paths import USER_CONFIG_FILE_PATH
 from pfeed.config_handler import ConfigHandler
 
 
-def load_config(config_file_path: str | Path):
-    config_file_path = Path(config_file_path)
-    if config_file_path.is_file():
-        with open(config_file_path, 'r') as f:
-            return yaml.safe_load(f) or {}
-    return {}
-
-
 def save_config(config: ConfigHandler, config_file_path: str | Path):
+    if type(config_file_path) is str:
+        config_file_path = Path(config_file_path)
+    config_file_path.parent.mkdir(parents=True, exist_ok=True)
     with open(config_file_path, 'w') as f:
         yaml.dump(config.__dict__, f, default_flow_style=False)
         
@@ -48,14 +43,16 @@ def config(ctx, **kwargs):
     if kwargs.get('list'):  # Check if --list was used
         del provided_options['list']
         assert not provided_options, "No options should be provided with --list"
-        click.echo(f"PFeed's config:\n{pformat(config.__dict__)}")
+        config_dict = config.__dict__
+        config_dict.update({'config_file_path': USER_CONFIG_FILE_PATH})
+        click.echo(f"PFeed's config:\n{pformat(config_dict)}")
         return
 
     if kwargs.get('reset'): # Check if --reset was used
         del provided_options['reset']
         assert not provided_options, "No options should be provided with --reset"
         remove_config(USER_CONFIG_FILE_PATH)
-        click.echo("Configuration successfully reset.")
+        click.echo("PFeed's config successfully reset.")
         return
     
     # prints out current config if no options are provided

{pfeed-0.0.1.dev3 → pfeed-0.0.1.dev5}/pfeed/cli/commands/docker_compose.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 import importlib.resources
 import subprocess
@@ -18,9 +19,15 @@ from pfeed.const.paths import PROJ_NAME
 def docker_compose(ctx, env_file_path, docker_file_path):
     """Forwards commands to docker-compose with the package's docker-compose.yml file if not specified."""
     if not env_file_path:
-        env_file_path = find_dotenv(usecwd=True, raise_error_if_not_found=True)
-        click.echo(f'.env file path is not specified, using env file in "{env_file_path}"')
-    load_dotenv(env_file_path)
+        if env_file_path := find_dotenv(usecwd=True, raise_error_if_not_found=False):
+            click.echo(f'.env file path is not specified, using env file in "{env_file_path}"')
+        else:
+            click.echo('.env file is not found')
+    load_dotenv(env_file_path, override=True)
+    
+    config = ctx.obj['config']
+    os.environ['PFEED_DATA_PATH'] = config.data_path
+    
     if not docker_file_path:
         package_dir = Path(importlib.resources.files(PROJ_NAME)).resolve().parents[0]
         docker_file_path = package_dir / 'docker-compose.yml'

pfeed-0.0.1.dev5/pfeed/cli/commands/download.py

@@ -0,0 +1,53 @@
+import importlib
+
+import click
+from dotenv import find_dotenv, load_dotenv
+
+from pfeed.const.commons import ALIASES, SUPPORTED_DOWNLOAD_DATA_SOURCES, SUPPORTED_DATA_TYPES
+
+
+# add aliases to supported download data sources
+SUPPORTED_DOWNLOAD_DATA_SOURCES_ALIASES_INCLUDED = SUPPORTED_DOWNLOAD_DATA_SOURCES + [k for k, v in ALIASES.items() if v in SUPPORTED_DOWNLOAD_DATA_SOURCES]
+
+# 'raw' data type is implicit since it doesn't have the timeframe specified, but still allow it for convenience
+# since for data source like bybit, there's only one raw data type, 'raw_tick', i.e. 'raw' will be converted to 'raw_tick'
+SUPPORTED_DATA_TYPES_IMPLICIT_RAW_ALLOWED = SUPPORTED_DATA_TYPES + ['raw']
+
+
+@click.command()
+@click.pass_context
+@click.option('--env-file', 'env_file_path', type=click.Path(exists=True), help='Path to the .env file')
+@click.option('--data-source', '-d', required=True, type=click.Choice(SUPPORTED_DOWNLOAD_DATA_SOURCES_ALIASES_INCLUDED, case_sensitive=False), help='Data source')
+@click.option('--dtype', '--dt', 'dtypes', multiple=True, default=['raw'], type=click.Choice(SUPPORTED_DATA_TYPES_IMPLICIT_RAW_ALLOWED, case_sensitive=False), help=f'{SUPPORTED_DATA_TYPES=}. How to pass in multiple values: --dt raw --dt tick')
+@click.option('--pdt', '-p', 'pdts', multiple=True, default=[], help='List of trading products')
+@click.option('--ptype', '--pt', 'ptypes', multiple=True, default=[], help='List of product types, e.g. PERP = get all perpetuals')
+@click.option('--start-date', '-s', type=click.DateTime(formats=["%Y-%m-%d"]), help='Start date in YYYY-MM-DD format')
+@click.option('--end-date', '-e', type=click.DateTime(formats=["%Y-%m-%d"]), help='End date in YYYY-MM-DD format')
+@click.option('--batch-size', default=8, type=int, help='batch size for Ray tasks')  # REVIEW
+@click.option('--no-ray', is_flag=True, help='if enabled, Ray will not be used')
+@click.option('--use-minio', is_flag=True, help='if enabled, data will be loaded into Minio')
+@click.option('--debug', is_flag=True, help='if enabled, debug mode will be enabled where logs at DEBUG level will be printed')
+def download(ctx, env_file_path, data_source, pdts, dtypes, ptypes, start_date, end_date, batch_size, no_ray, use_minio, debug):
+    if not env_file_path:
+        if env_file_path := find_dotenv(usecwd=True, raise_error_if_not_found=False):
+            click.echo(f'.env file path is not specified, using env file in "{env_file_path}"')
+        else:
+            click.echo('.env file is not found')
+    load_dotenv(env_file_path, override=True)
+    
+    if data_source in ALIASES:
+        data_source = ALIASES[data_source]
+        
+    pipeline = importlib.import_module(f'pfeed.sources.{data_source.lower()}.download')
+    pipeline.download_historical_data(
+        pdts=pdts,
+        dtypes=list(dtypes),
+        ptypes=list(ptypes),
+        start_date=start_date.date().strftime('%Y-%m-%d') if start_date else start_date,
+        end_date=end_date.date().strftime('%Y-%m-%d') if end_date else end_date,
+        batch_size=batch_size,
+        use_ray=not no_ray,
+        use_minio=use_minio,
+        debug=debug,
+        config=ctx.obj['config'],
+    )

{pfeed-0.0.1.dev3 → pfeed-0.0.1.dev5}/pfeed/cli/main.py

@@ -1,9 +1,8 @@
 import click
 
 from pfeed.config_handler import ConfigHandler
-from pfeed.const.paths import USER_CONFIG_FILE_PATH
 from pfeed.cli.commands.docker_compose import docker_compose
-from pfeed.cli.commands.config import config, load_config
+from pfeed.cli.commands.config import config
 from pfeed.cli.commands.download import download
 # from pfeed.cli.commands.stream import stream
 
@@ -14,8 +13,7 @@ from pfeed.cli.commands.download import download
 def pfeed_group(ctx):
     """PFeed's CLI"""
     ctx.ensure_object(dict)
-    config: dict = load_config(USER_CONFIG_FILE_PATH)
-    ctx.obj['config'] = ConfigHandler(**config)
+    ctx.obj['config'] = ConfigHandler.load_config()
 
 
 pfeed_group.add_command(docker_compose)