pfeed 0.0.2.dev3__tar.gz → 0.0.4__tar.gz

{pfeed-0.0.2.dev3 → pfeed-0.0.4}/PKG-INFO

@@ -1,58 +1,59 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: pfeed
-Version: 0.0.2.dev3
+Version: 0.0.4
 Summary: Data pipeline for algo-trading, getting and storing both real-time and historical data made easy.
-Home-page: https://pfund.ai
 License: Apache-2.0
-Keywords: trading,algo-trading,data pipeline,ETL,data lake,data warehouse,data integration,historical data,live data,data streaming
+Keywords: trading,algo-trading,data pipeline,ETL,data lake,data warehouse,data integration,historical data,live data,data streaming,data engineering
 Author: Stephen Yau
 Author-email: softwareentrepreneer+pfeed@gmail.com
-Requires-Python: >=3.10,<4.0
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Financial and Insurance Industry
+Classifier: Intended Audience :: Information Technology
+Classifier: Topic :: Office/Business :: Financial
+Classifier: Topic :: Office/Business :: Financial :: Investment
+Classifier: Topic :: Software Development :: Libraries
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Provides-Extra: all
+Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: core
 Provides-Extra: dask
-Provides-Extra: dataops
-Provides-Extra: dfs
-Provides-Extra: polars
+Provides-Extra: databento
+Provides-Extra: financial-modeling-prep
+Provides-Extra: fmp
 Provides-Extra: spark
-Provides-Extra: storage
-Requires-Dist: adlfs (>=2024.7.0,<2025.0.0) ; extra == "storage" or extra == "all"
 Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0.0)
-Requires-Dist: bytewax (>=0.21.0,<0.22.0) ; extra == "dataops" or extra == "all"
-Requires-Dist: coiled (>=1.54.0,<2.0.0) ; extra == "dask" or extra == "dfs" or extra == "all"
-Requires-Dist: confluent-kafka (>=2.5.3,<3.0.0) ; extra == "dataops" or extra == "all"
-Requires-Dist: connectorx (>=0.3.3,<0.4.0) ; extra == "storage" or extra == "all"
-Requires-Dist: dask[complete] (>=2024.9.1,<2025.0.0) ; extra == "dask" or extra == "dfs" or extra == "all"
-Requires-Dist: databento (>=0.42.0,<0.43.0) ; extra == "core" or extra == "all"
-Requires-Dist: databricks-connect (>=15.4.2,<16.0.0) ; extra == "spark" or extra == "dfs" or extra == "all"
-Requires-Dist: fastparquet (>=2024.2.0,<2025.0.0)
-Requires-Dist: gcsfs (>=2024.9.0,<2025.0.0) ; extra == "storage" or extra == "all"
-Requires-Dist: minio (>=7.2.8,<8.0.0) ; extra == "core" or extra == "all"
-Requires-Dist: modin[all] (>=0.32.0,<0.33.0) ; extra == "core" or extra == "all"
-Requires-Dist: pandas (>=2.2.0,<3.0.0)
-Requires-Dist: pfund (>=0.0.2.dev2,<0.0.3)
-Requires-Dist: polars (>=1.7.1,<2.0.0) ; extra == "polars" or extra == "dfs" or extra == "all"
-Requires-Dist: polars-xdt (>=0.16.0,<0.17.0) ; extra == "polars" or extra == "dfs" or extra == "all"
-Requires-Dist: polygon-api-client (>=1.14.2,<2.0.0) ; extra == "core" or extra == "all"
-Requires-Dist: prefect (>=3.0.5,<4.0.0) ; extra == "dataops" or extra == "all"
-Requires-Dist: psutil (>=6.0.0,<7.0.0) ; extra == "core" or extra == "all"
-Requires-Dist: psycopg2 (>=2.9.9,<3.0.0) ; extra == "storage" or extra == "all"
-Requires-Dist: pyarrow (>=17.0.0,<18.0.0) ; extra == "core" or extra == "all"
-Requires-Dist: pydantic (>=2.7.0,<3.0.0)
-Requires-Dist: pyspark (>=3.5.3,<4.0.0) ; extra == "spark" or extra == "dfs" or extra == "all"
-Requires-Dist: ray (>=2.35.0,<3.0.0) ; extra == "core" or extra == "all"
-Requires-Dist: s3fs (>=2024.9.0,<2025.0.0) ; extra == "storage" or extra == "all"
-Requires-Dist: yfinance (>=0.2.43,<0.3.0)
+Requires-Dist: bytewax (>=0.21.1,<0.22.0) ; extra == "core"
+Requires-Dist: coiled (>=1.89.0,<2.0.0) ; extra == "dask"
+Requires-Dist: confluent-kafka (>=2.8.2,<3.0.0) ; extra == "core"
+Requires-Dist: dask[complete,dataframe] (>=2025.3.0,<2026.0.0) ; extra == "dask"
+Requires-Dist: databento (>=0.50.0,<0.51.0) ; (python_version < "4.0.0") and (extra == "databento")
+Requires-Dist: deltalake (>=0.25.4,<0.26.0) ; extra == "core"
+Requires-Dist: duckdb (>=1.2.1,<2.0.0)
+Requires-Dist: fmp-api-client (>=0.0.1,<0.1.0) ; extra == "financial-modeling-prep"
+Requires-Dist: fmp-api-client (>=0.0.1,<0.1.0) ; extra == "fmp"
+Requires-Dist: minio (>=7.2.15,<8.0.0) ; extra == "core"
+Requires-Dist: narwhals (>=1.32.0,<2.0.0)
+Requires-Dist: pandas (>=2.2.3,<3.0.0)
+Requires-Dist: pandera (>=0.23.1,<0.24.0)
+Requires-Dist: pfund (>=0.0.2,<0.1.0)
+Requires-Dist: polars[polars-cloud] (>=1.26.0,<2.0.0)
+Requires-Dist: prefect (>=3.2.2,<4.0.0) ; extra == "core"
+Requires-Dist: pyarrow (>=18.0.0,<19.0.0)
+Requires-Dist: pydantic (>=2.10.6,<3.0.0)
+Requires-Dist: pyspark[connect,pandas-on-spark,sql] (>=3.5.3,<4.0.0) ; extra == "spark"
+Requires-Dist: ray (>=2.44.0,<3.0.0) ; extra == "core"
+Requires-Dist: yfinance (>=0.2.55,<0.3.0)
+Requires-Dist: zstandard (>=0.23.0,<0.24.0)
 Project-URL: Documentation, https://pfeed-docs.pfund.ai
+Project-URL: Homepage, https://pfund.ai
 Project-URL: Repository, https://github.com/PFund-Software-Ltd/pfeed
 Description-Content-Type: text/markdown
 
-# PFeed: Data Pipeline for Algo-Trading, Getting and Storing Real-Time and Historical Data Made Easy.
+# PFeed: The Single Source of Truth for Algo-Trading Data. Uniting Traders to Clean Once & Share with All.
 
 [![Twitter Follow](https://img.shields.io/twitter/follow/pfund_ai?style=social)](https://x.com/pfund_ai)
 ![GitHub stars](https://img.shields.io/github/stars/PFund-Software-Ltd/pfeed?style=social)
@@ -60,9 +61,10 @@ Description-Content-Type: text/markdown
 [![PyPI](https://img.shields.io/pypi/v/pfeed.svg)](https://pypi.org/project/pfeed)
 ![PyPI - Support Python Versions](https://img.shields.io/pypi/pyversions/pfeed)
 <!-- [![Jupyter Book Badge](https://raw.githubusercontent.com/PFund-Software-Ltd/pfeed/main/docs/images/jupyterbook.svg)](https://jupyterbook.org) -->
-[![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
+<!-- [![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/) -->
 
 [MinIO]: https://min.io/
+[Deltalake]: https://github.com/delta-io/delta-rs
 [PFund]: https://github.com/PFund-Software-Ltd/pfund
 [Polars]: https://github.com/pola-rs/polars
 [Dask]: https://www.dask.org/
@@ -75,21 +77,28 @@ Description-Content-Type: text/markdown
 [Databento]: https://databento.com/
 [Polygon]: https://polygon.io/
 [FirstRate Data]: https://firstratedata.com
+[Prefect]: https://www.prefect.io/
+[Bytewax]: https://www.bytewax.io
+
+> **This library is NOT stable yet. For a stable release, please wait for 0.1.0.** 
+
+## TLDR: use pfeed to manage your trading data; other traders will help you clean it
 
 ## Problem
-Starting algo-trading requires reliable, clean data. However, the time-consuming and mundane tasks of data cleaning and storage often discourage traders from embarking on their algo-trading journey.
+Starting algo-trading requires reliable and clean data, but traders often **work in silos**, each writing **duplicated code** to clean the same datasets—wasting time and effort. Accessing clean, ready-to-use data is a challenge, forcing traders to handle tedious data tasks before they can even start trading.
 
 ## Solution
-By leveraging modern data engineering tools, `pfeed` handles the tedious data work and **outputs backtesting-ready data**, allowing traders to focus on strategy development.
+`pfeed` leverages modern data engineering tools to centralize data cleaning efforts, automate ETL/ELT, store data in a **data lake with Delta Lake** support, and output **backtesting-ready data**, allowing traders to focus on strategy development.
 
 ---
-PFeed (/piː fiːd/) is a data pipeline for algorithmic trading, serving as a bridge between raw data sources and traders. It enables you to **download historical data**, **stream real-time data**, and **store cleaned data** in a **local data lake for quantitative analysis**, by automating the processes of data collection, cleaning, transformation, and storage.
+PFeed (/piː fiːd/) is the data engine for trading, serving as a pipeline between raw data sources and traders. It enables you to **download historical data**, **stream real-time data**, and **store cleaned data** in a **local data lake for quantitative analysis**, supporting both **batch processing** and **streaming** workflows through streamlined data collection, cleaning, transformation, and storage.
 
 ## Core Features
 - [x] Download or stream reliable, validated and **clean data** for research, backtesting, or live trading
 - [x] Get historical data (**dataframe**) or live data in standardized formats by just calling a **single** function
-- [x] **Own your data** by storing them locally using [MinIO], with the option to connect to the cloud
+- [x] **Own your data** by storing them locally using [MinIO] + [Deltalake], or in the cloud
 - [x] Interact with different kinds of data (including TradFi, CeFi and DeFi) using a **unified interface**
+- [x] Scale using modern data tools (e.g. [Polars], [Dask]) and workflow orchestration frameworks ([Prefect] for batch processing, [Bytewax] for streaming)
 
 ---
 
@@ -98,7 +107,7 @@ PFeed (/piː fiːd/) is a data pipeline for algorithmic trading, serving as a br
 
 - [Installation](#installation)
 - [Quick Start](#quick-start)
-    - [Get Historical Data in Dataframe](#1-get-historical-data-in-dataframe-no-storage)
+    - [Get Historical Data in Dataframe](#1-get-historical-data-in-dataframe)
     - [Download Historical Data on Command Line](#2-download-historical-data-on-the-command-line-interface-cli)
     - [Download Historical Data in Python](#3-download-historical-data-in-python)
 - [Supported Data Sources](#supported-data-sources)
@@ -112,67 +121,63 @@ PFeed (/piː fiːd/) is a data pipeline for algorithmic trading, serving as a br
 ## Installation
 > For more installation options, please refer to the [documentation](https://pfeed-docs.pfund.ai/installation).
 ```bash
-# [RECOMMENDED]: Full Features, choose this if you do not care about the package size
-pip install -U "pfeed[all]"
+# [RECOMMENDED]: Core Features, including Minio, Deltalake, Ray, etc.
+pip install -U "pfeed[core,prefect,bytewax]"
+
+# add your desired data sources, e.g. databento, polygon, etc.
+pip install -U "pfeed[core,databento,polygon]"
 
-# Minimal Features, only supports getting, downloading and streaming data
-pip install -U "pfeed[core]"
+# Minimal Features
+pip install -U "pfeed"
 ```
 
 
 
 ## Quick Start
-### 1. Get Historical Data in Dataframe (No storage)
-Get [Bybit]'s data in dataframe, e.g. 1-minute data (data is downloaded on the fly if not stored locally)
+### 1. Get Historical Data in Dataframe
+Get [Bybit]'s data in dataframe, e.g. 1-minute data (data is downloaded on the fly if not found in storage)
 
 ```python
 import pfeed as pe
 
-feed = pe.BybitFeed(data_tool='polars')
+bybit = pe.Bybit(data_tool='polars')
 
-df = feed.get_historical_data(
+df = bybit.get_historical_data(
     'BTC_USDT_PERP',
     resolution='1minute',  # 'raw' or '1tick'/'1t' or '2second'/'2s' etc.
-    start_date='2024-03-01',
-    end_date='2024-03-01',
+    start_date='2025-01-01',
+    end_date='2025-01-01',
 )
 ```
 
 Printing the first few rows of `df`:
-|    | ts                  | product       | resolution   |    open |    high |     low |   close |   volume |
-|---:|:--------------------|:--------------|:-------------|--------:|--------:|--------:|--------:|---------:|
-|  0 | 2024-03-01 00:00:00 | BTC_USDT_PERP | 1m           | 61184.1 | 61244.5 | 61175.8 | 61244.5 |  159.142 |
-|  1 | 2024-03-01 00:01:00 | BTC_USDT_PERP | 1m           | 61245.3 | 61276.5 | 61200.7 | 61232.2 |  227.242 |
-|  2 | 2024-03-01 00:02:00 | BTC_USDT_PERP | 1m           | 61232.2 | 61249   | 61180   | 61184.2 |   91.446 |
+| date                | resolution   | product       | symbol   |    open |    high |     low |   close |   volume |
+|:--------------------|:-------------|:--------------|:---------|--------:|--------:|--------:|--------:|---------:|
+| 2025-01-01 00:00:00 | 1m           | BTC_USDT_PERP | BTCUSDT  | 93530   | 93590.8 | 93501.3 | 93590.5 |   30.284 |
+| 2025-01-01 00:01:00 | 1m           | BTC_USDT_PERP | BTCUSDT  | 93590.5 | 93627.7 | 93571.8 | 93625   |   30.334 |
 
-> By using pfeed, you are just a few lines of code away from getting a standardized dataframe, how convenient!
+> By using pfeed, you are just **one function call** away from getting a standardized dataframe
 
 ### 2. Download Historical Data on the Command Line Interface (CLI)
 > For more CLI commands, please refer to the [documentation](https://pfeed-docs.pfund.ai/cli-commands).
 ```bash
-# download data, default data type (dtype) is 'raw' data
-pfeed download -d BYBIT -p BTC_USDT_PERP --start-date 2024-03-01 --end-date 2024-03-08
+# download BTC tick data
+pfeed download -d BYBIT -p BTC_USDT_PERP -r tick --start-date 2025-01-01 --end-date 2025-02-01
 
-# download multiple products BTC_USDT_PERP and ETH_USDT_PERP as minute data and store them locally
-pfeed download -d BYBIT -p BTC_USDT_PERP -p ETH_USDT_PERP --dtypes minute --use-minio
+# download data and store it in MinIO
+pfeed download -d BYBIT -p BTC_USDT_PERP --storage minio
 ```
 
 ### 3. Download Historical Data in Python
 ```python
 import pfeed as pe
 
-# compared to the CLI approach, this approach is more convenient for downloading multiple products
-pe.download(
-    data_source='bybit',
-    pdts=[
-        'BTC_USDT_PERP',
-        'ETH_USDT_PERP',
-        'BCH_USDT_PERP',
-    ],
-    dtypes=['raw'],  # data types, e.g. 'raw', 'tick', 'second', 'minute' etc.
-    start_date='2024-03-01',
-    end_date='2024-03-08',
-    use_minio=False,
+bybit = pe.Bybit()
+bybit.download(
+    product='BTC_USDT_PERP',
+    resolution='1s',  # 1-second data
+    rollback_period='1w',  # rollback 1 week
+    to_storage='local',
 )
 ```
 

{pfeed-0.0.2.dev3 → pfeed-0.0.4}/README.md

@@ -1,4 +1,4 @@
-# PFeed: Data Pipeline for Algo-Trading, Getting and Storing Real-Time and Historical Data Made Easy.
+# PFeed: The Single Source of Truth for Algo-Trading Data. Uniting Traders to Clean Once & Share with All.
 
 [![Twitter Follow](https://img.shields.io/twitter/follow/pfund_ai?style=social)](https://x.com/pfund_ai)
 ![GitHub stars](https://img.shields.io/github/stars/PFund-Software-Ltd/pfeed?style=social)
@@ -6,9 +6,10 @@
 [![PyPI](https://img.shields.io/pypi/v/pfeed.svg)](https://pypi.org/project/pfeed)
 ![PyPI - Support Python Versions](https://img.shields.io/pypi/pyversions/pfeed)
 <!-- [![Jupyter Book Badge](https://raw.githubusercontent.com/PFund-Software-Ltd/pfeed/main/docs/images/jupyterbook.svg)](https://jupyterbook.org) -->
-[![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
+<!-- [![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/) -->
 
 [MinIO]: https://min.io/
+[Deltalake]: https://github.com/delta-io/delta-rs
 [PFund]: https://github.com/PFund-Software-Ltd/pfund
 [Polars]: https://github.com/pola-rs/polars
 [Dask]: https://www.dask.org/
@@ -21,21 +22,28 @@
 [Databento]: https://databento.com/
 [Polygon]: https://polygon.io/
 [FirstRate Data]: https://firstratedata.com
+[Prefect]: https://www.prefect.io/
+[Bytewax]: https://www.bytewax.io
+
+> **This library is NOT stable yet. For a stable release, please wait for 0.1.0.** 
+
+## TLDR: use pfeed to manage your trading data; other traders will help you clean it
 
 ## Problem
-Starting algo-trading requires reliable, clean data. However, the time-consuming and mundane tasks of data cleaning and storage often discourage traders from embarking on their algo-trading journey.
+Starting algo-trading requires reliable and clean data, but traders often **work in silos**, each writing **duplicated code** to clean the same datasets—wasting time and effort. Accessing clean, ready-to-use data is a challenge, forcing traders to handle tedious data tasks before they can even start trading.
 
 ## Solution
-By leveraging modern data engineering tools, `pfeed` handles the tedious data work and **outputs backtesting-ready data**, allowing traders to focus on strategy development.
+`pfeed` leverages modern data engineering tools to centralize data cleaning efforts, automate ETL/ELT, store data in a **data lake with Delta Lake** support, and output **backtesting-ready data**, allowing traders to focus on strategy development.
 
 ---
-PFeed (/piː fiːd/) is a data pipeline for algorithmic trading, serving as a bridge between raw data sources and traders. It enables you to **download historical data**, **stream real-time data**, and **store cleaned data** in a **local data lake for quantitative analysis**, by automating the processes of data collection, cleaning, transformation, and storage.
+PFeed (/piː fiːd/) is the data engine for trading, serving as a pipeline between raw data sources and traders. It enables you to **download historical data**, **stream real-time data**, and **store cleaned data** in a **local data lake for quantitative analysis**, supporting both **batch processing** and **streaming** workflows through streamlined data collection, cleaning, transformation, and storage.
 
 ## Core Features
 - [x] Download or stream reliable, validated and **clean data** for research, backtesting, or live trading
 - [x] Get historical data (**dataframe**) or live data in standardized formats by just calling a **single** function
-- [x] **Own your data** by storing them locally using [MinIO], with the option to connect to the cloud
+- [x] **Own your data** by storing them locally using [MinIO] + [Deltalake], or in the cloud
 - [x] Interact with different kinds of data (including TradFi, CeFi and DeFi) using a **unified interface**
+- [x] Scale using modern data tools (e.g. [Polars], [Dask]) and workflow orchestration frameworks ([Prefect] for batch processing, [Bytewax] for streaming)
 
 ---
 
@@ -44,7 +52,7 @@ PFeed (/piː fiːd/) is a data pipeline for algorithmic trading, serving as a br
 
 - [Installation](#installation)
 - [Quick Start](#quick-start)
-    - [Get Historical Data in Dataframe](#1-get-historical-data-in-dataframe-no-storage)
+    - [Get Historical Data in Dataframe](#1-get-historical-data-in-dataframe)
     - [Download Historical Data on Command Line](#2-download-historical-data-on-the-command-line-interface-cli)
     - [Download Historical Data in Python](#3-download-historical-data-in-python)
 - [Supported Data Sources](#supported-data-sources)
@@ -58,67 +66,63 @@ PFeed (/piː fiːd/) is a data pipeline for algorithmic trading, serving as a br
 ## Installation
 > For more installation options, please refer to the [documentation](https://pfeed-docs.pfund.ai/installation).
 ```bash
-# [RECOMMENDED]: Full Features, choose this if you do not care about the package size
-pip install -U "pfeed[all]"
+# [RECOMMENDED]: Core Features, including Minio, Deltalake, Ray, etc.
+pip install -U "pfeed[core,prefect,bytewax]"
+
+# add your desired data sources, e.g. databento, polygon, etc.
+pip install -U "pfeed[core,databento,polygon]"
 
-# Minimal Features, only supports getting, downloading and streaming data
-pip install -U "pfeed[core]"
+# Minimal Features
+pip install -U "pfeed"
 ```
 
 
 
 ## Quick Start
-### 1. Get Historical Data in Dataframe (No storage)
-Get [Bybit]'s data in dataframe, e.g. 1-minute data (data is downloaded on the fly if not stored locally)
+### 1. Get Historical Data in Dataframe
+Get [Bybit]'s data in dataframe, e.g. 1-minute data (data is downloaded on the fly if not found in storage)
 
 ```python
 import pfeed as pe
 
-feed = pe.BybitFeed(data_tool='polars')
+bybit = pe.Bybit(data_tool='polars')
 
-df = feed.get_historical_data(
+df = bybit.get_historical_data(
     'BTC_USDT_PERP',
     resolution='1minute',  # 'raw' or '1tick'/'1t' or '2second'/'2s' etc.
-    start_date='2024-03-01',
-    end_date='2024-03-01',
+    start_date='2025-01-01',
+    end_date='2025-01-01',
 )
 ```
 
 Printing the first few rows of `df`:
-|    | ts                  | product       | resolution   |    open |    high |     low |   close |   volume |
-|---:|:--------------------|:--------------|:-------------|--------:|--------:|--------:|--------:|---------:|
-|  0 | 2024-03-01 00:00:00 | BTC_USDT_PERP | 1m           | 61184.1 | 61244.5 | 61175.8 | 61244.5 |  159.142 |
-|  1 | 2024-03-01 00:01:00 | BTC_USDT_PERP | 1m           | 61245.3 | 61276.5 | 61200.7 | 61232.2 |  227.242 |
-|  2 | 2024-03-01 00:02:00 | BTC_USDT_PERP | 1m           | 61232.2 | 61249   | 61180   | 61184.2 |   91.446 |
+| date                | resolution   | product       | symbol   |    open |    high |     low |   close |   volume |
+|:--------------------|:-------------|:--------------|:---------|--------:|--------:|--------:|--------:|---------:|
+| 2025-01-01 00:00:00 | 1m           | BTC_USDT_PERP | BTCUSDT  | 93530   | 93590.8 | 93501.3 | 93590.5 |   30.284 |
+| 2025-01-01 00:01:00 | 1m           | BTC_USDT_PERP | BTCUSDT  | 93590.5 | 93627.7 | 93571.8 | 93625   |   30.334 |
 
-> By using pfeed, you are just a few lines of code away from getting a standardized dataframe, how convenient!
+> By using pfeed, you are just **one function call** away from getting a standardized dataframe
 
 ### 2. Download Historical Data on the Command Line Interface (CLI)
 > For more CLI commands, please refer to the [documentation](https://pfeed-docs.pfund.ai/cli-commands).
 ```bash
-# download data, default data type (dtype) is 'raw' data
-pfeed download -d BYBIT -p BTC_USDT_PERP --start-date 2024-03-01 --end-date 2024-03-08
+# download BTC tick data
+pfeed download -d BYBIT -p BTC_USDT_PERP -r tick --start-date 2025-01-01 --end-date 2025-02-01
 
-# download multiple products BTC_USDT_PERP and ETH_USDT_PERP as minute data and store them locally
-pfeed download -d BYBIT -p BTC_USDT_PERP -p ETH_USDT_PERP --dtypes minute --use-minio
+# download data and store it in MinIO
+pfeed download -d BYBIT -p BTC_USDT_PERP --storage minio
 ```
 
 ### 3. Download Historical Data in Python
 ```python
 import pfeed as pe
 
-# compared to the CLI approach, this approach is more convenient for downloading multiple products
-pe.download(
-    data_source='bybit',
-    pdts=[
-        'BTC_USDT_PERP',
-        'ETH_USDT_PERP',
-        'BCH_USDT_PERP',
-    ],
-    dtypes=['raw'],  # data types, e.g. 'raw', 'tick', 'second', 'minute' etc.
-    start_date='2024-03-01',
-    end_date='2024-03-08',
-    use_minio=False,
+bybit = pe.Bybit()
+bybit.download(
+    product='BTC_USDT_PERP',
+    resolution='1s',  # 1-second data
+    rollback_period='1w',  # rollback 1 week
+    to_storage='local',
 )
 ```
 

pfeed-0.0.4/docker-compose.yml

@@ -0,0 +1,29 @@
+name: pfeed
+services:
+  minio:
+    container_name: pe-minio
+    image: minio/minio:RELEASE.2025-01-20T14-49-07Z
+    ports:
+      - "${MINIO_PORT:-9000}:9000"
+      - "${MINIO_CONSOLE_PORT:-9001}:9001"
+    volumes:
+      - ${MINIO_DATA_PATH}:/data
+    environment:
+      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-pfunder}
+      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-password}
+    command: server /data --console-address ":9001"
+    profiles:
+      - minio
+  # timescaledb:
+  #   container_name: pe-timescaledb
+  #   image: timescaledev/timescaledb-ha:pg17.2-ts2.17.2-all-oss
+  #   ports:
+  #     - "${TIMESCALEDB_PORT:-5432}:5432"
+  #   volumes:
+  #     - ${TIMESCALEDB_DATA_PATH}/timescaledb:/var/lib/postgresql/data
+  #   environment:
+  #     POSTGRES_DB: ${TIMESCALEDB_DB:-pfeed}
+  #     POSTGRES_USER: ${TIMESCALEDB_USER:-pfunder}
+  #     POSTGRES_PASSWORD: ${TIMESCALEDB_PASSWORD:-password}
+  #   profiles:
+  #     - timescaledb

pfeed-0.0.4/logging.yml

@@ -0,0 +1,56 @@
+version: 1
+# filename_format: '%Y-%m-%d_UTC%z'
+# incremental False = replaces the existing configuration
+incremental: False  
+# disable_existing_loggers will be ignored if incremental is True
+disable_existing_loggers: False
+loggers:
+  root:
+    level: 'WARNING'
+    handlers: ['compressed_timed_rotating_file_handler', 'stream_path_handler']
+    propagate: False
+  pfeed:
+    level: 'DEBUG'
+    handlers: ['compressed_timed_rotating_file_handler', 'stream_handler']
+    propagate: False
+  # NOTE: avoid name conflict with pfund, add "_data" suffix to e.g. bybit
+  yahoo_finance_data:
+    level: 'DEBUG'
+    handlers: ['compressed_timed_rotating_file_handler', 'stream_handler']
+    propagate: False
+  bybit_data:
+    level: 'DEBUG'
+    handlers: ['compressed_timed_rotating_file_handler', 'stream_handler']
+    propagate: False
+  databento_data:
+    level: 'DEBUG'
+    handlers: ['compressed_timed_rotating_file_handler', 'stream_handler']
+    propagate: False
+handlers:
+  file_handler:
+    class: 'logging.FileHandler'
+    level: 'DEBUG'
+    formatter: 'file'
+  compressed_timed_rotating_file_handler:
+    class: 'pfund.plogging.handlers.CompressedTimedRotatingFileHandler'
+    level: 'DEBUG'
+    formatter: 'file'
+    kwargs: {'when': 'midnight', 'backupCount': 7, 'utc': True, 'encoding': 'utf-8'}
+  stream_handler:
+    class: 'logging.StreamHandler'
+    level: 'INFO'
+    formatter: 'console'
+  stream_path_handler:
+    class: 'logging.StreamHandler'
+    level: 'INFO'
+    formatter: 'path'
+formatters:
+  path:
+    format: '%(asctime)s.%(msecs)03d | %(levelname)s | %(name)s | %(message)s | %(shortpath)s fn:%(funcName)s ln:%(lineno)d'
+    datefmt: '%Y-%m-%dT%H:%M:%S%z'
+  file:
+    format: '%(asctime)s.%(msecs)03d | %(levelname)s | %(message)s | %(filename)s fn:%(funcName)s ln:%(lineno)d'
+    datefmt: '%H:%M:%S%z'
+  console:
+    format: '%(asctime)s.%(msecs)03d | %(levelname)s | %(name)s | %(message)s | %(filename)s fn:%(funcName)s ln:%(lineno)d'
+    datefmt: '%Y-%m-%dT%H:%M:%S%z'

pfeed-0.0.4/pfeed/__init__.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from pfund_plugins.base_plugin import BasePlugin
+    # need these imports to support IDE hints:
+    aliases = ...
+    from pfeed.feeds.yahoo_finance.yahoo_finance import (
+        YahooFinance,
+        YahooFinance as YF,
+    )
+    from pfeed.feeds.bybit.bybit import (
+        BybitMarketFeed as Bybit,
+    )
+    from pfeed.feeds.financial_modeling_prep.financial_modeling_prep import (
+        FinancialModelingPrep,
+        FinancialModelingPrep as FMP,
+    )
+
+from importlib.metadata import version
+from pfeed.config import configure, get_config
+
+
+
+plugins = {}
+def add_plugin(plugin: BasePlugin):
+    plugins[plugin.name] = plugin
+    
+
+
+def __getattr__(name: str):
+    if name == 'aliases':
+        from pfeed.const.aliases import ALIASES
+        from pfund.const.aliases import ALIASES as PFUND_ALIASES
+        return {**ALIASES, **PFUND_ALIASES}
+    elif name in ('YahooFinance', 'YF'):
+        from pfeed.feeds.yahoo_finance.yahoo_finance import YahooFinance
+        return YahooFinance
+    elif name in ('Bybit',):
+        from pfeed.feeds.bybit.bybit import BybitMarketFeed as Bybit
+        return Bybit
+    elif name in ('FinancialModelingPrep', 'FMP'):
+        from pfeed.feeds.financial_modeling_prep.financial_modeling_prep import FinancialModelingPrep
+        return FinancialModelingPrep
+    elif name in plugins:
+        return plugins[name]
+    raise AttributeError(f"'{__name__}' object has no attribute '{name}'")
+    
+    
+# TODO: add llm plugin to explain "what is xxx?"
+def what_is(alias: str) -> str | None:
+    from pfeed.const.aliases import ALIASES
+    from pfund.const.aliases import ALIASES as PFUND_ALIASES
+    if alias in PFUND_ALIASES or alias.upper() in PFUND_ALIASES:
+        return PFUND_ALIASES.get(alias, PFUND_ALIASES.get(alias.upper(), None))
+    elif alias in ALIASES or alias.upper() in ALIASES:
+        return aliases.get(alias, aliases.get(alias.upper(), None))
+
+
+__version__ = version("pfeed")
+__all__ = (
+    "__version__",
+    "configure",
+    "get_config",
+    "add_plugin",
+    "aliases",
+    "what_is",
+    "YahooFinance", "YF",
+    "Bybit",
+    "FinancialModelingPrep", "FMP",
+)