pfeed 0.0.1.dev8__tar.gz → 0.0.1.dev10__tar.gz

{pfeed-0.0.1.dev8 → pfeed-0.0.1.dev10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pfeed
-Version: 0.0.1.dev8
+Version: 0.0.1.dev10
 Summary: Data pipeline for algo-trading, getting and storing both real-time and historical data made easy.
 Home-page: https://pfund.ai
 License: Apache-2.0
@@ -14,17 +14,18 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: boost
+Provides-Extra: data
 Provides-Extra: df
 Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0.0)
 Requires-Dist: connectorx (>=0.3.2,<0.4.0) ; extra == "boost"
-Requires-Dist: minio (>=7.2.5,<8.0.0)
+Requires-Dist: minio (>=7.2.5,<8.0.0) ; extra == "data"
 Requires-Dist: pandas (>=2.2.0,<3.0.0) ; extra == "df"
-Requires-Dist: pfund (>=0.0.1.dev8,<0.0.2)
+Requires-Dist: pfund (>=0.0.1.dev10,<0.0.2)
 Requires-Dist: polars (>=0.20.16,<0.21.0) ; extra == "df"
+Requires-Dist: psutil (>=5.9.8,<6.0.0) ; extra == "data"
 Requires-Dist: pyarrow (>=15.0.0,<16.0.0) ; extra == "boost"
 Requires-Dist: ray (>=2.10.0,<3.0.0) ; extra == "boost"
-Requires-Dist: s3fs (>=2024.3.1,<2025.0.0)
-Requires-Dist: tqdm (>=4.66.2,<5.0.0)
+Requires-Dist: s3fs (>=2024.3.1,<2025.0.0) ; extra == "data"
 Requires-Dist: yfinance (>=0.2.37,<0.3.0)
 Project-URL: Documentation, https://pfeed-docs.pfund.ai
 Project-URL: Repository, https://github.com/PFund-Software-Ltd/pfeed
@@ -100,14 +101,28 @@ However, preparing this data for use is not quick and easy. For example, sometim
 ## Installation
 ### Using [Poetry](https://python-poetry.org) (Recommended)
 ```bash
+# [RECOMMENDED]: dataframes (e.g. polars, pandas) + data storage (e.g. MinIO) + boosted performance
+poetry add "pfeed[df,data,boost]"
+
+# only for downloading data, e.g. Bybit and Yahoo Finance
 poetry add pfeed
+
+# update to the latest version:
+poetry update pfeed
 ```
 
 ### Using Pip
 ```bash
 pip install pfeed
+
+# install the latest version:
+pip install -U pfeed
 ```
 
+### Checking your installation
+```bash
+$ pfeed --version
+```
 
 ## Quick Start
 ### Main Usage: Data Feed

{pfeed-0.0.1.dev8 → pfeed-0.0.1.dev10}/README.md

@@ -68,14 +68,28 @@ However, preparing this data for use is not quick and easy. For example, sometim
 ## Installation
 ### Using [Poetry](https://python-poetry.org) (Recommended)
 ```bash
+# [RECOMMENDED]: dataframes (e.g. polars, pandas) + data storage (e.g. MinIO) + boosted performance
+poetry add "pfeed[df,data,boost]"
+
+# only for downloading data, e.g. Bybit and Yahoo Finance
 poetry add pfeed
+
+# update to the latest version:
+poetry update pfeed
 ```
 
 ### Using Pip
 ```bash
 pip install pfeed
+
+# install the latest version:
+pip install -U pfeed
 ```
 
+### Checking your installation
+```bash
+$ pfeed --version
+```
 
 ## Quick Start
 ### Main Usage: Data Feed

{pfeed-0.0.1.dev8 → pfeed-0.0.1.dev10}/pfeed/etl.py

@@ -1,18 +1,28 @@
 '''ETL = Extract, Transform, Load data'''
+from __future__ import annotations
+
 import io
 import logging
 import importlib
 
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from pfeed.types.common_literals import tSUPPORTED_DATA_TOOLS, tSUPPORTED_DOWNLOAD_DATA_SOURCES, tSUPPORTED_DATA_SINKS, tSUPPORTED_DATA_TYPES, tSUPPORTED_DATA_MODES
+
 import pandas as pd
-from minio.error import MinioException
 
-from pfeed.datastore import Datastore
+try:
+    from pfeed.datastore import Datastore, MinioException
+except ImportError:
+    pass
 from pfeed.filepath import FilePath
 from pfeed.config_handler import ConfigHandler
 from pfeed.const.commons import SUPPORTED_DATA_TYPES, SUPPORTED_DATA_SINKS, SUPPORTED_DOWNLOAD_DATA_SOURCES, SUPPORTED_DATA_MODES
-from pfeed.types.common_literals import tSUPPORTED_DATA_TOOLS, tSUPPORTED_DOWNLOAD_DATA_SOURCES, tSUPPORTED_DATA_SINKS, tSUPPORTED_DATA_TYPES, tSUPPORTED_DATA_MODES
-from pfeed.utils.monitor import print_disk_usage
 from pfund.datas.resolution import Resolution
+try:
+    from pfeed.utils.monitor import print_disk_usage
+except ImportError:
+    print_disk_usage = None
 
 
 logger = logging.getLogger('pfeed')
@@ -47,7 +57,7 @@ def get_data(
 
     Returns:
         bytes | None: The extracted data as bytes, or None if the data is not found.
-    """
+    """    
     for data_sink in SUPPORTED_DATA_SINKS:
         try:
             data: bytes = extract_data(data_sink, data_source, dtype, pdt, date, mode=mode)
@@ -173,7 +183,8 @@ def load_data(
         logger.info(f'loaded {data_source} data to MinIO object {object_name} {kwargs=}')
     else:
         raise NotImplementedError(f'{data_sink=}')
-    print_disk_usage(config.data_path)
+    if print_disk_usage:
+        print_disk_usage(config.data_path)
         
 
 def clean_raw_data(data_source: tSUPPORTED_DOWNLOAD_DATA_SOURCES, raw_data: bytes) -> bytes:
@@ -209,5 +220,9 @@ def clean_raw_tick_data(raw_tick: bytes) -> bytes:
 
 
 def resample_data(data: bytes, resolution: str | Resolution, data_tool: tSUPPORTED_DATA_TOOLS='polars', check_if_drop_last_bar=False) -> bytes:
-    data_tool = importlib.import_module(f'pfeed.data_tools.data_tool_{data_tool.lower()}')
+    try:
+        data_tool = importlib.import_module(f'pfeed.data_tools.data_tool_{data_tool.lower()}')
+    except ImportError:
+        # fallback data_tool to pandas
+        data_tool = importlib.import_module('pfeed.data_tools.data_tool_pandas')
     return data_tool.resample_data(data, resolution, check_if_drop_last_bar=check_if_drop_last_bar)

{pfeed-0.0.1.dev8 → pfeed-0.0.1.dev10}/pfeed/feeds/bybit_feed.py

@@ -1,21 +1,27 @@
 """High-level API for getting historical/streaming data from Bybit."""
+from __future__ import annotations
+
 import datetime
 
-from typing import Literal
+from typing import Literal, TYPE_CHECKING
+if TYPE_CHECKING:
+    from pfeed.types.common_literals import tSUPPORTED_DATA_TOOLS
 
-import polars as pl
-import pandas as pd
+try:
+    import pandas as pd
+    import polars as pl
+    from pfeed.data_tools.data_tool_polars import estimate_memory_usage
+except ImportError:
+    pass
 
 from pfeed import etl
 from pfeed.config_handler import ConfigHandler
 from pfeed.const.commons import SUPPORTED_DATA_TOOLS
-from pfeed.types.common_literals import tSUPPORTED_DATA_TOOLS
 from pfeed.feeds.base_feed import BaseFeed
 from pfeed.sources.bybit import api
 from pfeed.sources.bybit.const import DATA_SOURCE, SUPPORTED_PRODUCT_TYPES, create_efilename, SUPPORTED_RAW_DATA_TYPES
 from pfeed.utils.utils import get_dates_in_between, rollback_date_range
 from pfeed.utils.validate import validate_pdt
-from pfeed.data_tools.data_tool_polars import estimate_memory_usage
 # from pfund.exchanges.bybit.exchange import Exchange
 
 

{pfeed-0.0.1.dev8 → pfeed-0.0.1.dev10}/pfeed/filepath.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from dataclasses import dataclass, field
 from pathlib import Path
 

{pfeed-0.0.1.dev8 → pfeed-0.0.1.dev10}/pfeed/sources/bybit/__init__.py

@@ -1,4 +1,3 @@
-from pfeed import etl
 from pfeed.sources.bybit import api
 from pfeed.sources.bybit.const import DATA_SOURCE as name
 from pfeed.sources.bybit.download import download_historical_data, download_historical_data as download

{pfeed-0.0.1.dev8 → pfeed-0.0.1.dev10}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pfeed"
-version = "0.0.1.dev8"
+version = "0.0.1.dev10"
 description = "Data pipeline for algo-trading, getting and storing both real-time and historical data made easy."
 license = "Apache-2.0"
 authors = ["Stephen Yau <softwareentrepreneer+pfeed@gmail.com>"]
@@ -12,20 +12,21 @@ keywords = ["trading", "algo-trading", "data pipeline", "ETL", "data lake", "dat
 
 [tool.poetry.dependencies]
 python = ">=3.10 <3.13"
-pfund = "^0.0.1.dev8"
+pfund = "^0.0.1.dev10"
 beautifulsoup4 = "^4.12.3"
-tqdm = "^4.66.2"
-s3fs = "^2024.3.1"
-minio = "^7.2.5"
 yfinance = "^0.2.37"
-pandas = { version = "^2.2.0", optional = true}
-polars = { version = "^0.20.16", optional = true}
-connectorx = { version = "^0.3.2", optional = true}
+psutil = { version = "^5.9.8", optional = true }
+s3fs = { version = "^2024.3.1", optional = true }
+minio = { version = "^7.2.5", optional = true }
+pandas = { version = "^2.2.0", optional = true }
+polars = { version = "^0.20.16", optional = true }
+connectorx = { version = "^0.3.2", optional = true }
 pyarrow = { version = "^15.0.0", optional = true }
-ray = { version = "^2.10.0", optional = true}
+ray = { version = "^2.10.0", optional = true }
 
 [tool.poetry.extras]
 df = ["pandas", "polars"]
+data = ["psutil", "minio", "s3fs"]
 boost = ["pyarrow", "connectorx", "ray"]
 
 [tool.poetry.scripts]