pfeed 0.0.1.dev7__tar.gz → 0.0.1.dev9__tar.gz

{pfeed-0.0.1.dev7 → pfeed-0.0.1.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pfeed
-Version: 0.0.1.dev7
+Version: 0.0.1.dev9
 Summary: Data pipeline for algo-trading, getting and storing both real-time and historical data made easy.
 Home-page: https://pfund.ai
 License: Apache-2.0
@@ -13,17 +13,19 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: boost
+Provides-Extra: data
+Provides-Extra: df
 Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0.0)
-Requires-Dist: click (>=8.1.7,<9.0.0)
-Requires-Dist: minio (>=7.2.5,<8.0.0)
-Requires-Dist: pfund (>=0.0.1.dev7,<0.0.2)
-Requires-Dist: platformdirs (>=4.2.0,<5.0.0)
-Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
-Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
-Requires-Dist: requests (>=2.31.0,<3.0.0)
-Requires-Dist: rich (>=13.7.0,<14.0.0)
-Requires-Dist: s3fs (>=2024.3.1,<2025.0.0)
-Requires-Dist: tqdm (>=4.66.2,<5.0.0)
+Requires-Dist: connectorx (>=0.3.2,<0.4.0) ; extra == "boost"
+Requires-Dist: minio (>=7.2.5,<8.0.0) ; extra == "data"
+Requires-Dist: pandas (>=2.2.0,<3.0.0) ; extra == "df"
+Requires-Dist: pfund (>=0.0.1.dev9,<0.0.2)
+Requires-Dist: polars (>=0.20.16,<0.21.0) ; extra == "df"
+Requires-Dist: psutil (>=5.9.8,<6.0.0) ; extra == "data"
+Requires-Dist: pyarrow (>=15.0.0,<16.0.0) ; extra == "boost"
+Requires-Dist: ray (>=2.10.0,<3.0.0) ; extra == "boost"
+Requires-Dist: s3fs (>=2024.3.1,<2025.0.0) ; extra == "data"
 Requires-Dist: yfinance (>=0.2.37,<0.3.0)
 Project-URL: Documentation, https://pfeed-docs.pfund.ai
 Project-URL: Repository, https://github.com/PFund-Software-Ltd/pfeed
@@ -99,14 +101,28 @@ However, preparing this data for use is not quick and easy. For example, sometim
 ## Installation
 ### Using [Poetry](https://python-poetry.org) (Recommended)
 ```bash
+# [RECOMMENDED]: dataframes (e.g. polars, pandas) + data storage (e.g. MinIO) + boosted performance
+poetry add "pfeed[df,data,boost]"
+
+# only for downloading data, e.g. Bybit and Yahoo Finance
 poetry add pfeed
+
+# update to the latest version:
+poetry update pfeed
 ```
 
 ### Using Pip
 ```bash
 pip install pfeed
+
+# install the latest version:
+pip install -U pfeed
 ```
 
+### Checking your installation
+```bash
+$ pfeed --version
+```
 
 ## Quick Start
 ### Main Usage: Data Feed

{pfeed-0.0.1.dev7 → pfeed-0.0.1.dev9}/README.md

@@ -68,14 +68,28 @@ However, preparing this data for use is not quick and easy. For example, sometim
 ## Installation
 ### Using [Poetry](https://python-poetry.org) (Recommended)
 ```bash
+# [RECOMMENDED]: dataframes (e.g. polars, pandas) + data storage (e.g. MinIO) + boosted performance
+poetry add "pfeed[df,data,boost]"
+
+# only for downloading data, e.g. Bybit and Yahoo Finance
 poetry add pfeed
+
+# update to the latest version:
+poetry update pfeed
 ```
 
 ### Using Pip
 ```bash
 pip install pfeed
+
+# install the latest version:
+pip install -U pfeed
 ```
 
+### Checking your installation
+```bash
+$ pfeed --version
+```
 
 ## Quick Start
 ### Main Usage: Data Feed

{pfeed-0.0.1.dev7 → pfeed-0.0.1.dev9}/pfeed/etl.py

@@ -3,16 +3,24 @@ import io
 import logging
 import importlib
 
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from pfeed.types.common_literals import tSUPPORTED_DATA_TOOLS, tSUPPORTED_DOWNLOAD_DATA_SOURCES, tSUPPORTED_DATA_SINKS, tSUPPORTED_DATA_TYPES, tSUPPORTED_DATA_MODES
+
 import pandas as pd
-from minio.error import MinioException
 
-from pfeed.datastore import Datastore
+try:
+    from pfeed.datastore import Datastore, MinioException
+except ImportError:
+    pass
 from pfeed.filepath import FilePath
 from pfeed.config_handler import ConfigHandler
 from pfeed.const.commons import SUPPORTED_DATA_TYPES, SUPPORTED_DATA_SINKS, SUPPORTED_DOWNLOAD_DATA_SOURCES, SUPPORTED_DATA_MODES
-from pfeed.types.common_literals import tSUPPORTED_DATA_TOOLS, tSUPPORTED_DOWNLOAD_DATA_SOURCES, tSUPPORTED_DATA_SINKS, tSUPPORTED_DATA_TYPES, tSUPPORTED_DATA_MODES
-from pfeed.utils.monitor import print_disk_usage
 from pfund.datas.resolution import Resolution
+try:
+    from pfeed.utils.monitor import print_disk_usage
+except ImportError:
+    print_disk_usage = None
 
 
 logger = logging.getLogger('pfeed')
@@ -47,7 +55,7 @@ def get_data(
 
     Returns:
         bytes | None: The extracted data as bytes, or None if the data is not found.
-    """
+    """    
     for data_sink in SUPPORTED_DATA_SINKS:
         try:
             data: bytes = extract_data(data_sink, data_source, dtype, pdt, date, mode=mode)
@@ -173,7 +181,8 @@ def load_data(
         logger.info(f'loaded {data_source} data to MinIO object {object_name} {kwargs=}')
     else:
         raise NotImplementedError(f'{data_sink=}')
-    print_disk_usage(config.data_path)
+    if print_disk_usage:
+        print_disk_usage(config.data_path)
         
 
 def clean_raw_data(data_source: tSUPPORTED_DOWNLOAD_DATA_SOURCES, raw_data: bytes) -> bytes:
@@ -209,5 +218,9 @@ def clean_raw_tick_data(raw_tick: bytes) -> bytes:
 
 
 def resample_data(data: bytes, resolution: str | Resolution, data_tool: tSUPPORTED_DATA_TOOLS='polars', check_if_drop_last_bar=False) -> bytes:
-    data_tool = importlib.import_module(f'pfeed.data_tools.data_tool_{data_tool.lower()}')
+    try:
+        data_tool = importlib.import_module(f'pfeed.data_tools.data_tool_{data_tool.lower()}')
+    except ImportError:
+        # fallback data_tool to pandas
+        data_tool = importlib.import_module('pfeed.data_tools.data_tool_pandas')
     return data_tool.resample_data(data, resolution, check_if_drop_last_bar=check_if_drop_last_bar)

{pfeed-0.0.1.dev7 → pfeed-0.0.1.dev9}/pfeed/sources/bybit/__init__.py

@@ -1,4 +1,3 @@
-from pfeed import etl
 from pfeed.sources.bybit import api
 from pfeed.sources.bybit.const import DATA_SOURCE as name
 from pfeed.sources.bybit.download import download_historical_data, download_historical_data as download

{pfeed-0.0.1.dev7 → pfeed-0.0.1.dev9}/pfeed/sources/bybit/download.py

@@ -1,5 +1,4 @@
 """Downloads Bybit historical data"""
-import os
 import logging
 import datetime
 from collections import defaultdict

{pfeed-0.0.1.dev7 → pfeed-0.0.1.dev9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pfeed"
-version = "0.0.1.dev7"
+version = "0.0.1.dev9"
 description = "Data pipeline for algo-trading, getting and storing both real-time and historical data made easy."
 license = "Apache-2.0"
 authors = ["Stephen Yau <softwareentrepreneer+pfeed@gmail.com>"]
@@ -12,51 +12,28 @@ keywords = ["trading", "algo-trading", "data pipeline", "ETL", "data lake", "dat
 
 [tool.poetry.dependencies]
 python = ">=3.10 <3.13"
-pfund = "^0.0.1.dev7"
-python-dotenv = "^1.0.1"
-pyyaml = "^6.0.1"
+pfund = "^0.0.1.dev9"
 beautifulsoup4 = "^4.12.3"
-requests = "^2.31.0"
-rich = "^13.7.0"
-tqdm = "^4.66.2"
-click = "^8.1.7"
-s3fs = "^2024.3.1"
-minio = "^7.2.5"
 yfinance = "^0.2.37"
-platformdirs = "^4.2.0"
+psutil = { version = "^5.9.8", optional = true }
+s3fs = { version = "^2024.3.1", optional = true }
+minio = { version = "^7.2.5", optional = true }
+pandas = { version = "^2.2.0", optional = true }
+polars = { version = "^0.20.16", optional = true }
+connectorx = { version = "^0.3.2", optional = true }
+pyarrow = { version = "^15.0.0", optional = true }
+ray = { version = "^2.10.0", optional = true }
+
+[tool.poetry.extras]
+df = ["pandas", "polars"]
+data = ["psutil", "minio", "s3fs"]
+boost = ["pyarrow", "connectorx", "ray"]
 
 [tool.poetry.scripts]
 pfeed = "pfeed.main:run_cli"
 
-[tool.poetry.group.pyodide-incompatible.dependencies]
-pandas = "^2.2.0"
-pyarrow = "^15.0.0"
-polars = "^0.20.16"
-connectorx = "^0.3.2"
-ray = "^2.10.0"
-
-[tool.poetry.group.dev]
-optional = true
-
 [tool.poetry.group.dev.dependencies]
 pfund = {path = "../pfund", develop = true}
-pytest = "^8.0.0"
-pre-commit = "^3.6.0"
-bandit = "^1.7.7"
-pytest-xdist = "^3.5.0"
-faker = "^24.4.0"
-tox = "^4.14.2"
-mypy = "^1.9.0"
-ruff = "^0.3.5"
-pytest-mock = "^3.14.0"
-pytest-cov = "^5.0.0"
-
-[tool.poetry.group.doc]
-optional = true
-
-[tool.poetry.group.doc.dependencies]
-jupyter-book = "^1.0.0"
-notebook = "^7.1.0"
 
 [build-system]
 requires = ["poetry-core"]