pfeed 0.0.1.dev9__tar.gz → 0.0.1.dev11__tar.gz

{pfeed-0.0.1.dev9 → pfeed-0.0.1.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pfeed
-Version: 0.0.1.dev9
+Version: 0.0.1.dev11
 Summary: Data pipeline for algo-trading, getting and storing both real-time and historical data made easy.
 Home-page: https://pfund.ai
 License: Apache-2.0
@@ -20,7 +20,7 @@ Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0.0)
 Requires-Dist: connectorx (>=0.3.2,<0.4.0) ; extra == "boost"
 Requires-Dist: minio (>=7.2.5,<8.0.0) ; extra == "data"
 Requires-Dist: pandas (>=2.2.0,<3.0.0) ; extra == "df"
-Requires-Dist: pfund (>=0.0.1.dev9,<0.0.2)
+Requires-Dist: pfund (>=0.0.1.dev13,<0.0.2)
 Requires-Dist: polars (>=0.20.16,<0.21.0) ; extra == "df"
 Requires-Dist: psutil (>=5.9.8,<6.0.0) ; extra == "data"
 Requires-Dist: pyarrow (>=15.0.0,<16.0.0) ; extra == "boost"

pfeed-0.0.1.dev11/pfeed/__init__.py

@@ -0,0 +1,45 @@
+from importlib.metadata import version
+
+from pfeed.config_handler import configure
+
+
+# NOTE: dynamically import modules to avoid click cli latency (reduced from ~4s to ~0.2s)
+def __getattr__(name):
+    """
+    Dynamically import and return modules and classes based on their name.
+    
+    Supports dynamic loading of data sources and feed classes to minimize
+    initial load time.
+
+    Supported Names:
+    - "bybit" -> Dynamically imports from pfeed.sources.bybit
+    - Includes any class containing 'Feed' from pfeed.feeds
+    """
+    import importlib
+    if 'Feed' in name:
+        Feed = getattr(importlib.import_module('pfeed.feeds'), name)
+        globals()[name] = Feed
+        return Feed
+    else:
+        name = name.lower()
+        data_source = importlib.import_module(f'pfeed.sources.{name}')
+        globals()[name] = data_source
+        return data_source
+    
+
+# NOTE: dummy classes/modules for type hinting
+# e.g. import pfeed as pe, when you type "pe.", 
+# you will still see the following suggestions even they are dynamically imported:
+bybit: ...
+YahooFinanceFeed: ...
+BybitFeed: ...
+
+
+__version__ = version('pfeed')
+__all__ = (
+    '__version__',
+    'configure',
+    'bybit',
+    'YahooFinanceFeed',
+    'BybitFeed',
+)

{pfeed-0.0.1.dev9 → pfeed-0.0.1.dev11}/pfeed/etl.py

@@ -1,4 +1,6 @@
 '''ETL = Extract, Transform, Load data'''
+from __future__ import annotations
+
 import io
 import logging
 import importlib

{pfeed-0.0.1.dev9 → pfeed-0.0.1.dev11}/pfeed/feeds/bybit_feed.py

@@ -1,21 +1,26 @@
 """High-level API for getting historical/streaming data from Bybit."""
+from __future__ import annotations
+
 import datetime
 
-from typing import Literal
+from typing import Literal, TYPE_CHECKING
+if TYPE_CHECKING:
+    from pfeed.types.common_literals import tSUPPORTED_DATA_TOOLS
 
-import polars as pl
-import pandas as pd
+try:
+    import pandas as pd
+    import polars as pl
+    from pfeed.data_tools.data_tool_polars import estimate_memory_usage
+except ImportError:
+    pass
 
-from pfeed import etl
 from pfeed.config_handler import ConfigHandler
 from pfeed.const.commons import SUPPORTED_DATA_TOOLS
-from pfeed.types.common_literals import tSUPPORTED_DATA_TOOLS
 from pfeed.feeds.base_feed import BaseFeed
 from pfeed.sources.bybit import api
 from pfeed.sources.bybit.const import DATA_SOURCE, SUPPORTED_PRODUCT_TYPES, create_efilename, SUPPORTED_RAW_DATA_TYPES
 from pfeed.utils.utils import get_dates_in_between, rollback_date_range
 from pfeed.utils.validate import validate_pdt
-from pfeed.data_tools.data_tool_polars import estimate_memory_usage
 # from pfund.exchanges.bybit.exchange import Exchange
 
 
@@ -55,7 +60,8 @@ class BybitFeed(BaseFeed):
                 If the memory usage exceeds the limit, the output DataFrame will be converted to a polars LazyFrame.
         """
         from pfund.datas.resolution import Resolution
-        
+        from pfeed import etl
+         
         # exchange = Exchange(env='LIVE')
         # adapter = exchange.adapter
         # product = exchange.create_product(*pdt.split('_'))

{pfeed-0.0.1.dev9 → pfeed-0.0.1.dev11}/pfeed/filepath.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from dataclasses import dataclass, field
 from pathlib import Path
 

pfeed-0.0.1.dev11/pfeed/main.py

@@ -0,0 +1,18 @@
+import atexit
+
+from pfeed.cli import pfeed_group
+
+
+def exit_cli():
+    """Application Exitpoint."""
+    print("Cleanup actions here...")
+
+
+def run_cli() -> None:
+    """Application Entrypoint."""
+    # atexit.register(exit_cli)
+    pfeed_group(obj={})
+
+
+if __name__ == '__main__':
+    run_cli()

{pfeed-0.0.1.dev9 → pfeed-0.0.1.dev11}/pfeed/sources/bybit/__init__.py

@@ -1,4 +1,3 @@
-from pfeed.sources.bybit import api
 from pfeed.sources.bybit.const import DATA_SOURCE as name
 from pfeed.sources.bybit.download import download_historical_data, download_historical_data as download
 from pfeed.sources.bybit.stream import stream_realtime_data, stream_realtime_data as stream

{pfeed-0.0.1.dev9 → pfeed-0.0.1.dev11}/pfeed/sources/bybit/download.py

@@ -7,7 +7,6 @@ from logging.handlers import QueueHandler, QueueListener
 from tqdm import tqdm
 from rich.console import Console
 
-from pfeed import etl
 from pfeed.config_handler import ConfigHandler
 from pfeed.utils.utils import get_dates_in_between
 from pfeed.utils.validate import validate_pdts_and_ptypes
@@ -39,6 +38,7 @@ def create_pdts_using_ptypes(ptypes) -> list[str]:
 
 
 def run_etl(product: BaseProduct, date, dtypes, use_minio):
+    from pfeed import etl
     pdt = product.pdt
     if raw_data := api.get_data(pdt, date):
         raw_tick: bytes = etl.clean_raw_data(DATA_SOURCE, raw_data)

{pfeed-0.0.1.dev9 → pfeed-0.0.1.dev11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pfeed"
-version = "0.0.1.dev9"
+version = "0.0.1.dev11"
 description = "Data pipeline for algo-trading, getting and storing both real-time and historical data made easy."
 license = "Apache-2.0"
 authors = ["Stephen Yau <softwareentrepreneer+pfeed@gmail.com>"]
@@ -12,7 +12,7 @@ keywords = ["trading", "algo-trading", "data pipeline", "ETL", "data lake", "dat
 
 [tool.poetry.dependencies]
 python = ">=3.10 <3.13"
-pfund = "^0.0.1.dev9"
+pfund = "^0.0.1.dev13"
 beautifulsoup4 = "^4.12.3"
 yfinance = "^0.2.37"
 psutil = { version = "^5.9.8", optional = true }
@@ -32,8 +32,30 @@ boost = ["pyarrow", "connectorx", "ray"]
 [tool.poetry.scripts]
 pfeed = "pfeed.main:run_cli"
 
+[tool.poetry.group.dev]
+optional = true
+
 [tool.poetry.group.dev.dependencies]
-pfund = {path = "../pfund", develop = true}
+# pfund = { path = "../pfund", develop = true }
+commitizen = "^3.24.0"
+mypy = "^1.9.0"
+ruff = "^0.3.5"
+grayskull = "^2.5.3"
+
+[tool.poetry.group.test.dependencies]
+pytest = "^8.0.0"
+pytest-xdist = "^3.5.0"
+pytest-mock = "^3.14.0"
+pytest-cov = "^5.0.0"
+pre-commit = "^3.6.1"
+tox = "^4.14.2"
+faker = "^24.4.0"
+bandit = "^1.7.7"
+
+[tool.poetry.group.doc.dependencies]
+jupyter-book = "^1.0.0"
+notebook = "^7.1.2"
+sphinxawesome-theme = "^5.1.1"
 
 [build-system]
 requires = ["poetry-core"]

pfeed-0.0.1.dev9/pfeed/__init__.py

@@ -1,17 +0,0 @@
-from pfeed.config_handler import configure
-from pfeed.sources import bybit
-from pfeed.feeds import YahooFinanceFeed, BybitFeed
-from importlib.metadata import version
-from pfeed import etl
-
-
-__version__ = version('pfeed')
-
-
-__all__ = (
-    '__version__',
-    'configure',
-    'bybit',
-    'YahooFinanceFeed',
-    'BybitFeed',
-)

pfeed-0.0.1.dev9/pfeed/main.py

@@ -1,10 +0,0 @@
-from pfeed.cli import pfeed_group
-
-
-def run_cli() -> None:
-    """Application Entrypoint."""
-    pfeed_group(obj={})
-
-
-if __name__ == '__main__':
-    run_cli()

pfeed-0.0.1.dev9/pfeed/sources/__init__.py

@@ -1 +0,0 @@
-from pfeed.sources import bybit