pfeed 0.0.1.dev6__tar.gz → 0.0.1.dev7__tar.gz

{pfeed-0.0.1.dev6 → pfeed-0.0.1.dev7}/PKG-INFO

@@ -1,29 +1,25 @@
 Metadata-Version: 2.1
 Name: pfeed
-Version: 0.0.1.dev6
+Version: 0.0.1.dev7
 Summary: Data pipeline for algo-trading, getting and storing both real-time and historical data made easy.
 Home-page: https://pfund.ai
 License: Apache-2.0
 Keywords: trading,algo-trading,data pipeline,ETL,data lake,data warehouse,data integration,historical data,live data,data streaming
 Author: Stephen Yau
 Author-email: softwareentrepreneer+pfeed@gmail.com
-Requires-Python: >=3.10,<3.12
+Requires-Python: >=3.10,<3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
-Requires-Dist: connectorx (>=0.3.2,<0.4.0)
 Requires-Dist: minio (>=7.2.5,<8.0.0)
-Requires-Dist: pandas (>=2.2.0,<3.0.0)
-Requires-Dist: pfund (>=0.0.1.dev5,<0.0.2)
+Requires-Dist: pfund (>=0.0.1.dev7,<0.0.2)
 Requires-Dist: platformdirs (>=4.2.0,<5.0.0)
-Requires-Dist: polars (>=0.20.16,<0.21.0)
-Requires-Dist: pyarrow (>=15.0.0,<16.0.0)
 Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
-Requires-Dist: ray (>=2.10.0,<3.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: rich (>=13.7.0,<14.0.0)
 Requires-Dist: s3fs (>=2024.3.1,<2025.0.0)

{pfeed-0.0.1.dev6 → pfeed-0.0.1.dev7}/pfeed/const/commons.py

@@ -3,8 +3,9 @@ SUPPORTED_DATA_TYPES = ['raw_tick', 'raw_second', 'raw_minute', 'raw_hour', 'raw
                         'tick', 'second', 'minute', 'hour', 'daily']
 SUPPORTED_DATA_SINKS = ['local', 'minio']
 SUPPORTED_DOWNLOAD_DATA_SOURCES = ['BYBIT']
+SUPPORTED_DATA_MODES = ['historical', 'streaming']
+SUPPORTED_DATA_TOOLS = ['pandas', 'polars', 'pyspark']
+
 ALIASES = {
     'YF': 'YAHOO_FINANCE',
-}
-SUPPORTED_DATA_MODES = ['historical', 'streaming']
-SUPPORTED_DATA_TOOLS = ['pandas', 'polars', 'pyspark']
+}

{pfeed-0.0.1.dev6 → pfeed-0.0.1.dev7}/pfeed/etl.py

@@ -3,8 +3,6 @@ import io
 import logging
 import importlib
 
-from typing import Literal
-
 import pandas as pd
 from minio.error import MinioException
 
@@ -12,17 +10,11 @@ from pfeed.datastore import Datastore
 from pfeed.filepath import FilePath
 from pfeed.config_handler import ConfigHandler
 from pfeed.const.commons import SUPPORTED_DATA_TYPES, SUPPORTED_DATA_SINKS, SUPPORTED_DOWNLOAD_DATA_SOURCES, SUPPORTED_DATA_MODES
+from pfeed.types.common_literals import tSUPPORTED_DATA_TOOLS, tSUPPORTED_DOWNLOAD_DATA_SOURCES, tSUPPORTED_DATA_SINKS, tSUPPORTED_DATA_TYPES, tSUPPORTED_DATA_MODES
 from pfeed.utils.monitor import print_disk_usage
 from pfund.datas.resolution import Resolution
 
 
-DataSink = Literal['local', 'minio']
-DataSource = Literal['BYBIT']
-DataTool = Literal['pandas', 'polars', 'pyspark']
-DataType = Literal['raw_tick', 'raw_second', 'raw_minute', 'raw_hour', 'raw_daily', 'raw', 'tick', 'second', 'minute', 'hour', 'daily']
-DataMode = Literal['historical', 'streaming']
-
-
 logger = logging.getLogger('pfeed')
 
 
@@ -37,11 +29,11 @@ def _convert_raw_dtype_to_explicit(data_source: str, dtype: str):
 
 
 def get_data(
-    data_source: DataSource,
-    dtype: DataType,
+    data_source: tSUPPORTED_DOWNLOAD_DATA_SOURCES,
+    dtype: tSUPPORTED_DATA_TYPES,
     pdt: str,
     date: str,
-    mode: DataMode='historical',
+    mode: tSUPPORTED_DATA_MODES='historical',
 ) -> bytes | None:
     """Extract data without specifying the data origin. 
     This function will try to extract data from all supported data origins.
@@ -68,12 +60,12 @@ def get_data(
 
 
 def extract_data(
-    data_sink: DataSink,
-    data_source: DataSource,
-    dtype: DataType,
+    data_sink: tSUPPORTED_DATA_SINKS,
+    data_source: tSUPPORTED_DOWNLOAD_DATA_SOURCES,
+    dtype: tSUPPORTED_DATA_TYPES,
     pdt: str,
     date: str,
-    mode: DataMode='historical',
+    mode: tSUPPORTED_DATA_MODES='historical',
 ) -> bytes | None:
     """
     Extracts data from a specified data source and returns it as bytes.
@@ -126,13 +118,13 @@ def extract_data(
 
 
 def load_data(
-    data_sink: DataSink,
-    data_source: DataSource,
+    data_sink: tSUPPORTED_DATA_SINKS,
+    data_source: tSUPPORTED_DOWNLOAD_DATA_SOURCES,
     data: bytes,
-    dtype: DataType,
+    dtype: tSUPPORTED_DATA_TYPES,
     pdt: str,
     date: str,
-    mode: DataMode = 'historical',
+    mode: tSUPPORTED_DATA_MODES = 'historical',
     **kwargs
 ) -> None:
     """
@@ -184,7 +176,7 @@ def load_data(
     print_disk_usage(config.data_path)
         
 
-def clean_raw_data(data_source: DataSource, raw_data: bytes) -> bytes:
+def clean_raw_data(data_source: tSUPPORTED_DOWNLOAD_DATA_SOURCES, raw_data: bytes) -> bytes:
     module = importlib.import_module(f'pfeed.sources.{data_source.lower()}.const')
     RENAMING_COLS = getattr(module, 'RENAMING_COLS')
     MAPPING_COLS = getattr(module, 'MAPPING_COLS')
@@ -216,6 +208,6 @@ def clean_raw_tick_data(raw_tick: bytes) -> bytes:
     return tick_data
 
 
-def resample_data(data: bytes, resolution: str | Resolution, data_tool: DataTool='polars', check_if_drop_last_bar=False) -> bytes:
+def resample_data(data: bytes, resolution: str | Resolution, data_tool: tSUPPORTED_DATA_TOOLS='polars', check_if_drop_last_bar=False) -> bytes:
     data_tool = importlib.import_module(f'pfeed.data_tools.data_tool_{data_tool.lower()}')
     return data_tool.resample_data(data, resolution, check_if_drop_last_bar=check_if_drop_last_bar)

{pfeed-0.0.1.dev6 → pfeed-0.0.1.dev7}/pfeed/feeds/bybit_feed.py

@@ -9,6 +9,7 @@ import pandas as pd
 from pfeed import etl
 from pfeed.config_handler import ConfigHandler
 from pfeed.const.commons import SUPPORTED_DATA_TOOLS
+from pfeed.types.common_literals import tSUPPORTED_DATA_TOOLS
 from pfeed.feeds.base_feed import BaseFeed
 from pfeed.sources.bybit import api
 from pfeed.sources.bybit.const import DATA_SOURCE, SUPPORTED_PRODUCT_TYPES, create_efilename, SUPPORTED_RAW_DATA_TYPES
@@ -18,9 +19,6 @@ from pfeed.data_tools.data_tool_polars import estimate_memory_usage
 # from pfund.exchanges.bybit.exchange import Exchange
 
 
-DataTool = Literal['pandas', 'polars', 'pyspark']
-
-
 __all__ = ['BybitFeed']
 
 
@@ -36,7 +34,7 @@ class BybitFeed(BaseFeed):
         resolution: str | Literal['raw', 'raw_tick']='1d',  
         start_date: str=None,
         end_date: str=None,
-        data_tool: Literal['pandas', 'polars', 'pyspark']='pandas',
+        data_tool: tSUPPORTED_DATA_TOOLS='pandas',
         memory_usage_limit_in_gb: int=2,  # in GB
     ) -> pd.DataFrame | pl.LazyFrame:
         """Get historical data from Bybit.
@@ -90,8 +88,8 @@ class BybitFeed(BaseFeed):
                 local_data_dtype = ''
             
             if local_data:
-                data_str = f'{source} {pdt} {date}'
-                self.logger.info(f'loaded {data_str} {local_data_dtype} data locally')
+                data_str = f'{source} {pdt} {date} {local_data_dtype}'
+                self.logger.info(f'loaded {data_str} data locally')
                 # REVIEW: last bar is very likely incomplete when e.g. 20 days of daily data is resampled to '3d'
                 check_if_drop_last_bar = (local_data_dtype == dtype and resolution.period != 1)
                 if local_data_dtype == dtype and resolution.period == 1:

{pfeed-0.0.1.dev6 → pfeed-0.0.1.dev7}/pfeed/filepath.py

@@ -1,17 +1,16 @@
 from dataclasses import dataclass, field
 from pathlib import Path
 
-from typing import Literal
-
+from pfeed.types.common_literals import tSUPPORTED_DOWNLOAD_DATA_SOURCES, tSUPPORTED_DATA_MODES, tSUPPORTED_DATA_TYPES
 from pfeed.utils.utils import create_filename
 from pfeed.const.paths import DATA_PATH
 
 
 @dataclass
 class FilePath:
-    data_source: Literal['BYBIT']
-    mode: Literal['historical', 'streaming']
-    dtype: Literal['raw_tick', 'raw_second', 'raw_minute', 'raw_hour', 'raw_daily', 'tick', 'second', 'minute', 'hour', 'daily']
+    data_source: tSUPPORTED_DOWNLOAD_DATA_SOURCES
+    mode: tSUPPORTED_DATA_MODES
+    dtype: tSUPPORTED_DATA_TYPES
     pdt: str
     date: str
     file_extension: str = '.parquet.gz'

pfeed-0.0.1.dev7/pfeed/types/common_literals.py

@@ -0,0 +1,10 @@
+from typing import Literal
+
+# since Literal doesn't support variables as inputs, define variables in commons.py here with prefix 't'
+tSUPPORTED_DATA_FEEDS = Literal['YAHOO_FINANCE', 'BYBIT']
+tSUPPORTED_DATA_TYPES = Literal['raw_tick', 'raw_second', 'raw_minute', 'raw_hour', 'raw_daily', 
+                                'tick', 'second', 'minute', 'hour', 'daily']
+tSUPPORTED_DATA_SINKS = Literal['local', 'minio']
+tSUPPORTED_DOWNLOAD_DATA_SOURCES = Literal['BYBIT']
+tSUPPORTED_DATA_MODES = Literal['historical', 'streaming']
+tSUPPORTED_DATA_TOOLS = Literal['pandas', 'polars', 'pyspark']

{pfeed-0.0.1.dev6 → pfeed-0.0.1.dev7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pfeed"
-version = "0.0.1.dev6"
+version = "0.0.1.dev7"
 description = "Data pipeline for algo-trading, getting and storing both real-time and historical data made easy."
 license = "Apache-2.0"
 authors = ["Stephen Yau <softwareentrepreneer+pfeed@gmail.com>"]
@@ -11,28 +11,30 @@ documentation = "https://pfeed-docs.pfund.ai"
 keywords = ["trading", "algo-trading", "data pipeline", "ETL", "data lake", "data warehouse", "data integration", "historical data", "live data", "data streaming"]
 
 [tool.poetry.dependencies]
-python = "^3.10 <3.12"
-pfund = "^0.0.1.dev5"
+python = ">=3.10 <3.13"
+pfund = "^0.0.1.dev7"
 python-dotenv = "^1.0.1"
 pyyaml = "^6.0.1"
 beautifulsoup4 = "^4.12.3"
 requests = "^2.31.0"
 rich = "^13.7.0"
 tqdm = "^4.66.2"
-pandas = "^2.2.0"
-pyarrow = "^15.0.0"
 click = "^8.1.7"
-platformdirs = "^4.2.0"
-polars = "^0.20.16"
 s3fs = "^2024.3.1"
-connectorx = "^0.3.2"
-ray = "^2.10.0"
 minio = "^7.2.5"
 yfinance = "^0.2.37"
+platformdirs = "^4.2.0"
 
 [tool.poetry.scripts]
 pfeed = "pfeed.main:run_cli"
 
+[tool.poetry.group.pyodide-incompatible.dependencies]
+pandas = "^2.2.0"
+pyarrow = "^15.0.0"
+polars = "^0.20.16"
+connectorx = "^0.3.2"
+ray = "^2.10.0"
+
 [tool.poetry.group.dev]
 optional = true
 
@@ -41,11 +43,13 @@ pfund = {path = "../pfund", develop = true}
 pytest = "^8.0.0"
 pre-commit = "^3.6.0"
 bandit = "^1.7.7"
-ruff = "^0.1.15"
-pyright = "^1.1.349"
 pytest-xdist = "^3.5.0"
 faker = "^24.4.0"
 tox = "^4.14.2"
+mypy = "^1.9.0"
+ruff = "^0.3.5"
+pytest-mock = "^3.14.0"
+pytest-cov = "^5.0.0"
 
 [tool.poetry.group.doc]
 optional = true