pfeed 0.0.2.dev1__py3-none-any.whl → 0.0.2.dev2__py3-none-any.whl

pfeed/__init__.py

@@ -8,32 +8,32 @@ import importlib
 from importlib.metadata import version
 
 from pfeed.config_handler import configure, get_config
-from pfeed.const.common import ALIASES
+from pfeed.const.common import ALIASES as aliases
 from pfeed.sources import bybit
 from pfeed.feeds import BybitFeed, YahooFinanceFeed
 
 
 def download_historical_data(
     data_source: tSUPPORTED_DOWNLOAD_DATA_SOURCES,
-    pdts: str | list[str] | None = None,
+    products: str | list[str] | None = None,
     dtypes: tSUPPORTED_DATA_TYPES | list[tSUPPORTED_DATA_TYPES] | None = None,
     ptypes: str | list[str] | None = None,
     start_date: str | None = None,
     end_date: str | None = None,
     use_minio: bool = False,
     use_ray: bool = True,
-    ray_num_cpus: int = 8,
+    num_cpus: int = 8,
 ):
     data_source = importlib.import_module(f"pfeed.sources.{data_source.lower()}")
     return data_source.download_historical_data(
-        pdts=pdts,
+        products=products,
         dtypes=dtypes,
         ptypes=ptypes,
         start_date=start_date,
         end_date=end_date,
         use_minio=use_minio,
         use_ray=use_ray,
-        ray_num_cpus=ray_num_cpus,
+        num_cpus=num_cpus,
     )
 
 
@@ -53,7 +53,7 @@ __all__ = (
     "__version__",
     "configure",
     "get_config",
-    "ALIASES",
+    "aliases",
     "bybit",
     "binance",
     "YahooFinanceFeed",

pfeed/cli/commands/config.py

@@ -32,33 +32,30 @@ def remove_config(config_file_path: str | Path):
 @click.option('--use-fork-process', type=bool, help='If True, multiprocessing.set_start_method("fork")')
 @click.option('--use-custom-excepthook', type=bool, help='If True, log uncaught exceptions to file')
 @click.option('--env-file', 'env_file_path', type=click.Path(resolve_path=True, exists=True), help='Path to the .env file')
-@click.option('--debug', is_flag=True, help='if enabled, debug mode will be enabled where logs at DEBUG level will be printed')
-@click.option('--list', '-l', is_flag=True, is_eager=True, help='List all available options')
-@click.option('--reset', is_flag=True, is_eager=True, help='Reset the configuration to defaults') 
-def config(ctx, **kwargs):
+@click.option('--debug', '-d', type=bool, help='If True, enable debug mode where logs at DEBUG level will be printed')
+@click.option('--list', '-l', 'is_list', is_flag=True, is_eager=True, help='List all available options')
+@click.option('--reset', 'is_reset', is_flag=True, is_eager=True, help='Reset the configuration to defaults') 
+def config(ctx, is_list, is_reset, **kwargs):
     """Configures pfeed settings."""
     config: ConfigHandler = ctx.obj['config']
     
     # Filter out options that were not provided by the user
-    provided_options = {k: v for k, v in kwargs.items() if v is not None and v is not False}
+    provided_options = {k: v for k, v in kwargs.items() if v is not None}
     
-    if kwargs.get('list'):  # Check if --list was used
-        del provided_options['list']
+    if is_list:  # Check if --list was used
         assert not provided_options, "No options should be provided with --list"
         config_dict = config.__dict__
         config_dict.update({'config_file_path': USER_CONFIG_FILE_PATH})
         click.echo(f"PFeed's config:\n{pformat(config_dict)}")
         return
 
-    if kwargs.get('reset'): # Check if --reset was used
-        del provided_options['reset']
+    if is_reset:  # Check if --reset was used
         assert not provided_options, "No options should be provided with --reset"
         remove_config(USER_CONFIG_FILE_PATH)
         click.echo("PFeed's config successfully reset.")
-        return
     
     # prints out current config if no options are provided
-    if not provided_options:
+    if not provided_options and not is_list and not is_reset:
         raise click.UsageError("No options provided. Use --list to see all available options.")
     else:
         for option, value in provided_options.items():

pfeed/cli/commands/download.py

@@ -21,7 +21,7 @@ SUPPORTED_DATA_TYPES_IMPLICIT_RAW_ALLOWED = SUPPORTED_DATA_TYPES + ['raw']
 
 @click.command()
 @click.option('--data-source', '-d', required=True, type=click.Choice(SUPPORTED_DOWNLOAD_DATA_SOURCES_ALIASES_INCLUDED, case_sensitive=False), help='Data source')
-@click.option('--pdts', '-p', 'pdts', multiple=True, default=[], help='List of trading products')
+@click.option('--products', '-p', 'products', multiple=True, default=[], help='List of trading products')
 @click.option('--dtypes', '--dt', 'dtypes', multiple=True, default=['raw'], type=click.Choice(SUPPORTED_DATA_TYPES_IMPLICIT_RAW_ALLOWED, case_sensitive=False), help=f'{SUPPORTED_DATA_TYPES=}. How to pass in multiple values: --dt raw --dt tick')
 @click.option('--ptypes', '--pt', 'ptypes', multiple=True, default=[], type=click.Choice(SUPPORTED_PRODUCT_TYPES, case_sensitive=False), help='List of product types, e.g. PERP = get all perpetuals')
 @click.option('--start-date', '-s', type=click.DateTime(formats=["%Y-%m-%d"]), help='Start date in YYYY-MM-DD format')
@@ -31,17 +31,17 @@ SUPPORTED_DATA_TYPES_IMPLICIT_RAW_ALLOWED = SUPPORTED_DATA_TYPES + ['raw']
 @click.option('--no-ray', is_flag=True, help='if enabled, Ray will not be used')
 @click.option('--env-file', 'env_file_path', type=click.Path(exists=True), help='Path to the .env file')
 @click.option('--debug', is_flag=True, help='if enabled, debug mode will be enabled where logs at DEBUG level will be printed')
-def download(data_source, pdts, dtypes, ptypes, start_date, end_date, num_cpus, no_ray, use_minio, env_file_path, debug):
+def download(data_source, products, dtypes, ptypes, start_date, end_date, num_cpus, no_ray, use_minio, env_file_path, debug):
     pe.configure(env_file_path=env_file_path, debug=debug)
     data_source = ALIASES.get(data_source, data_source)
     pipeline = importlib.import_module(f'pfeed.sources.{data_source.lower()}.download')
     pipeline.download_historical_data(
-        pdts=pdts,
+        products=products,
         dtypes=dtypes,
         ptypes=ptypes,
         start_date=start_date.date().strftime('%Y-%m-%d') if start_date else start_date,
         end_date=end_date.date().strftime('%Y-%m-%d') if end_date else end_date,
         use_ray=not no_ray,
-        ray_num_cpus=num_cpus,
+        num_cpus=num_cpus,
         use_minio=use_minio,
     )

pfeed/cli/main.py

@@ -6,6 +6,7 @@ from pfeed.cli.commands.config import config
 from pfeed.cli.commands.download import download
 # from pfeed.cli.commands.stream import stream
 from pfeed.cli.commands.open import open
+from pfeed.cli.commands.doc import doc
 
 
 @click.group(context_settings={"help_option_names": ["-h", "--help"]})
@@ -21,4 +22,5 @@ pfeed_group.add_command(docker_compose)
 pfeed_group.add_command(config)
 pfeed_group.add_command(download)
 # pfeed_group.add_command(stream)
-pfeed_group.add_command(open)
+pfeed_group.add_command(open)
+pfeed_group.add_command(doc)

pfeed/config_handler.py

@@ -33,7 +33,7 @@ class ConfigHandler:
     logging_config: dict | None = None
     use_fork_process: bool = True
     use_custom_excepthook: bool = False
-    env_file_path: str | None=None
+    env_file_path: str = ''
     debug: bool = False
     
     @classmethod
@@ -57,9 +57,9 @@ class ConfigHandler:
         return cls(**config)
     
     def __post_init__(self):
-        self.initialize()
+        self._initialize()
     
-    def initialize(self):
+    def _initialize(self):
         self.logging_config = self.logging_config or {}
         
         for path in [self.data_path]:
@@ -76,19 +76,15 @@ class ConfigHandler:
         self.load_env_file(self.env_file_path)
         
         if self.debug:
-            is_loggers_set_up = bool(logging.getLogger('pfeed').handlers)
-            if is_loggers_set_up:
-                print('loggers are already set up, ignoring enabling debug mode')
-            else:
-                self.enable_debug_mode()
+            self.enable_debug_mode()
     
-    def load_env_file(self, env_file_path: str | None):
+    def load_env_file(self, env_file_path: str=''):
         from dotenv import find_dotenv, load_dotenv
         
         if not env_file_path:
-            found_env_file_path = find_dotenv(usecwd=True, raise_error_if_not_found=False)
-            if found_env_file_path:
-                print(f'.env file path is not specified, using env file in "{found_env_file_path}"')
+            env_file_path = find_dotenv(usecwd=True, raise_error_if_not_found=False)
+            if env_file_path:
+                print(f'.env file path is not specified, using env file in "{env_file_path}"')
             else:
                 # print('.env file is not found')
                 return
@@ -96,6 +92,10 @@ class ConfigHandler:
     
     def enable_debug_mode(self):
         '''Enables debug mode by setting the log level to DEBUG for all stream handlers'''
+        is_loggers_set_up = bool(logging.getLogger('pfeed').handlers)
+        if is_loggers_set_up:
+            print('loggers are already set up, ignoring debug mode')
+            return
         if 'handlers' not in self.logging_config:
             self.logging_config['handlers'] = {}
         for handler in ['stream_handler', 'stream_path_handler']:
@@ -145,7 +145,7 @@ def configure(
         else:
             raise AttributeError(f'{k} is not an attribute of ConfigHandler')
     
-    _global_config.initialize()
+    _global_config._initialize()
     return _global_config
 
 

pfeed/const/common.py

@@ -1,9 +1,25 @@
+from pfeed.types.common_literals import tSUPPORTED_DATA_TOOLS, tSUPPORTED_DATA_ENGINES
+
+
 SUPPORTED_ENVIRONMENTS = ['BACKTEST', 'SANDBOX', 'PAPER', 'LIVE']
 SUPPORTED_DATA_FEEDS = ['YAHOO_FINANCE', 'BYBIT', 'BINANCE']
 SUPPORTED_STORAGES = ['local', 'minio']
 SUPPORTED_DOWNLOAD_DATA_SOURCES = ['BYBIT', 'BINANCE']
 SUPPORTED_CRYPTO_EXCHANGES = ['BYBIT', 'BINANCE']
-SUPPORTED_DATA_TOOLS = ['pandas', 'polars']
+SUPPORTED_DATA_TOOLS: dict[tSUPPORTED_DATA_TOOLS, tSUPPORTED_DATA_ENGINES | bool] = {
+    'pandas': ['dask'],
+    'polars': ['ray'],
+    # True means the data tool is also an execution engine
+    'dask': True,
+    'spark': True,
+}
+SUPPORTED_DATA_ENGINES: dict[tSUPPORTED_DATA_ENGINES, list[str]] = {
+    # execution engine: supported cloud services
+    'dask': ['coiled'],
+    'spark': ['databricks'],
+    'ray': ['aws'],
+}
+SUPPORTED_CLOUDS = ['aws']
 SUPPORTED_PRODUCT_TYPES = ['SPOT', 'PERP', 'IPERP', 'FUT', 'IFUT']
 SUPPORTED_DATA_TYPES = [
     'raw_tick', 'raw_second', 'raw_minute', 'raw_hour', 'raw_daily',
@@ -12,4 +28,5 @@ SUPPORTED_DATA_TYPES = [
 
 ALIASES = {
     'YF': 'YAHOO_FINANCE',
+    'FRD': 'FIRSTRATE_DATA'
 }

pfeed/feeds/base_feed.py

@@ -1,15 +1,9 @@
 from __future__ import annotations
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
-    try:
-        import pandas as pd
-        import polars as pl
-    except ImportError:
-        pass
-    from pfeed.types.common_literals import tSUPPORTED_DATA_TOOLS, tSUPPORTED_STORAGES
-    from pfeed.sources.bybit.types import tSUPPORTED_DATA_TYPES
+    from pfeed.types.common_literals import tSUPPORTED_DATA_TOOLS, tSUPPORTED_STORAGES, tSUPPORTED_DATA_TYPES
+    from pfeed.types.core import tDataFrame
     from pfeed.resolution import ExtendedResolution
-    DataFrame = pd.DataFrame | pl.LazyFrame
     
 import os
 import io
@@ -19,11 +13,6 @@ import logging
 import datetime
 import importlib
 
-try:
-    import polars as pl
-except ImportError:
-    pass
-
 from pfeed.config_handler import get_config
 from pfeed.const.common import SUPPORTED_DATA_FEEDS, SUPPORTED_DATA_TOOLS, SUPPORTED_STORAGES
 from pfeed.utils.utils import (
@@ -99,7 +88,7 @@ class BaseFeed:
         dates: list[datetime.date] = get_dates_in_between(start_date, end_date)
         return dates
             
-    def _get_historical_data_from_storages(self, trading_venue: str, pdt: str, resolution: ExtendedResolution, dates: list[datetime.date], storage: tSUPPORTED_STORAGES='') -> DataFrame | None:
+    def _get_historical_data_from_storages(self, trading_venue: str, pdt: str, resolution: ExtendedResolution, dates: list[datetime.date], storage: tSUPPORTED_STORAGES='') -> tDataFrame | None:
         from pfeed import etl
         default_raw_resolution = self.utils.get_default_raw_resolution()
         storages = [storage] if storage else SUPPORTED_STORAGES
@@ -113,7 +102,7 @@ class BaseFeed:
             self.logger.debug(f'transformed {self.name} raw data to {resolution=}')
             return transformed_df
     
-    def _get_historical_data_from_temp(self, trading_venue: str, pdt: str, resolution: ExtendedResolution, dates: list[datetime.date]) -> DataFrame | None:
+    def _get_historical_data_from_temp(self, trading_venue: str, pdt: str, resolution: ExtendedResolution, dates: list[datetime.date]) -> tDataFrame | None:
         from pfeed import etl
         default_raw_resolution = self.utils.get_default_raw_resolution()
         temp_file_paths = [self._create_temp_file_path(trading_venue, pdt, resolution, date) for date in dates]
@@ -134,7 +123,7 @@ class BaseFeed:
     ) -> list[bytes]:
         raise NotImplementedError(f"{self.name} _get_historical_data_from_source() is not implemented")
     
-    def _get_historical_data(self, trading_venue: str, pdt: str, resolution: ExtendedResolution, dates: list[datetime.date], storage: tSUPPORTED_STORAGES='') -> DataFrame:
+    def _get_historical_data(self, trading_venue: str, pdt: str, resolution: ExtendedResolution, dates: list[datetime.date], storage: tSUPPORTED_STORAGES='') -> tDataFrame:
         if (df := self._get_historical_data_from_storages(trading_venue, pdt, resolution, dates, storage=storage)) is not None:
             pass
         elif (df := self._get_historical_data_from_temp(trading_venue, pdt, resolution, dates)) is not None:
@@ -153,22 +142,23 @@ class BaseFeed:
     
     def get_historical_data(
         self,
-        pdt: str,
-        rollback_period: str="1w",
+        product: str,
         resolution: str="1d",
+        rollback_period: str="1w",
         start_date: str="",
         end_date: str="",
         trading_venue: str='',
         storage: tSUPPORTED_STORAGES='',
-    ) -> DataFrame:
+    ) -> tDataFrame:
         """Get historical data from the data source.
         Args:
-            pdt: Product symbol, e.g. BTC_USDT_PERP, where PERP = product type "perpetual".
+            product: Product symbol, e.g. BTC_USDT_PERP, where PERP = product type "perpetual".
             rollback_period:
                 Period to rollback from today, only used when `start_date` is not specified.
                 Default is '1w' = 1 week.
             resolution: Data resolution. e.g. '1m' = 1 minute as the unit of each data bar/candle.
                 Also supports raw resolution such as 'r1m', where 'r' stands for raw.            
+                If resolution is 'raw', the default raw resolution of the data type will be used.
                 Default is '1d' = 1 day.
             start_date: Start date.
             end_date: End date.
@@ -178,13 +168,17 @@ class BaseFeed:
         from pfeed import etl
         from pfeed.resolution import ExtendedResolution
         
-        pdt, trading_venue, storage = pdt.upper(), trading_venue.upper(), storage.lower()
+        pdt, trading_venue, storage = product.upper(), trading_venue.upper(), storage.lower()
         assert validate_pdt(
             self.name, pdt
         ), f'"{pdt}" does not match the required format "XXX_YYY_PTYPE" or has an unsupported product type. (PTYPE means product type, e.g. PERP, Supported types for {self.name} are: {self.const.SUPPORTED_PRODUCT_TYPES})'
         if storage:
             assert storage in SUPPORTED_STORAGES, f"Invalid {storage=}, {SUPPORTED_STORAGES=}"
         self._prepare_temp_dir()
+        if resolution == 'raw':
+            assert self.const.SUPPORTED_DATA_TYPES[0].startswith('raw_')
+            default_raw_dtype = self.const.SUPPORTED_DATA_TYPES[0]
+            resolution = self.const.DTYPES_TO_RAW_RESOLUTIOS[default_raw_dtype]
         resolution = ExtendedResolution(resolution)
         trading_venue = trading_venue or derive_trading_venue(self.name)
         dates: list[datetime.date] = self._prepare_dates(start_date, end_date, rollback_period)
@@ -202,26 +196,26 @@ class BaseFeed:
     
     def download_historical_data(
         self,
-        pdts: str | list[str] | None = None,
+        products: str | list[str] | None = None,
         dtypes: tSUPPORTED_DATA_TYPES | list[tSUPPORTED_DATA_TYPES] | None = None,
         ptypes: str | list[str] | None = None,
         start_date: str | None = None,
         end_date: str | None = None,
         use_minio: bool = False,
         use_ray: bool = True,
-        ray_num_cpus: int = 8,
+        num_cpus: int = 8,
     ):
         try:
             data_source = getattr(self, self.name.lower())
             data_source.download_historical_data(
-                pdts=pdts,
+                products=products,
                 dtypes=dtypes,
                 ptypes=ptypes,
                 start_date=start_date,
                 end_date=end_date,
                 use_minio=use_minio,          
                 use_ray=use_ray,
-                ray_num_cpus=ray_num_cpus,
+                num_cpus=num_cpus,
             )
         except AttributeError:
             raise Exception(f'{self.name} does not support download_historical_data()')

pfeed/feeds/yahoo_finance_feed.py

@@ -37,7 +37,7 @@ class YahooFinanceFeed(BaseFeed):
         "y": [1, 2, 5, 10],
     }
     # yfinance's valid intervals
-    SUPPORTED_INTERVALS = {
+    SUPPORTED_TIMEFRAMES_AND_PERIODS = {
         "m": [1, 2, 5, 15, 30, 60, 90],
         "h": [1],
         "d": [1, 5],
@@ -56,17 +56,19 @@ class YahooFinanceFeed(BaseFeed):
     def get_historical_data(
         self,
         symbol: str,
-        rollback_period: str | Literal["ytd", "max"] = "1M",
         resolution: str = "1d",
+        rollback_period: str | Literal["ytd", "max"] = "1M",
         start_date: str = "",
         end_date: str = "",
         use_pfeed_resample: bool = True,
+        product: str = "",
         **kwargs,
     ) -> pd.DataFrame | pl.DataFrame:
         """Simple Wrapper of yfinance history().
         For the details of args and kwargs, please refer to https://github.com/ranaroussi/yfinance
 
         Args:
+            symbol: ticker symbol used in yfinance
             rollback_period: Data resolution or 'ytd' or 'max'
                 Period to rollback from today, only used when `start_date` is not specified.
                 Default is '1M' = 1 month.
@@ -78,6 +80,7 @@ class YahooFinanceFeed(BaseFeed):
             use_pfeed_resample: Whether to use pfeed's resampling logic.
                 Default is True.
                 This will automatically be triggered if yfinance does not support the resolution.
+            product: Product symbol, e.g. AAPL_USD_STK. If provided, it will be used to create a column 'product' in the output dataframe.
             **kwargs: kwargs supported by `yfinance`
         """
         from pfeed import etl
@@ -113,11 +116,11 @@ class YahooFinanceFeed(BaseFeed):
             etimeframe = self._ADAPTER["timeframe"].get(timeframe, timeframe)
             interval = str(resolution.period) + etimeframe
 
-        # manipulate the input resolution and support e.g. '2d' resolution even it is not in the SUPPORTED_INTERVALS
+        # manipulate the input resolution and support e.g. '2d' resolution even it is not in the SUPPORTED_TIMEFRAMES_AND_PERIODS
         if (use_pfeed_resample and resolution.period != 1) or (
-            timeframe in self.SUPPORTED_INTERVALS
-            and resolution.period not in self.SUPPORTED_INTERVALS[timeframe]
-            and 1 in self.SUPPORTED_INTERVALS[timeframe]
+            timeframe in self.SUPPORTED_TIMEFRAMES_AND_PERIODS
+            and resolution.period not in self.SUPPORTED_TIMEFRAMES_AND_PERIODS[timeframe]
+            and 1 in self.SUPPORTED_TIMEFRAMES_AND_PERIODS[timeframe]
         ):
             # if resolution (e.g. '2d') is not supported in yfinance, using "1d" instead'
             interval = "1" + etimeframe
@@ -167,12 +170,17 @@ class YahooFinanceFeed(BaseFeed):
             df = etl.resample_data(df, resolution)
         
         df["symbol"] = symbol
+        if product:
+            df["product"] = product
         df["resolution"] = repr(resolution)
         # reorder columns
-        left_cols = ["ts", "symbol", "resolution"]
+        if "product" in df.columns:
+            left_cols = ["ts", "symbol", "product", "resolution"]
+        else:
+            left_cols = ["ts", "symbol", "resolution"]
         df = df[left_cols + [col for col in df.columns if col not in left_cols]]
         
-        if self.data_tool == "pandas":
+        if self.data_tool.name == "pandas":
             return df
-        elif self.data_tool == "polars":
+        elif self.data_tool.name == "polars":
             return pl.from_pandas(df)

pfeed/sources/binance/download.py

@@ -141,7 +141,7 @@ def download_historical_data(
             pdt_splits = pdt.split('_')
             ptype = pdt_splits[-1].upper()
             exchange = Exchange(env='LIVE', ptype=ptype)
-            product = exchange.create_product(*pdt_splits)
+            product = exchange.create_product(pdt)
         except KeyError:
             raise ValueError(f'"{pdt}" is not a valid product in {source}')
         efilenames = api.get_efilenames(pdt)

pfeed/sources/bybit/const.py

@@ -2,6 +2,7 @@
 DATA_SOURCE = 'BYBIT'
 SUPPORTED_PRODUCT_TYPES = ['SPOT', 'PERP', 'IPERP', 'FUT', 'IFUT']
 SUPPORTED_DATA_TYPES = ['raw_tick', 'tick', 'second', 'minute', 'hour', 'daily']
+# this specifies the raw resolution of the data type, e.g. 'raw_minute': 'r5m', meaning raw_minute is 5-minute data
 DTYPES_TO_RAW_RESOLUTIOS = {
     'raw_tick': 'r1tick',
 }

pfeed/sources/bybit/download.py

@@ -98,14 +98,14 @@ def _run_etl(storage: tSUPPORTED_STORAGES, product: CryptoProduct, date: datetim
 
 
 def download_historical_data(
-    pdts: str | list[str] | None=None, 
+    products: str | list[str] | None=None, 
     dtypes: tSUPPORTED_DATA_TYPES | list[tSUPPORTED_DATA_TYPES] | None=None,
     ptypes: tSUPPORTED_PRODUCT_TYPES | list[tSUPPORTED_PRODUCT_TYPES] | None=None, 
     start_date: str | None=None,
     end_date: str | None=None,
     use_minio: bool=False,
     use_ray: bool=True,
-    ray_num_cpus: int=8,
+    num_cpus: int=8,
 ) -> None:
     from pfund.plogging import set_up_loggers
     
@@ -116,12 +116,12 @@ def download_historical_data(
     logger = logging.getLogger(DATA_SOURCE.lower() + '_data')
 
     print(f'''Hint: 
-        You can use the command "pfeed config --data-path ..." to set your data path that stores downloaded data.
+        You can use the command "pfeed config --data-path {{your_path}}" to set your data path that stores downloaded data.
         The current data path is: {config.data_path}.
     ''')
     
     resolutions: list[ExtendedResolution] = _convert_dtypes_to_resolutions(dtypes)
-    pdts = _prepare_pdts(pdts, ptypes)
+    pdts = _prepare_pdts(products, ptypes)
     start_date, end_date = _prepare_dates(start_date, end_date)
     dates: list[datetime.date] = get_dates_in_between(start_date, end_date)
     
@@ -136,7 +136,7 @@ def download_historical_data(
     ray_tasks = defaultdict(list)
     for pdt in pdts if use_ray else tqdm(pdts, desc=f'Downloading {DATA_SOURCE} historical data by product', colour='green'):
         try:
-            product = exchange.create_product(*pdt.split('_'))
+            product = exchange.create_product(pdt)
         except KeyError:
             raise ValueError(f'"{pdt}" is not a valid product in {DATA_SOURCE}')
         efilenames = api.get_efilenames(pdt)
@@ -176,7 +176,7 @@ def download_historical_data(
         try:
             log_listener = None
             logical_cpus = os.cpu_count()
-            num_cpus = min(ray_num_cpus, logical_cpus)
+            num_cpus = min(num_cpus, logical_cpus)
             ray.init(num_cpus=num_cpus)
             batch_size = num_cpus
             print(f"Ray's num_cpus is set to {num_cpus}")

pfeed/types/common_literals.py

@@ -1,12 +1,14 @@
 from typing import Literal
 
+
 # since Literal doesn't support variables as inputs, define variables in common.py here with prefix 't'
 tSUPPORTED_ENVIRONMENTS = Literal['BACKTEST', 'SANDBOX', 'PAPER', 'LIVE']
 tSUPPORTED_DATA_FEEDS = Literal['YAHOO_FINANCE', 'BYBIT']
 tSUPPORTED_STORAGES = Literal['local', 'minio']
 tSUPPORTED_DOWNLOAD_DATA_SOURCES = Literal['BYBIT', 'BINANCE']
 tSUPPORTED_CRYPTO_EXCHANGES = Literal['BYBIT', 'BINANCE']
-tSUPPORTED_DATA_TOOLS = Literal['pandas', 'polars']
+tSUPPORTED_DATA_TOOLS = Literal['pandas', 'polars', 'dask', 'spark']
+tSUPPORTED_DATA_ENGINES = Literal['ray', 'dask', 'spark']
 tSUPPORTED_DATA_TYPES = Literal[
     'raw_tick', 'raw_second', 'raw_minute', 'raw_hour', 'raw_daily',
     'tick', 'second', 'minute', 'hour', 'daily'

pfeed/types/core.py

@@ -0,0 +1,11 @@
+from typing import TypeVar
+
+try:
+    import pandas as pd
+    import polars as pl
+except ImportError:
+    pass
+
+
+tDataFrame = TypeVar('tDataFrame', pd.DataFrame, pl.DataFrame, pl.LazyFrame)
+tSeries = TypeVar('tSeries', pd.Series, pl.Series)

pfeed-0.0.2.dev2.dist-info/METADATA

@@ -0,0 +1,213 @@
+Metadata-Version: 2.1
+Name: pfeed
+Version: 0.0.2.dev2
+Summary: Data pipeline for algo-trading, getting and storing both real-time and historical data made easy.
+Home-page: https://pfund.ai
+License: Apache-2.0
+Keywords: trading,algo-trading,data pipeline,ETL,data lake,data warehouse,data integration,historical data,live data,data streaming
+Author: Stephen Yau
+Author-email: softwareentrepreneer+pfeed@gmail.com
+Requires-Python: >=3.10,<4.0
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: all
+Provides-Extra: core
+Provides-Extra: dask
+Provides-Extra: dataops
+Provides-Extra: dfs
+Provides-Extra: polars
+Provides-Extra: spark
+Provides-Extra: storage
+Requires-Dist: adlfs (>=2024.7.0,<2025.0.0) ; extra == "storage" or extra == "all"
+Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0.0)
+Requires-Dist: bytewax (>=0.21.0,<0.22.0) ; extra == "dataops" or extra == "all"
+Requires-Dist: coiled (>=1.54.0,<2.0.0) ; extra == "dask" or extra == "dfs" or extra == "all"
+Requires-Dist: confluent-kafka (>=2.5.3,<3.0.0) ; extra == "dataops" or extra == "all"
+Requires-Dist: connectorx (>=0.3.3,<0.4.0) ; extra == "storage" or extra == "all"
+Requires-Dist: dask[complete] (>=2024.9.1,<2025.0.0) ; extra == "dask" or extra == "dfs" or extra == "all"
+Requires-Dist: databento (>=0.42.0,<0.43.0) ; extra == "core" or extra == "all"
+Requires-Dist: databricks-connect (>=15.4.2,<16.0.0) ; extra == "spark" or extra == "dfs" or extra == "all"
+Requires-Dist: fastparquet (>=2024.2.0,<2025.0.0)
+Requires-Dist: gcsfs (>=2024.9.0,<2025.0.0) ; extra == "storage" or extra == "all"
+Requires-Dist: minio (>=7.2.8,<8.0.0) ; extra == "core" or extra == "all"
+Requires-Dist: modin[all] (>=0.32.0,<0.33.0) ; extra == "core" or extra == "all"
+Requires-Dist: pandas (>=2.2.0,<3.0.0)
+Requires-Dist: pfund (>=0.0.2.dev1,<0.0.3)
+Requires-Dist: polars (>=1.7.1,<2.0.0) ; extra == "polars" or extra == "dfs" or extra == "all"
+Requires-Dist: polars-xdt (>=0.16.0,<0.17.0) ; extra == "polars" or extra == "dfs" or extra == "all"
+Requires-Dist: polygon-api-client (>=1.14.2,<2.0.0) ; extra == "core" or extra == "all"
+Requires-Dist: prefect (>=3.0.5,<4.0.0) ; extra == "dataops" or extra == "all"
+Requires-Dist: psutil (>=6.0.0,<7.0.0) ; extra == "core" or extra == "all"
+Requires-Dist: psycopg2 (>=2.9.9,<3.0.0) ; extra == "storage" or extra == "all"
+Requires-Dist: pyarrow (>=17.0.0,<18.0.0) ; extra == "core" or extra == "all"
+Requires-Dist: pydantic (>=2.7.0,<3.0.0)
+Requires-Dist: pyspark (>=3.5.3,<4.0.0) ; extra == "spark" or extra == "dfs" or extra == "all"
+Requires-Dist: ray (>=2.35.0,<3.0.0) ; extra == "core" or extra == "all"
+Requires-Dist: s3fs (>=2024.9.0,<2025.0.0) ; extra == "storage" or extra == "all"
+Requires-Dist: yfinance (>=0.2.43,<0.3.0)
+Project-URL: Documentation, https://pfeed-docs.pfund.ai
+Project-URL: Repository, https://github.com/PFund-Software-Ltd/pfeed
+Description-Content-Type: text/markdown
+
+# PFeed: Data Pipeline for Algo-Trading, Getting and Storing Real-Time and Historical Data Made Easy.
+
+[![Twitter Follow](https://img.shields.io/twitter/follow/pfund_ai?style=social)](https://x.com/pfund_ai)
+![GitHub stars](https://img.shields.io/github/stars/PFund-Software-Ltd/pfeed?style=social)
+![PyPI downloads](https://img.shields.io/pypi/dm/pfeed?label=downloads)
+[![PyPI](https://img.shields.io/pypi/v/pfeed.svg)](https://pypi.org/project/pfeed)
+![PyPI - Support Python Versions](https://img.shields.io/pypi/pyversions/pfeed)
+<!-- [![Jupyter Book Badge](https://raw.githubusercontent.com/PFund-Software-Ltd/pfeed/main/docs/images/jupyterbook.svg)](https://jupyterbook.org) -->
+[![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
+
+[MinIO]: https://min.io/
+[PFund]: https://github.com/PFund-Software-Ltd/pfund
+[Polars]: https://github.com/pola-rs/polars
+[Dask]: https://www.dask.org/
+[Spark]: https://spark.apache.org/docs/latest/api/python/index.html
+[PyTrade.org]: https://pytrade.org
+[Yahoo Finance]: https://github.com/ranaroussi/yfinance
+[Bybit]: https://public.bybit.com
+[Binance]: https://data.binance.vision
+[OKX]: https://www.okx.com/data-download
+[Databento]: https://databento.com/
+[Polygon]: https://polygon.io/
+[FirstRate Data]: https://firstratedata.com
+
+## Problem
+Starting algo-trading requires reliable, clean data. However, the time-consuming and mundane tasks of data cleaning and storage often discourage traders from embarking on their algo-trading journey.
+
+## Solution
+By leveraging modern data engineering tools, `pfeed` handles the tedious data work and **outputs backtesting-ready data**, allowing traders to focus on strategy development.
+
+---
+PFeed (/piː fiːd/) is a data pipeline for algorithmic trading, serving as a bridge between raw data sources and traders. It enables you to **download historical data**, **stream real-time data**, and **store cleaned data** in a **local data lake for quantitative analysis**, by automating the processes of data collection, cleaning, transformation, and storage.
+
+## Core Features
+- [x] Download or stream reliable, validated and **clean data** for research, backtesting, or live trading
+- [x] Get historical data (**dataframe**) or live data in standardized formats by just calling a **single** function
+- [x] **Own your data** by storing them locally using [MinIO], with the option to connect to the cloud
+- [x] Interact with different kinds of data (including TradFi, CeFi and DeFi) using a **unified interface**
+
+---
+
+<details>
+<summary>Table of Contents</summary>
+
+- [Installation](#installation)
+- [Quick Start](#quick-start)
+    - [Get Historical Data in Dataframe](#1-get-historical-data-in-dataframe-no-storage)
+    - [Download Historical Data on Command Line](#2-download-historical-data-on-the-command-line-interface-cli)
+    - [Download Historical Data in Python](#3-download-historical-data-in-python)
+- [Supported Data Sources](#supported-data-sources)
+- [Related Projects](#related-projects)
+- [Disclaimer](#disclaimer)
+
+</details>
+
+
+
+## Installation
+> For more installation options, please refer to the [documentation](https://pfeed-docs.pfund.ai/installation).
+```bash
+# [RECOMMENDED]: Full Features, choose this if you do not care about the package size
+pip install -U "pfeed[all]"
+
+# Minimal Features, only supports getting, downloading and streaming data
+pip install -U "pfeed[core]"
+```
+
+
+
+## Quick Start
+### 1. Get Historical Data in Dataframe (No storage)
+Get [Bybit]'s data in dataframe, e.g. 1-minute data (data is downloaded on the fly if not stored locally)
+
+```python
+import pfeed as pe
+
+feed = pe.BybitFeed(data_tool='polars')
+
+df = feed.get_historical_data(
+    'BTC_USDT_PERP',
+    resolution='1minute',  # 'raw' or '1tick'/'1t' or '2second'/'2s' etc.
+    start_date='2024-03-01',
+    end_date='2024-03-01',
+)
+```
+
+Printing the first few rows of `df`:
+|    | ts                  | product       | resolution   |    open |    high |     low |   close |   volume |
+|---:|:--------------------|:--------------|:-------------|--------:|--------:|--------:|--------:|---------:|
+|  0 | 2024-03-01 00:00:00 | BTC_USDT_PERP | 1m           | 61184.1 | 61244.5 | 61175.8 | 61244.5 |  159.142 |
+|  1 | 2024-03-01 00:01:00 | BTC_USDT_PERP | 1m           | 61245.3 | 61276.5 | 61200.7 | 61232.2 |  227.242 |
+|  2 | 2024-03-01 00:02:00 | BTC_USDT_PERP | 1m           | 61232.2 | 61249   | 61180   | 61184.2 |   91.446 |
+
+> By using pfeed, you are just a few lines of code away from getting a standardized dataframe, how convenient!
+
+### 2. Download Historical Data on the Command Line Interface (CLI)
+> For more CLI commands, please refer to the [documentation](https://pfeed-docs.pfund.ai/cli-commands).
+```bash
+# download data, default data type (dtype) is 'raw' data
+pfeed download -d BYBIT -p BTC_USDT_PERP --start-date 2024-03-01 --end-date 2024-03-08
+
+# download multiple products BTC_USDT_PERP and ETH_USDT_PERP as minute data and store them locally
+pfeed download -d BYBIT -p BTC_USDT_PERP -p ETH_USDT_PERP --dtypes minute --use-minio
+```
+
+### 3. Download Historical Data in Python
+```python
+import pfeed as pe
+
+# compared to the CLI approach, this approach is more convenient for downloading multiple products
+pe.download(
+    data_source='bybit',
+    pdts=[
+        'BTC_USDT_PERP',
+        'ETH_USDT_PERP',
+        'BCH_USDT_PERP',
+    ],
+    dtypes=['raw'],  # data types, e.g. 'raw', 'tick', 'second', 'minute' etc.
+    start_date='2024-03-01',
+    end_date='2024-03-08',
+    use_minio=False,
+)
+```
+
+
+
+## Supported Data Sources
+| Data Source          | Get Historical Data | Download Historical Data | Get Live Data | Stream Live Data |
+| -------------------- | ------------------- | ------------------------ | --------------| ---------------- |
+| [Yahoo Finance]      | 🟢                  | ⚪                        | ⚪            | ⚪               |
+| [Bybit]              | 🟢                  | 🟢                        | 🟡            | 🔴               |
+| *Interactive Brokers | 🔴                  | ⚪                        | 🔴            | 🔴               |
+| *[FirstRate Data]    | 🔴                  | 🔴                        | ⚪            | ⚪               |
+| *[Databento]         | 🔴                  | 🔴                        | 🔴            | 🔴               |
+| *[Polygon]           | 🔴                  | 🔴                        | 🔴            | 🔴               |
+| [Binance]            | 🔴                  | 🔴                        | 🔴            | 🔴               |
+| [OKX]                | 🔴                  | 🔴                        | 🔴            | 🔴               |
+
+🟢 = finished \
+🟡 = in progress \
+🔴 = todo \
+⚪ = not applicable \
+\* = paid data
+
+
+
+## Related Projects
+- [PFund] — A Complete Algo-Trading Framework for Machine Learning, TradFi, CeFi and DeFi ready. Supports Vectorized and Event-Driven Backtesting, Paper and Live Trading
+- [PyTrade.org] - A curated list of Python libraries and resources for algorithmic trading.
+
+
+
+## Disclaimer
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+This framework is intended for educational and research purposes only. It should not be used for real trading without understanding the risks involved. Trading in financial markets involves significant risk, and there is always the potential for loss. Your trading results may vary. No representation is being made that any account will or is likely to achieve profits or losses similar to those discussed on this platform.
+
+The developers of this framework are not responsible for any financial losses incurred from using this software. This includes but not limited to losses resulting from inaccuracies in any financial data output by PFeed. Users should conduct their due diligence, verify the accuracy of any data produced by PFeed, and consult with a professional financial advisor before engaging in real trading activities.
+

{pfeed-0.0.2.dev1.dist-info → pfeed-0.0.2.dev2.dist-info}/RECORD

@@ -1,47 +1,49 @@
-pfeed/__init__.py,sha256=wzxcCqGSNuWIy3oOPiZFQIuqfQBUUltIq_f28uiz_vk,1658
+pfeed/__init__.py,sha256=zD1yEHw4-oIX-fNIR2dEVh-tQ5cyVhBpwi0IjOadYaU,1669
 pfeed/cli/__init__.py,sha256=xRBbc1F6E4xiWFhNKHA12OjKXd29QI9T2rA11iZDhrk,66
 pfeed/cli/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pfeed/cli/commands/config.py,sha256=hs-j7cYpAADBenA8h_SPZcLznNZGwYXQOEsm26p81nc,3195
+pfeed/cli/commands/config.py,sha256=hXQ2lljvUz8mWcWrEXgYDU3IJxRJdhon_MB7LpJsiAE,3131
+pfeed/cli/commands/doc.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pfeed/cli/commands/docker_compose.py,sha256=xFN2nbZ10pSZSWxgUZRdUnxj8pbq12RplEPvms95kt4,1160
-pfeed/cli/commands/download.py,sha256=yRWuPhnwkA_IZwHrsH7DsePrfBh6tdA_ys2LRVqsnPg,2884
+pfeed/cli/commands/download.py,sha256=ZxnAv4Jx9hbEo3M33xM-b0hSLwjRZiE7RE4ivUtJnek,2900
 pfeed/cli/commands/open.py,sha256=VGrSGwowQJMTJ1j8R67J4rkOcIS7Xr6PsFnwYYPpOCY,1728
 pfeed/cli/commands/stream.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pfeed/cli/main.py,sha256=1qcI8uQg3isLYJ4GGPVjm2DnGruFw9E4dd4IADo8AAA,698
-pfeed/config_handler.py,sha256=Aqz_SHuF01S7z85lCguVOB4UOJLmdhOIZve978kv3dw,5223
-pfeed/const/common.py,sha256=_LeGAFeaTL0vo9ujvrswohwtkeztL8r7fbq1si8c0tE,569
+pfeed/cli/main.py,sha256=55spaJbEQY7QgAIByR3EPQYYUz9bmDWT4qhGBiNoOXA,767
+pfeed/config_handler.py,sha256=vIQwicjRKMen_S3F6amrvbIMcpXtq_PPXRbxK7Xt3uo,5173
+pfeed/const/common.py,sha256=c-DMmjaPdafgmIb6W56g5suneGZb67WlDz-l6Km_pnY,1093
 pfeed/const/paths.py,sha256=IKBV15R5q1poI4AM2jd-CSqtB9DPewwZSZ8H3YJA72w,477
 pfeed/data_tools/data_tool_pandas.py,sha256=Fn0ej5tS9twO5NzSsxUvc35ud5PMps1EPgQMr-qWLb0,2247
 pfeed/data_tools/data_tool_polars.py,sha256=qsmrz92JweEOK9bwapukezN0C18WHKHrFX5725Mser4,2501
 pfeed/datastore.py,sha256=mKEM1cBeAZnbpV392-pOoYTuxqxXjHGrZhv43P0a2k0,5196
 pfeed/etl.py,sha256=fLHxUZMx_AE-0Pq5S2rTuafHXozDYGbt0fSQUDEXd7M,15446
 pfeed/feeds/__init__.py,sha256=bY7wGBlW4BdMIJRKiZ_dHZf_-ag-CAqN-__iFV5XpK8,153
-pfeed/feeds/base_feed.py,sha256=uIqVGxMrX87v8LZ6p00Mg--D4d717TstLbXpqshjeqg,12100
+pfeed/feeds/base_feed.py,sha256=HQOOGCII28ga0i9WGryecDsqWMnrb8SfnElgdLeRkSQ,12264
 pfeed/feeds/binance_feed.py,sha256=nS_-WeXeaGtgjAaEV9ZOFi4xqkWHLoRJaiCSMWNHma4,632
 pfeed/feeds/bybit_feed.py,sha256=Im1sHJtqYhKFRxTMHz3iQ46KVQCUAA5Ug2jZSop9RFk,2093
 pfeed/feeds/custom_csv_feed.py,sha256=qTkanCebga7Vcrz9Gus0YXX6ZZkhxwiXw02ID9zmZ5M,281
-pfeed/feeds/yahoo_finance_feed.py,sha256=iPQNSq8DTCLMBr9BPo-AK89uNQWCKnfll-DK0BEAjhU,6855
+pfeed/feeds/yahoo_finance_feed.py,sha256=UTDsQ1aIKrAzdt685bPVGdnfqYM_VyOhwDavTind9qg,7323
 pfeed/filepath.py,sha256=Jl__1G314foQ8Fvkh6EoaBxzyIq0wyyXltC6VO-Zr1Y,2819
 pfeed/main.py,sha256=J9ATB3AT4VY7IlCTeK7xeDpOlRJV5cetQ1LZxQn2CDc,294
 pfeed/resolution.py,sha256=YA4q-2Ohl3xaYzjwzEWSEhxepkS8h38Xz1YKcUx3ShE,2409
 pfeed/sources/binance/__init__.py,sha256=GHdr0Tf_sRirKOfqJ__nN4D9L0hZTn657kt-ar_zgLY,498
 pfeed/sources/binance/api.py,sha256=v_WIyke_JQJupdQliTVQ8lqZ3reD7xCQiMFTH-Saguo,3710
 pfeed/sources/binance/const.py,sha256=7XSPvfhGLbi0U0IMWmv0OWb86PPEzObpbBv3WUAzKT8,2034
-pfeed/sources/binance/download.py,sha256=-AFtq9Oeu8UAqn4o2g1B2H72wVE1gSSk9gUA0GafBU8,7650
+pfeed/sources/binance/download.py,sha256=go-YRGDtCW4TwTHm4-LGkfmWtB9ehlHHB312TPArFGA,7642
 pfeed/sources/binance/stream.py,sha256=WSyzvrInWCaFPOoeZ2KnBbKef4HKSrUY8zwHanfyAOw,72
 pfeed/sources/bybit/__init__.py,sha256=PgmOrdy3FWqmii122brz5rC08ig15erH70gIra6-Kn0,317
 pfeed/sources/bybit/api.py,sha256=EP6CJo_aaBrbezkBs1z2Y5PMXDqVBfRyjjV3X8tag5A,2790
-pfeed/sources/bybit/const.py,sha256=XRWEWs2kZg8yOs6J8Lr3hY4T6SbuQpaQ02CqjrxlBYo,1082
-pfeed/sources/bybit/download.py,sha256=Q-DCW-yk62me3tYNfySaw1-YXU-sAiRH3pyscr3cz9Y,9318
+pfeed/sources/bybit/const.py,sha256=NEr8uU9KJvt26w_UXmn1PJtZh5Y7HuNCpBV2xICoDWE,1198
+pfeed/sources/bybit/download.py,sha256=AXqAoXQMWDd3Jyl2V4aXgv2lu4AWqdd_EXRGq0BS5sE,9316
 pfeed/sources/bybit/stream.py,sha256=b0frKmuhC2jTmq4flX5_ff0i7U-ksjPY6OjGTjsD1Zs,144
 pfeed/sources/bybit/types.py,sha256=gOZKjHe9keq50FHz4KAid2JH2gw4M3iMp__NeMh56AU,191
 pfeed/sources/bybit/utils.py,sha256=HiMbM62WxVr9gwKhiMKvWSdY1RXw-rEq4OPwadFrZk0,1628
-pfeed/types/common_literals.py,sha256=wHF4VLJsFc3XlCY_-LqIFdU_Zm0XshIhLWRVjajVKYg,636
+pfeed/types/common_literals.py,sha256=lNbTgXJ5HY_UmahmnvoIlzNJAYHpDUxl7AO0r936feU,712
+pfeed/types/core.py,sha256=BX-XSON6mvRKfuxiiRP3_e47eeO36DHHIY_lfxESfhw,240
 pfeed/utils/file_formats.py,sha256=iEkPmXWN5neDCB-faoQ2pDWjyQ1FmuYutS-9Fr-bgko,2019
 pfeed/utils/monitor.py,sha256=S5roDtzHVpiiuCqVl4oT7MGuky4LPTAQ8MJuyVVqcno,480
 pfeed/utils/utils.py,sha256=LaMctJw12RwEUF9XZQ-T3PZKpnwHZi3MoYes8IZcSkI,4999
 pfeed/utils/validate.py,sha256=UNBd6OIKao1b_5qSVHwfqLKX87Gv9Z4KW7nkFmKckco,1909
-pfeed-0.0.2.dev1.dist-info/LICENSE,sha256=QBDNo3Na8pzY1OUD6k6KEIhRvTP_zHg_q78IkYDLzIo,11355
-pfeed-0.0.2.dev1.dist-info/METADATA,sha256=Mf-j2Sg22Inx78iwifuEgUdEy3VaSXEfPfWwc73jhzc,12182
-pfeed-0.0.2.dev1.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-pfeed-0.0.2.dev1.dist-info/entry_points.txt,sha256=hK2aA9xFMsx6MJX2lKTVtH3uMtM1aCce6vLv9a67RF0,44
-pfeed-0.0.2.dev1.dist-info/RECORD,,
+pfeed-0.0.2.dev2.dist-info/LICENSE,sha256=QBDNo3Na8pzY1OUD6k6KEIhRvTP_zHg_q78IkYDLzIo,11355
+pfeed-0.0.2.dev2.dist-info/METADATA,sha256=203cAqP7LT32wAwqlYnnAq9MfCvz3zRWXrBiq2pg1ss,11567
+pfeed-0.0.2.dev2.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+pfeed-0.0.2.dev2.dist-info/entry_points.txt,sha256=hK2aA9xFMsx6MJX2lKTVtH3uMtM1aCce6vLv9a67RF0,44
+pfeed-0.0.2.dev2.dist-info/RECORD,,

pfeed-0.0.2.dev1.dist-info/METADATA

@@ -1,267 +0,0 @@
-Metadata-Version: 2.1
-Name: pfeed
-Version: 0.0.2.dev1
-Summary: Data pipeline for algo-trading, getting and storing both real-time and historical data made easy.
-Home-page: https://pfund.ai
-License: Apache-2.0
-Keywords: trading,algo-trading,data pipeline,ETL,data lake,data warehouse,data integration,historical data,live data,data streaming
-Author: Stephen Yau
-Author-email: softwareentrepreneer+pfeed@gmail.com
-Requires-Python: >=3.10,<3.13
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Provides-Extra: all
-Provides-Extra: boost
-Provides-Extra: data
-Provides-Extra: df
-Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0.0)
-Requires-Dist: connectorx (>=0.3.3,<0.4.0) ; extra == "boost" or extra == "all"
-Requires-Dist: fastparquet (>=2024.5.0,<2025.0.0)
-Requires-Dist: minio (>=7.2.8,<8.0.0) ; extra == "data" or extra == "all"
-Requires-Dist: pandas (>=2.2.2,<3.0.0) ; extra == "df" or extra == "all"
-Requires-Dist: pfund (>=0.0.1.dev13,<0.0.2)
-Requires-Dist: polars (>=1.7.1,<2.0.0) ; extra == "df" or extra == "all"
-Requires-Dist: psutil (>=6.0.0,<7.0.0) ; extra == "data" or extra == "all"
-Requires-Dist: pyarrow (>=15.0.0,<16.0.0) ; extra == "df" or extra == "all"
-Requires-Dist: ray (>=2.35.0,<3.0.0) ; extra == "boost" or extra == "all"
-Requires-Dist: s3fs (>=2024.9.0,<2025.0.0) ; extra == "data" or extra == "all"
-Requires-Dist: yfinance (>=0.2.43,<0.3.0)
-Project-URL: Documentation, https://pfeed-docs.pfund.ai
-Project-URL: Repository, https://github.com/PFund-Software-Ltd/pfeed
-Description-Content-Type: text/markdown
-
-# PFeed: Data Pipeline for Algo-Trading, Getting and Storing Real-Time and Historical Data Made Easy.
-
-![GitHub stars](https://img.shields.io/github/stars/PFund-Software-Ltd/pfeed?style=social)
-![PyPI downloads](https://img.shields.io/pypi/dm/pfeed?label=downloads)
-[![PyPI](https://img.shields.io/pypi/v/pfeed.svg)](https://pypi.org/project/pfeed)
-![PyPI - Support Python Versions](https://img.shields.io/pypi/pyversions/pfeed)
-[![Jupyter Book Badge](https://raw.githubusercontent.com/PFund-Software-Ltd/pfeed/main/docs/images/jupyterbook.svg)](https://jupyterbook.org)
-[![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
-
-[MinIO]: https://min.io/
-[PFund]: https://github.com/PFund-Software-Ltd/pfund
-[Ray]: https://github.com/ray-project/ray
-[Polars]: https://github.com/pola-rs/polars
-[Prefect]: https://www.prefect.io
-[Timescaledb]: https://www.timescale.com/
-[Dask]: https://www.dask.org/
-[Spark]: https://spark.apache.org/docs/latest/api/python/index.html
-[DuckDB]: https://github.com/duckdb/duckdb
-[Daft]: https://github.com/Eventual-Inc/Daft
-[PyTrade.org]: https://pytrade.org
-[Databento]: https://databento.com/
-[Polygon]: https://polygon.io/
-[Bybit]: https://bybit.com/
-[FirstRate Data]: https://firstratedata.com
-
-## Problem
-Starting algo-trading requires reliable, clean data. However, the time-consuming and mundane tasks of data cleaning and storage often discourage traders from embarking on their algo-trading journey.
-
-## Solution
-By leveraging modern data engineering tools, `pfeed` handles the tedious data work and **outputs backtesting-ready data**, accelerating traders to get to the strategy development phase.
-
----
-PFeed (/piː fiːd/) is a data pipeline for algorithmic trading, serving as a bridge between raw data sources and traders by automating the process of data collection, cleaning, transformation, and storage, loading clean data into a **local data lake for quantitative analysis**.
-
-## Core Features
-- [x] Unified approach for interacting with various [data sources](#supported-data-sources) and obtaining historical and live data
-- [x] ETL data pipline for transforming raw data to clean data and storing it in [MinIO] (optional)
-- [x] Fast data downloading, utilizing [Ray] for parallelization
-- [x] Supports multiple data tools (e.g. Pandas, [Polars], [Dask], [Spark], [DuckDB], [Daft])
-- [ ] Integrates with [Prefect] to control data flows
-- [ ] Listens to PFund's trade engine and adds trade history to a local database [Timescaledb] (optional)
-
-> It is designed to be used alongside [PFund] — A Complete Algo-Trading Framework for Machine Learning, TradFi, CeFi and DeFi ready.
-
----
-
-<details>
-<summary>Table of Contents</summary>
-
-- [Installation](#installation)
-- [Quick Start](#quick-start)
-    - [Main Usage: Data Feed](#main-usage-data-feed)
-    - [Download Historical Data on Command Line](#download-historical-data-on-command-line)
-    - [Download Historical Data in Python](#download-historical-data-in-python)
-    - [List Current Config](#list-current-config)
-    - [Run PFeed's docker-compose.yml](#run-pfeeds-docker-composeyml)
-- [Supported Data Sources](#supported-data-sources)
-- [Supported Data Tools](#supported-data-tools)
-- [Related Projects](#related-projects)
-- [Disclaimer](#disclaimer)
-
-</details>
-
-
-## Installation
-### Using [Poetry](https://python-poetry.org) (Recommended)
-```bash
-# [RECOMMENDED]: Download data (e.g. Bybit and Yahoo Finance) + Data tools (e.g. pandas, polars) + Data storage (e.g. MinIO) + Boosted performance (e.g. Ray)
-poetry add "pfeed[all]"
-
-# [Download data + Data tools + Data storage]
-poetry add "pfeed[df,data]"
-
-# [Download data + Data tools]
-poetry add "pfeed[df]"
-
-# [Download data only]:
-poetry add pfeed
-
-# update to the latest version:
-poetry update pfeed
-```
-
-### Using Pip
-```bash
-# same as above, you can choose to install "pfeed[all]", "pfeed[df,data]", "pfeed[df]" or "pfeed"
-pip install "pfeed[all]"
-
-# install the latest version:
-pip install -U pfeed
-```
-
-### Checking your installation
-```bash
-$ pfeed --version
-```
-
-## Quick Start
-### 1. Get Historical Data in Dataframe (No storage)
-Get [Bybit]'s data in dataframe, e.g. 1-minute data (data is downloaded on the fly if not stored locally)
-
-```python
-import pfeed as pe
-
-feed = pe.BybitFeed(data_tool='polars')
-
-df = feed.get_historical_data(
-    'BTC_USDT_PERP',
-    resolution='1minute',  # 'raw' or '1tick'/'1t' or '2second'/'2s' etc.
-    start_date='2024-03-01',
-    end_date='2024-03-01',
-)
-```
-
-Printing the first few rows of `df`:
-|    | ts                  | product       | resolution   |    open |    high |     low |   close |   volume |
-|---:|:--------------------|:--------------|:-------------|--------:|--------:|--------:|--------:|---------:|
-|  0 | 2024-03-01 00:00:00 | BTC_USDT_PERP | 1m           | 61184.1 | 61244.5 | 61175.8 | 61244.5 |  159.142 |
-|  1 | 2024-03-01 00:01:00 | BTC_USDT_PERP | 1m           | 61245.3 | 61276.5 | 61200.7 | 61232.2 |  227.242 |
-|  2 | 2024-03-01 00:02:00 | BTC_USDT_PERP | 1m           | 61232.2 | 61249   | 61180   | 61184.2 |   91.446 |
-
-> By using pfeed, you are just a few lines of code away from a standardized dataframe, how convenient!
-
-
-
-### 2. Download Historical Data on the Command Line Interface (CLI)
-```bash
-# download data, default data type (dtype) is 'raw' data
-pfeed download -d BYBIT -p BTC_USDT_PERP --start-date 2024-03-01 --end-date 2024-03-08
-
-# download multiple products BTC_USDT_PERP and ETH_USDT_PERP and minute data
-pfeed download -d BYBIT -p BTC_USDT_PERP -p ETH_USDT_PERP --dtypes minute
-
-# download all perpetuals data from bybit
-pfeed download -d BYBIT --ptypes PERP
-
-# download all the data from bybit (CAUTION: your local machine probably won't have enough space for this!)
-pfeed download -d BYBIT
-
-# store data into MinIO (need to start MinIO by running `pfeed docker-compose up -d` first)
-pfeed download -d BYBIT -p BTC_USDT_PERP --use-minio
-
-# enable debug mode and turn off using Ray
-pfeed download -d BYBIT -p BTC_USDT_PERP --debug --no-ray
-```
-
-### 3. Download Historical Data in Python
-```python
-import pfeed as pe
-
-# compared to the CLI approach, this approach is more convenient for downloading multiple products
-pe.download(
-    data_source='bybit',
-    pdts=[
-        'BTC_USDT_PERP',
-        'ETH_USDT_PERP',
-        'BCH_USDT_PERP',
-    ],
-    dtypes=['raw'],  # data types, e.g. 'raw', 'tick', 'second', 'minute' etc.
-    start_date='2024-03-01',
-    end_date='2024-03-08',
-    use_minio=False,
-)
-```
-
-### List Current Config
-```bash
-# list the current config:
-pfeed config --list
-
-# change the data storage location to your local project's 'data' folder:
-pfeed config --data-path ./data
-
-# for more commands:
-pfeed --help
-```
-
-### Run PFeed's docker-compose.yml
-```bash
-# same as 'docker-compose', only difference is it has pointed to pfeed's docker-compose.yml file
-pfeed docker-compose [COMMAND]
-
-# e.g. start services
-pfeed docker-compose up -d
-
-# e.g. stop services
-pfeed docker-compose down
-```
-
-
-## Supported Data Sources
-| Data Source               | Get Historical Data | Download Historical Data | Get Live/Paper Data | Stream Live/Paper Data |
-| ------------------------- | ------------------- | ------------------------ | ------------------- | ---------------------- |
-| Yahoo Finance             | 🟢                  | ⚪                        | ⚪                  | ⚪                     |
-| Bybit                     | 🟢                  | 🟢                        | 🟡                  | 🔴                     |
-| *Interactive Brokers (IB) | 🔴                  | ⚪                        | 🔴                  | 🔴                     |
-| *[FirstRate Data]         | 🔴                  | 🔴                        | ⚪                  | ⚪                     |
-| [Databento]               | 🔴                  | 🔴                        | 🔴                  | 🔴                     |
-| [Polygon]                 | 🔴                  | 🔴                        | 🔴                  | 🔴                     |
-| Binance                   | 🔴                  | 🔴                        | 🔴                  | 🔴                     |
-| OKX                       | 🔴                  | 🔴                        | 🔴                  | 🔴                     |
-
-🟢 = finished \
-🟡 = in progress \
-🔴 = todo \
-⚪ = not applicable \
-\* = paid data
-
-
-## Supported Data Tools
-| Data Tools               | Supported |
-| ------------------------ | --------- |
-| Pandas                   | 🟢        |
-| [Polars]                 | 🟢        |
-| [Dask]                   | 🔴        |
-| [Spark]                  | 🔴        |
-| [DuckDB]                 | 🔴        |
-| [Daft]                   | 🔴        |
-
-
-## Related Projects
-- [PFund] — A Complete Algo-Trading Framework for Machine Learning, TradFi, CeFi and DeFi ready. Supports Vectorized and Event-Driven Backtesting, Paper and Live Trading
-- [PyTrade.org] - A curated list of Python libraries and resources for algorithmic trading.
-
-
-## Disclaimer
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-This framework is intended for educational and research purposes only. It should not be used for real trading without understanding the risks involved. Trading in financial markets involves significant risk, and there is always the potential for loss. Your trading results may vary. No representation is being made that any account will or is likely to achieve profits or losses similar to those discussed on this platform.
-
-The developers of this framework are not responsible for any financial losses incurred from using this software. This includes but not limited to losses resulting from inaccuracies in any financial data output by PFeed. Users should conduct their due diligence, verify the accuracy of any data produced by PFeed, and consult with a professional financial advisor before engaging in real trading activities.
-