pfeed 0.0.1.dev1__tar.gz → 0.0.1.dev2__tar.gz

{pfeed-0.0.1.dev1 → pfeed-0.0.1.dev2}/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright [2024] [Stephen Yau, PFund.ai]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

pfeed-0.0.1.dev2/PKG-INFO

@@ -0,0 +1,141 @@
+Metadata-Version: 2.1
+Name: pfeed
+Version: 0.0.1.dev2
+Summary: Data pipeline for algo-trading, getting and storing both real-time and historical data made easy.
+Home-page: https://pfund.ai
+License: Apache-2.0
+Keywords: trading,algo-trading,data pipeline,ETL,data lake,data warehouse,data integration,historical data,live data,data streaming
+Author: Stephen Yau
+Author-email: softwareentrepreneer+pfeed@gmail.com
+Requires-Python: >=3.10,<4.0
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0.0)
+Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: minio (>=7.2.3,<8.0.0)
+Requires-Dist: orjson (>=3.9.12,<4.0.0)
+Requires-Dist: pandas (>=2.2.0,<3.0.0)
+Requires-Dist: pfund (>=0.0.1.dev2,<0.0.2)
+Requires-Dist: platformdirs (>=4.2.0,<5.0.0)
+Requires-Dist: pyarrow (>=15.0.0,<16.0.0)
+Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
+Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
+Requires-Dist: ray (>=2.9.1,<3.0.0)
+Requires-Dist: requests (>=2.31.0,<3.0.0)
+Requires-Dist: rich (>=13.7.0,<14.0.0)
+Requires-Dist: tqdm (>=4.66.1,<5.0.0)
+Requires-Dist: yfinance (>=0.2.36,<0.3.0)
+Project-URL: Documentation, https://pfund.ai/docs
+Project-URL: Repository, https://github.com/PFund-Software-Ltd/pfeed
+Description-Content-Type: text/markdown
+
+# PFeed: Data Pipeline for Algo-Trading, Getting and Storing Real-Time and Historical Data Made Easy.
+
+[![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
+[![PyPI](https://img.shields.io/pypi/v/pfeed.svg)](https://pypi.org/project/pfeed)
+![PyPI - Support Python Versions](https://img.shields.io/pypi/pyversions/pfeed)
+
+PFeed (/piː fiːd/) is a data integration library tailored for algorithmic trading, 
+serving as an ETL (Extract, Transform, Load) data pipeline between raw data sources and traders,
+helping them in creating a local data lake for quantitative research.
+
+PFeed allows traders to download historical, paper, and live data from various data sources, both free and paid,
+and stores them into a local data lake using [MinIO](https://min.io/).
+
+It is designed to be used alongside [PFund](https://github.com/PFund-Software-Ltd/pfund) — A Complete Algo-Trading Framework for Machine Learning, TradFi, CeFi and DeFi ready. Supports Vectorized and Event-Driven Backtesting, Paper and Live Trading.
+
+<details>
+<summary>Table of Contents</summary>
+
+- [Project Status](#project-status)
+- [Mission](#mission)
+- [Core Features](#core-features)
+- [Installation](#installation)
+- [Quick Start](#quick-start)
+    - [Download Historical Data on Command Line](#download-historical-data-on-command-line)
+    - [Download Historical Data in Python](#download-historical-data-in-python)
+- [Supported Data Sources](#supported-data-sources)
+- [Related Projects](#related-projects)
+
+</details>
+
+
+## Project Status
+**_Caution: PFeed is at a VERY EARLY stage, use it at your own risk._**
+
+PFeed is currently under active development, the framework design will be prioritized first over
+stability and scalability. 
+
+Please note that the available version is a *dev* version, not a *stable* one. \
+You are encouraged to play with the *dev* version, but only use it when a *stable* version is released.
+
+> PFeed for the time being only supports [Bybit](https://bybit.com/) and Yahoo Finance for testing purpose.
+
+## Mission
+Algo-trading has always been a complicated task due to the multitude of components and procedures involved. \
+Data collection and processing is probably the most mundane and yet critical part of it, as all results and findings 
+are derived from the data.
+
+However, preparing this data for use is not quick and easy. For example, sometimes even when the data is available (e.g. [Bybit data](https://public.bybit.com/trading/)), it is often in raw form and requires some cleaning.
+
+> PFeed's mission is to **_free traders from the tedious data work_** by providing cleaned data in a standard format that is ready for use, making them significantly faster to get to the analysis and strategy development phase.
+
+
+## Core Features
+- [x] Unified approach for interacting with various data sources, obtaining historical and real-time data
+- [x] ETL data pipline for transforming raw data and storing it in [MinIO](https://min.io/) (optional)
+- [x] Utilizes [Ray](https://github.com/ray-project/ray) for parallel data downloading
+- [ ] Integrates with [Prefect](https://www.prefect.io) to control data flows
+- [ ] Listens to PFund's trade engine and adds trade history to a local database [Timescaledb](https://www.timescale.com/) (optional)
+
+
+## Installation
+### Using [Poetry](https://python-poetry.org) (Recommended)
+> If you don't have poetry installed, please visit https://python-poetry.org/docs/
+```bash
+poetry add pfeed
+```
+
+### Using Pip
+```bash
+pip install pfeed
+```
+
+
+## Quick Start
+### Download Historical Data on Command Line
+```bash
+pfeed download -s bybit -p BTC_USDT_PERP --no-minio
+```
+
+### Download Historical Data in Python
+```python
+from pfeed import bybit
+bybit.download(pdts=['BTC_USDT_PERP'])
+```
+
+
+## Supported Data Sources
+| Data Source                                  | Get Historical Data | Download Historical Data | Get Live/Paper Data | Download Live/Paper Data |
+| -------------------------------------------- | ------------------- | ------------------------ | ------------------- | ------------------------ |
+| Yahoo Finance                                | 🟢                  | ⚪                       | ⚪                  | ⚪                       |
+| Bybit                                        | 🟢                  | 🟢                       | 🟡                  | 🔴                       |
+| *Interactive Brokers (IB)                    | 🔴                  | ⚪                       | 🔴                  | 🔴                       |
+| *[FirstRate Data](https://firstratedata.com) | 🔴                  | 🔴                       | ⚪                  | ⚪                       |
+| Binance                                      | 🔴                  | 🔴                       | 🔴                  | 🔴                       |
+| OKX                                          | 🔴                  | 🔴                       | 🔴                  | 🔴                       |
+
+🟢 = finished \
+🟡 = in progress \
+🔴 = todo \
+⚪ = not applicable \
+\* = paid data \
+get data = store it in memory for python to use \
+download data = store it in local machine for later research
+
+
+## Related Projects
+- [PFund](https://github.com/PFund-Software-Ltd/pfund) — A Complete Algo-Trading Framework for Machine Learning, TradFi, CeFi and DeFi ready. Supports Vectorized and Event-Driven Backtesting, Paper and Live Trading

pfeed-0.0.1.dev2/README.md

@@ -0,0 +1,107 @@
+# PFeed: Data Pipeline for Algo-Trading, Getting and Storing Real-Time and Historical Data Made Easy.
+
+[![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
+[![PyPI](https://img.shields.io/pypi/v/pfeed.svg)](https://pypi.org/project/pfeed)
+![PyPI - Support Python Versions](https://img.shields.io/pypi/pyversions/pfeed)
+
+PFeed (/piː fiːd/) is a data integration library tailored for algorithmic trading, 
+serving as an ETL (Extract, Transform, Load) data pipeline between raw data sources and traders,
+helping them in creating a local data lake for quantitative research.
+
+PFeed allows traders to download historical, paper, and live data from various data sources, both free and paid,
+and stores them into a local data lake using [MinIO](https://min.io/).
+
+It is designed to be used alongside [PFund](https://github.com/PFund-Software-Ltd/pfund) — A Complete Algo-Trading Framework for Machine Learning, TradFi, CeFi and DeFi ready. Supports Vectorized and Event-Driven Backtesting, Paper and Live Trading.
+
+<details>
+<summary>Table of Contents</summary>
+
+- [Project Status](#project-status)
+- [Mission](#mission)
+- [Core Features](#core-features)
+- [Installation](#installation)
+- [Quick Start](#quick-start)
+    - [Download Historical Data on Command Line](#download-historical-data-on-command-line)
+    - [Download Historical Data in Python](#download-historical-data-in-python)
+- [Supported Data Sources](#supported-data-sources)
+- [Related Projects](#related-projects)
+
+</details>
+
+
+## Project Status
+**_Caution: PFeed is at a VERY EARLY stage, use it at your own risk._**
+
+PFeed is currently under active development, the framework design will be prioritized first over
+stability and scalability. 
+
+Please note that the available version is a *dev* version, not a *stable* one. \
+You are encouraged to play with the *dev* version, but only use it when a *stable* version is released.
+
+> PFeed for the time being only supports [Bybit](https://bybit.com/) and Yahoo Finance for testing purpose.
+
+## Mission
+Algo-trading has always been a complicated task due to the multitude of components and procedures involved. \
+Data collection and processing is probably the most mundane and yet critical part of it, as all results and findings 
+are derived from the data.
+
+However, preparing this data for use is not quick and easy. For example, sometimes even when the data is available (e.g. [Bybit data](https://public.bybit.com/trading/)), it is often in raw form and requires some cleaning.
+
+> PFeed's mission is to **_free traders from the tedious data work_** by providing cleaned data in a standard format that is ready for use, making them significantly faster to get to the analysis and strategy development phase.
+
+
+## Core Features
+- [x] Unified approach for interacting with various data sources, obtaining historical and real-time data
+- [x] ETL data pipline for transforming raw data and storing it in [MinIO](https://min.io/) (optional)
+- [x] Utilizes [Ray](https://github.com/ray-project/ray) for parallel data downloading
+- [ ] Integrates with [Prefect](https://www.prefect.io) to control data flows
+- [ ] Listens to PFund's trade engine and adds trade history to a local database [Timescaledb](https://www.timescale.com/) (optional)
+
+
+## Installation
+### Using [Poetry](https://python-poetry.org) (Recommended)
+> If you don't have poetry installed, please visit https://python-poetry.org/docs/
+```bash
+poetry add pfeed
+```
+
+### Using Pip
+```bash
+pip install pfeed
+```
+
+
+## Quick Start
+### Download Historical Data on Command Line
+```bash
+pfeed download -s bybit -p BTC_USDT_PERP --no-minio
+```
+
+### Download Historical Data in Python
+```python
+from pfeed import bybit
+bybit.download(pdts=['BTC_USDT_PERP'])
+```
+
+
+## Supported Data Sources
+| Data Source                                  | Get Historical Data | Download Historical Data | Get Live/Paper Data | Download Live/Paper Data |
+| -------------------------------------------- | ------------------- | ------------------------ | ------------------- | ------------------------ |
+| Yahoo Finance                                | 🟢                  | ⚪                       | ⚪                  | ⚪                       |
+| Bybit                                        | 🟢                  | 🟢                       | 🟡                  | 🔴                       |
+| *Interactive Brokers (IB)                    | 🔴                  | ⚪                       | 🔴                  | 🔴                       |
+| *[FirstRate Data](https://firstratedata.com) | 🔴                  | 🔴                       | ⚪                  | ⚪                       |
+| Binance                                      | 🔴                  | 🔴                       | 🔴                  | 🔴                       |
+| OKX                                          | 🔴                  | 🔴                       | 🔴                  | 🔴                       |
+
+🟢 = finished \
+🟡 = in progress \
+🔴 = todo \
+⚪ = not applicable \
+\* = paid data \
+get data = store it in memory for python to use \
+download data = store it in local machine for later research
+
+
+## Related Projects
+- [PFund](https://github.com/PFund-Software-Ltd/pfund) — A Complete Algo-Trading Framework for Machine Learning, TradFi, CeFi and DeFi ready. Supports Vectorized and Event-Driven Backtesting, Paper and Live Trading

pfeed-0.0.1.dev2/pfeed/__init__.py

@@ -0,0 +1,16 @@
+from pfeed.config_handler import configure
+from pfeed.sources import bybit
+from pfeed.feeds import YahooFinanceFeed, BybitFeed
+from importlib.metadata import version
+
+
+__version__ = version('pfeed')
+
+
+__all__ = (
+    '__version__',
+    'configure',
+    'bybit',
+    'YahooFinanceFeed',
+    'BybitFeed',
+)

pfeed-0.0.1.dev2/pfeed/cli/__init__.py

@@ -0,0 +1,4 @@
+from pfeed.cli.main import pfeed_group
+
+
+__all__ = ["pfeed_group"]

pfeed-0.0.1.dev2/pfeed/cli/commands/config.py

@@ -0,0 +1,67 @@
+import os
+import sys
+import yaml
+from pathlib import Path
+from pprint import pformat
+
+import click
+
+from pfeed.const.paths import USER_CONFIG_FILE_PATH
+from pfeed.config_handler import ConfigHandler
+
+
+def load_config(config_file_path: str | Path):
+    config_file_path = Path(config_file_path)
+    if config_file_path.is_file():
+        with open(config_file_path, 'r') as f:
+            return yaml.safe_load(f) or {}
+    return {}
+
+
+def save_config(config: ConfigHandler, config_file_path: str | Path):
+    with open(config_file_path, 'w') as f:
+        yaml.dump(config.__dict__, f, default_flow_style=False)
+        
+
+def remove_config(config_file_path: str | Path):
+    config_file_path = Path(config_file_path)
+    if config_file_path.is_file():
+        os.remove(config_file_path)
+
+
+@click.group(invoke_without_command=True)
+@click.pass_context
+@click.option('--data-path', type=click.Path(), help='Set the data path')
+@click.option('--log-path', type=click.Path(), help='Set the log path')
+@click.option('--logging-path', 'logging_config_file_path', type=click.Path(exists=True), help='Set the logging config file path')
+@click.option('--logging-config', type=dict, help='Set the logging config')
+@click.option('--use-fork-process', type=bool, help='If True, multiprocessing.set_start_method("fork")')
+@click.option('--use-custom-excepthook', type=bool, help='If True, log uncaught exceptions to file')
+def config(ctx, **kwargs):
+    """Configures pfeed settings."""
+    # if theres command invoked after config (e.g. pfeed config reset), skip this
+    if ctx.invoked_subcommand:
+        return
+    
+    config: ConfigHandler = ctx.obj['config']
+    
+    # Filter out options that were not provided by the user
+    provided_options = {k: v for k, v in kwargs.items() if v is not None}
+    
+    # prints out current config if no options are provided
+    if not provided_options:
+        click.echo(f"PFeed's config:\n{pformat(config.__dict__)}")
+    else:
+        for option, value in provided_options.items():
+            setattr(config, option, value)
+            click.echo(f"{option} set to: {value}")
+        
+        save_config(config, USER_CONFIG_FILE_PATH)
+        click.echo(f"config saved to {USER_CONFIG_FILE_PATH}.")
+
+
+@config.command()
+def reset():
+    """Resets the application configuration by removing the existing configuration file."""
+    remove_config(USER_CONFIG_FILE_PATH)
+    click.echo("Configuration successfully reset.")

pfeed-0.0.1.dev2/pfeed/cli/commands/docker_compose.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+import importlib.resources
+import subprocess
+
+from dotenv import find_dotenv
+import click
+
+from pfeed.const.paths import PROJ_NAME
+
+
+@click.command(context_settings=dict(
+    ignore_unknown_options=True,
+    allow_extra_args=True,
+))
+@click.pass_context
+def docker_compose(ctx):
+    """Forwards commands to docker-compose with the package's docker-compose.yml file."""
+    package_dir = Path(importlib.resources.files(PROJ_NAME)).resolve().parents[0]
+    env_file_path = find_dotenv(usecwd=True, raise_error_if_not_found=True)
+    click.echo(f'env file: {env_file_path}')
+    docker_compose_file = package_dir / 'docker-compose.yml'
+
+    command = ['docker-compose', '-f', str(docker_compose_file)] + ctx.args
+    subprocess.run(command)
+
+
+

pfeed-0.0.1.dev2/pfeed/cli/commands/download.py

@@ -0,0 +1,44 @@
+import importlib
+
+import click
+
+from pfeed.utils.validate import validate_pdts_and_ptypes
+from pfeed.const.commons import ALIASES, SUPPORTED_DOWNLOAD_DATA_SOURCES, SUPPORTED_DATA_TYPES
+
+
+# add aliases to supported download data sources
+SUPPORTED_DOWNLOAD_DATA_SOURCES_ALIASES_INCLUDED = SUPPORTED_DOWNLOAD_DATA_SOURCES + [k for k, v in ALIASES.items() if v in SUPPORTED_DOWNLOAD_DATA_SOURCES]
+
+
+@click.command()
+@click.pass_context
+@click.option('-s', '--source', required=True, type=click.Choice(SUPPORTED_DOWNLOAD_DATA_SOURCES_ALIASES_INCLUDED, case_sensitive=False), help='Data source')
+@click.option('-p', '--pdts', multiple=True, default=[], help='List of trading products')
+@click.option('--dtypes', '--dt', multiple=True, type=click.Choice(SUPPORTED_DATA_TYPES, case_sensitive=False), help='List of data types, e.g. raw, tick, second, minute, hour, daily')
+@click.option('--ptypes', '--pt', multiple=True, default=[], help='List of product types, e.g. PERP = get all perpetuals')
+@click.option('-b', '--start-date', type=click.DateTime(formats=["%Y-%m-%d"]), help='Start date in YYYY-MM-DD format')
+@click.option('-n', '--end-date', type=click.DateTime(formats=["%Y-%m-%d"]), help='End date in YYYY-MM-DD format')
+@click.option('--batch-size', default=8, type=int, help='batch size for Ray tasks')  # REVIEW
+@click.option('--no-ray', is_flag=True)
+@click.option('--no-minio', is_flag=True)
+def download(ctx, source, pdts, dtypes, ptypes, start_date, end_date, batch_size, no_ray, no_minio):
+    if source in ALIASES:
+        source = ALIASES[source]
+    pdts = [pdt.replace('-', '_') for pdt in pdts]
+    validate_pdts_and_ptypes(source, pdts, ptypes, is_cli=True)
+    if start_date:
+        start_date = start_date.date()
+    if end_date:
+        end_date = end_date.date()
+    pipeline = importlib.import_module(f'pfeed.sources.{source.lower()}.download')
+    pipeline.run(
+        pdts=list(pdts),
+        dtypes=list(dtypes),
+        ptypes=list(ptypes),
+        start_date=start_date,
+        end_date=end_date,
+        batch_size=batch_size,
+        use_ray=not no_ray,
+        use_minio=not no_minio,
+        config=ctx.obj['config'],
+    )

pfeed-0.0.1.dev2/pfeed/cli/main.py

@@ -0,0 +1,24 @@
+import click
+
+from pfeed.config_handler import ConfigHandler
+from pfeed.const.paths import USER_CONFIG_FILE_PATH
+from pfeed.cli.commands.docker_compose import docker_compose
+from pfeed.cli.commands.config import config, load_config
+from pfeed.cli.commands.download import download
+# from pfeed.cli.commands.stream import stream
+
+
+@click.group(context_settings={"help_option_names": ["-h", "--help"]})
+@click.pass_context
+@click.version_option()
+def pfeed_group(ctx):
+    """PFeed's CLI"""
+    ctx.ensure_object(dict)
+    config: dict = load_config(USER_CONFIG_FILE_PATH)
+    ctx.obj['config'] = ConfigHandler(**config)
+
+
+pfeed_group.add_command(docker_compose)
+pfeed_group.add_command(config)
+pfeed_group.add_command(download)
+# pfeed_group.add_command(stream)

{pfeed-0.0.1.dev1 → pfeed-0.0.1.dev2}/pfeed/config/logging.yml

@@ -27,7 +27,7 @@ handlers:
     level: 'DEBUG'
     formatter: 'file'
   compressed_timed_rotating_file_handler:
-    class: 'pfund.logging.handlers.CompressedTimedRotatingFileHandler'
+    class: 'pfund.plogging.handlers.CompressedTimedRotatingFileHandler'
     level: 'DEBUG'
     formatter: 'file'
     kwargs: {'when': 'midnight', 'backupCount': 7, 'utc': True, 'encoding': 'utf-8'}

pfeed-0.0.1.dev2/pfeed/config_handler.py

@@ -0,0 +1,57 @@
+import os
+import sys
+import multiprocessing
+import logging
+from types import TracebackType
+from dataclasses import dataclass
+
+from pfeed.const.paths import PROJ_NAME, PROJ_CONFIG_PATH, LOG_PATH, DATA_PATH
+
+
+def _custom_excepthook(exception_class: type[BaseException], exception: BaseException, traceback: TracebackType):
+    '''Catches any uncaught exceptions and logs them'''
+    # sys.__excepthook__(exception_class, exception, traceback)
+    try:
+        raise exception
+    except:
+        logging.getLogger(PROJ_NAME).exception('Uncaught exception:')
+
+
+@dataclass
+class ConfigHandler:
+    data_path: str = str(DATA_PATH)
+    log_path: str = str(LOG_PATH)
+    logging_config_file_path: str = f'{PROJ_CONFIG_PATH}/logging.yml'
+    logging_config: dict | None = None
+    use_fork_process: bool = True
+    use_custom_excepthook: bool = True
+    
+    def __post_init__(self):
+        for path in [self.data_path]:
+            if not os.path.exists(path):
+                os.makedirs(path)
+                print(f'created {path}')
+                
+        if self.use_fork_process and sys.platform != 'win32':
+            multiprocessing.set_start_method('fork', force=True)
+        
+        if self.use_custom_excepthook:
+            sys.excepthook = _custom_excepthook
+            
+        
+def configure(
+    data_path: str = str(DATA_PATH),
+    log_path: str = str(LOG_PATH),
+    logging_config_file_path: str = f'{PROJ_CONFIG_PATH}/logging.yml',
+    logging_config: dict | None=None,
+    use_fork_process: bool=True,
+    use_custom_excepthook: bool=True,
+):
+    return ConfigHandler(
+        data_path=data_path,
+        log_path=log_path,
+        logging_config_file_path=logging_config_file_path,
+        logging_config=logging_config,
+        use_fork_process=use_fork_process,
+        use_custom_excepthook=use_custom_excepthook,
+    )

pfeed-0.0.1.dev2/pfeed/const/commons.py

@@ -0,0 +1,6 @@
+SUPPORTED_DATA_TYPES = ['raw', 'tick', 'second', 'minute', 'hour', 'daily']
+SUPPORTED_DATA_FEEDS = ['YAHOO_FINANCE', 'BYBIT']
+SUPPORTED_DOWNLOAD_DATA_SOURCES = ['BYBIT']
+ALIASES = {
+    'YF': 'YAHOO_FINANCE',
+}

pfeed-0.0.1.dev2/pfeed/const/paths.py

@@ -0,0 +1,16 @@
+from pathlib import Path
+from platformdirs import user_log_dir, user_data_dir, user_config_dir
+
+
+# project paths
+PROJ_NAME = Path(__file__).resolve().parents[2].name
+MAIN_PATH = Path(__file__).resolve().parents[3]
+PROJ_PATH = MAIN_PATH / PROJ_NAME / PROJ_NAME
+PROJ_CONFIG_PATH = PROJ_PATH / 'config'
+
+
+# user paths
+LOG_PATH = Path(user_log_dir()) / PROJ_NAME
+DATA_PATH = Path(user_data_dir()) / PROJ_NAME
+USER_CONFIG_PATH = Path(user_config_dir()) / PROJ_NAME
+USER_CONFIG_FILE_PATH = USER_CONFIG_PATH / f'{PROJ_NAME}_config.yml'

{pfeed-0.0.1.dev1 → pfeed-0.0.1.dev2}/pfeed/datastore.py

@@ -7,8 +7,6 @@ from typing import Generator
 from minio import Minio, S3Error
 from minio.api import ObjectWriteResult
 
-from pfeed.const.paths import PROJ_NAME
-
 
 logger = logging.getLogger('minio')
 
@@ -16,11 +14,11 @@ logger = logging.getLogger('minio')
 # EXTEND, currently only consider using MinIO
 class Datastore:
     DATA_PART_SIZE = 5 * (1024 ** 2)  # part size for S3, 5 MB
-    BUCKET_NAME = PROJ_NAME + '-' + os.getenv('PFEED_ENV', 'DEV').lower()
+    BUCKET_NAME = 'pfeed' + '-' + os.getenv('PFEED_ENV', 'DEV').lower()
     
     def __init__(self, **kwargs):
         self.minio = Minio(
-            endpoint=os.getenv('MINIO_ENDPOINT'),
+            endpoint=os.getenv('MINIO_HOST')+':'+os.getenv('MINIO_PORT', '9000'),
             access_key=os.getenv('MINIO_ACCESS_KEY'),
             secret_key=os.getenv('MINIO_SECRET_KEY'),
             # turn off TLS, i.e. not using HTTPS

{pfeed-0.0.1.dev1 → pfeed-0.0.1.dev2}/pfeed/feeds/bybit_feed.py

@@ -85,7 +85,11 @@ class BybitFeed(BaseFeed):
             df = pd.read_parquet(io.BytesIO(resampled_data))
             dfs.append(df)
         return pd.concat(dfs)
-                
+    
+    # TODO?: maybe useful if used as a standalone program, not useful at all if used with PFund
+    def get_real_time_data(self, env='LIVE'):
+        pass
+    
         
 if __name__ == '__main__':
     feed = BybitFeed()

{pfeed-0.0.1.dev1 → pfeed-0.0.1.dev2}/pfeed/filepath.py

@@ -25,9 +25,9 @@ class FilePath:
         self.data_type = self.dtype = data_type.lower()
         self.mode = mode
         self.pdt = pdt.lower()
-        self.date = date
+        self.date = str(date)
         self.file_extension = file_extension
-        self.filename = create_filename(pdt.upper(), date, file_extension)
+        self.filename = create_filename(pdt.upper(), self.date, file_extension)
         self.storage_Path = self.sPath = Path(self.env) / self.data_source / self.mode / self.dtype / pdt.upper() / self.filename
         self.storage_path = self.spath = str(self.storage_Path)
         self.file_Path = self.fPath = self.data_Path / self.storage_Path

pfeed-0.0.1.dev2/pfeed/main.py

@@ -0,0 +1,10 @@
+from pfeed.cli import pfeed_group
+
+
+def run_cli() -> None:
+    """Application Entrypoint."""
+    pfeed_group(obj={})
+
+
+if __name__ == '__main__':
+    run_cli()

pfeed-0.0.1.dev2/pfeed/sources/bybit/__init__.py

@@ -0,0 +1,2 @@
+from pfeed.sources.bybit.download import download_historical_data as download
+from pfeed.sources.bybit.stream import stream_real_time_data as stream

{pfeed-0.0.1.dev1 → pfeed-0.0.1.dev2}/pfeed/sources/bybit/const.py

@@ -1,7 +1,7 @@
 DATA_SOURCE = 'BYBIT'
 # EXTEND: support FUT and IFUT
-# SUPPORTED_CRYPTO_PRODUCT_TYPES = ['SPOT', 'PERP', 'IPERP', 'FUT', 'IFUT']
-SUPPORTED_CRYPTO_PRODUCT_TYPES = ['SPOT', 'PERP', 'IPERP']
+# SUPPORTED_PRODUCT_TYPES = ['SPOT', 'PERP', 'IPERP', 'FUT', 'IFUT']
+SUPPORTED_PRODUCT_TYPES = ['SPOT', 'PERP', 'IPERP']
 RAW_DATA_TYPE = 'tick'
 # do not need to be precise
 DATA_START_DATE = '2020-01-01'