pfeed 0.0.1.dev1__py3-none-any.whl

pfeed/sources/bybit/etl.py

@@ -0,0 +1,161 @@
+'''ETL = Extract, Transform, Load data'''
+import os
+import io
+import logging
+
+from typing import Literal
+
+import numpy as np
+import pandas as pd
+
+from pfeed.const.paths import DATA_PATH
+from pfeed.sources.bybit.const import DATA_SOURCE, SELECTED_RAW_COLS, RENAMING_COLS, RAW_DATA_TIMESTAMP_UNITS
+from pfeed.datastore import Datastore
+from pfeed.filepath import FilePath
+
+
+logger = logging.getLogger(DATA_SOURCE.lower())
+
+
+def extract_data(
+    pdt: str,
+    date: str,
+    dtype: Literal['raw', 'tick', 'second', 'minute', 'hour', 'daily'],
+    env: Literal['PAPER', 'LIVE']='LIVE',
+    mode: Literal['historical', 'streaming']='historical',
+    data_path: str=str(DATA_PATH),
+) -> bytes:
+    file_extension = '.csv.gz' if dtype == 'raw' else '.parquet.gz'
+    fp = FilePath(data_path, env, DATA_SOURCE, dtype, mode, pdt, date, file_extension)
+    if os.path.exists(fp.file_path):
+        with open(fp.file_path, 'rb') as f:
+            data: bytes = f.read()
+            logger.debug(f'read data from {fp.file_path}')
+            return data
+    else:
+        datastore = Datastore()
+        object_name = fp.storage_path
+        data: bytes | None = datastore.get_object(object_name)
+        if data:
+            logger.debug(f'extracted data from MinIO object {object_name}')
+        else:
+            logger.error(f'failed to extract data from MinIO object {object_name}')
+        return data
+
+
+def load_data(
+    pdt: str,
+    date: str,
+    dtype: Literal['raw', 'tick', 'second', 'minute', 'hour', 'daily'],
+    data: bytes,
+    env: Literal['PAPER', 'LIVE']='LIVE',
+    mode: Literal['historical', 'streaming']='historical',
+    data_path: str=str(DATA_PATH),
+    use_minio=True,
+    **kwargs
+):  
+    if not os.path.exists(data_path):
+        os.makedirs(data_path)
+        print(f'created {data_path=}')
+    file_extension = '.csv.gz' if dtype == 'raw' else '.parquet.gz'
+    fp = FilePath(data_path, env, DATA_SOURCE, dtype, mode, pdt, date, file_extension)
+    if use_minio:
+        datastore = Datastore()
+        object_name = fp.storage_path
+        datastore.put_object(object_name, data, **kwargs)
+        logger.debug(f'loaded data to object {object_name} {kwargs=}')
+    else:
+        directory = os.path.dirname(fp.file_path)
+        if not os.path.exists(directory):
+            os.makedirs(directory)
+        with open(fp.file_path, 'wb') as f:
+            f.write(data)
+            logger.debug(f'loaded data to {fp.file_path}')
+
+
+def clean_data(category: str, data: bytes) -> bytes:
+    df = pd.read_csv(io.BytesIO(data), compression='gzip')
+    df = df.loc[:, SELECTED_RAW_COLS[category]]
+    df['side'] = df['side'].map({'Buy': 1, 'Sell': -1})
+    df = df.rename(columns=RENAMING_COLS[category])
+    return df.to_parquet()
+
+
+def resample_data(data: bytes, resolution: str, is_tick=False, category='') -> bytes:
+    '''
+    Args:
+        is_tick: if True, use tick data to resample data
+        resolution: # + unit (s/m/h/d), e.g. 1s
+    '''
+    def find_high_or_low_first(series: pd.Series):
+        if not series.empty:
+            argmax, argmin = series.argmax(), series.argmin()
+            if argmax < argmin:
+                return 'H'
+            elif argmax > argmin:
+                return 'L'
+            else:
+                return 'N'
+    # T means minute, refer to https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#dateoffset-objects
+    resolution = resolution.upper().replace('M', 'T')
+    df = pd.read_parquet(io.BytesIO(data))
+    if 'ts' in df.columns:
+        assert category, 'category must be provided'
+        df.set_index('ts', inplace=True)
+        # NOTE: this may make the `ts` value inaccurate, 
+        # e.g. 1671580800.9906 -> 1671580800.990600192
+        df.index = pd.to_datetime(df.index, unit=RAW_DATA_TIMESTAMP_UNITS[category])
+    if is_tick:
+        df['num_buys'] = df['side'].map({1: 1, -1: np.nan})
+        df['num_sells'] = df['side'].map({1: np.nan, -1: 1})
+        df['buy_volume'] = np.where(df['side'] == 1, df['volume'], np.nan)
+        df['sell_volume'] = np.where(df['side'] == -1, df['volume'], np.nan)
+        df['first'] = df['price']
+        resample_logic = {
+            'num_buys': 'sum',
+            'num_sells': 'sum',
+            'volume': 'sum',
+            'buy_volume': 'sum',
+            'sell_volume': 'sum',
+            'price': 'ohlc',
+            'first': find_high_or_low_first
+        }
+    else:
+        df['arg_high'] = df['high']
+        df['arg_low'] = df['low']
+        resample_logic = {
+            'num_buys': 'sum',
+            'num_sells': 'sum',
+            'buy_volume': 'sum',
+            'sell_volume': 'sum',
+            'volume': 'sum',
+            'open': 'first',
+            'high': 'max',
+            'low': 'min',
+            'close': 'last',
+            'arg_high': lambda series: series.argmax() if not series.empty else None,
+            'arg_low': lambda series: series.argmin() if not series.empty else None,
+        }
+    resampled_df = (
+        df
+        .resample(resolution)
+        .apply(resample_logic)
+        .dropna()
+    )
+    if is_tick:
+        resampled_df = resampled_df.droplevel(0, axis=1)
+        # convert float to int
+        resampled_df['num_buys'] = resampled_df['num_buys'].astype(int)
+        resampled_df['num_sells'] = resampled_df['num_sells'].astype(int)
+    else:
+        resampled_df['first'] = (resampled_df['arg_high'] < resampled_df['arg_low']).map({True: 'H', False: 'L'})
+        resampled_df['first'].where(resampled_df['arg_high'] != resampled_df['arg_low'], other='N', inplace=True)
+        resampled_df.drop(columns=['arg_high', 'arg_low'], inplace=True)
+    return resampled_df.to_parquet()
+
+
+if __name__ == '__main__':
+    pdt = 'BTC_USDT_PERP'
+    date = '2023-11-02'
+    dtype = 'raw'
+    data: bytes = extract_data(pdt, date, dtype)

pfeed/sources/bybit/historical.py

@@ -0,0 +1,151 @@
+import os
+import logging
+import datetime
+from collections import defaultdict
+from logging.handlers import QueueHandler, QueueListener
+
+import yaml
+from tqdm import tqdm
+
+from pfeed.utils.utils import get_dates_in_between
+from pfeed.const.commons import SUPPORTED_DATA_TYPES
+from pfeed.const.paths import PROJ_PATH, DATA_PATH, LOG_PATH, CONFIG_PATH
+from pfeed.sources.bybit.const import DATA_START_DATE, DATA_SOURCE, SUPPORTED_CRYPTO_PRODUCT_TYPES, create_efilename
+from pfeed.sources.bybit import api
+from pfeed.sources.bybit import etl
+
+
+logger = logging.getLogger(DATA_SOURCE.lower())
+__all__ = ['run']
+
+
+def _load_config():
+    file_path = f'{PROJ_PATH}/sources/{DATA_SOURCE.lower()}/config.yml'
+    short_file_path = '/'.join(file_path.split("/")[-4:])
+    if os.path.exists(file_path):
+        with open(file_path, 'r') as f:
+            print(f'loaded config {short_file_path}')
+            return yaml.safe_load(f.read())
+
+
+def get_pdts(exchange, pdts, ptypes) -> list[str]:
+    if not pdts:
+        adapter = exchange.adapter
+        pdts = []
+        if not ptypes:
+            ptypes = SUPPORTED_CRYPTO_PRODUCT_TYPES
+        for ptype in ptypes:
+            assert ptype in SUPPORTED_CRYPTO_PRODUCT_TYPES, f'{ptype} is not supported, {SUPPORTED_CRYPTO_PRODUCT_TYPES=}'
+            category = exchange.categorize_product(ptype)
+            epdts = api.get_epdts(category, ptype)
+            # NOTE: if adapter(epdt, ref_key=category) == epdt, i.e. key is not found in pdt matching, meaning the product has been delisted
+            pdts.extend([adapter(epdt, ref_key=category) for epdt in epdts if adapter(epdt, ref_key=category) != epdt])
+    return pdts
+
+
+def resample_raw_data(category: str, pdt: str, date: str, raw_data: bytes) -> dict[str, bytes]:
+    tick_data = etl.clean_data(category, raw_data)
+    second_data = etl.resample_data(tick_data, resolution='1s', is_tick=True, category=category)
+    minute_data = etl.resample_data(second_data, resolution='1m')
+    hour_data = etl.resample_data(minute_data, resolution='1h')
+    daily_data = etl.resample_data(hour_data, resolution='1d')
+    logger.debug(f'resampled {DATA_SOURCE} {pdt} {date} data')
+    return {
+        'raw': raw_data,
+        'tick': tick_data,
+        'second': second_data,
+        'minute': minute_data,
+        'hour': hour_data,
+        'daily': daily_data,
+    }
+
+
+def run(
+    dtypes: list[str] | None=None,
+    ptypes: list[str] | None=None, 
+    pdts: list[str] | None=None, 
+    start_date: str | None=None, 
+    end_date: str | None=None, 
+    log_path: str=str(LOG_PATH),
+    data_path: str=str(DATA_PATH),
+    batch_size: int=8,
+    use_ray: bool=True, 
+    use_minio: bool=True
+):
+    from pfund.exchanges.bybit.exchange import Exchange
+    from pfund.logging import set_up_loggers
+    from pfeed import cprint
+    
+    env = 'LIVE'  # historical data is from LIVE env
+    mode = 'historical'
+    dtypes = [dtype.lower() for dtype in dtypes] if dtypes else SUPPORTED_DATA_TYPES[:]
+    assert all(dtype in SUPPORTED_DATA_TYPES for dtype in dtypes), f'{dtypes=} but {SUPPORTED_DATA_TYPES=}'
+    ptypes = [ptype.upper() for ptype in ptypes] if ptypes else []
+    pdts = [pdt.upper() for pdt in pdts] if pdts else []
+    set_up_loggers(log_path=f'{log_path}/{env}', config_path=CONFIG_PATH)
+        
+    cprint(f'PFeed ({DATA_SOURCE} {env} server): getting {mode.upper()} data', style='bold yellow')
+    
+    # load config if any
+    if config := _load_config():
+        ptypes = ptypes or config.get('ptypes', None)
+        pdts = pdts or config.get('pdts', None)
+        start_date = start_date or config.get('start_date', None)
+        end_date = end_date or config.get('end_date', None)
+        
+    exchange = Exchange(env)
+    adapter = exchange.adapter
+    pdts = get_pdts(exchange, pdts, ptypes)
+    start_date: str = start_date or DATA_START_DATE
+    end_date: str = end_date or datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%d')
+    dates: list[str] = get_dates_in_between(start_date, end_date)
+    ray_tasks = defaultdict(list)
+
+    for pdt in pdts if use_ray else tqdm(pdts, desc=f'Downloading {DATA_SOURCE} historical data by product', colour='green'):
+        ptype = pdt.split('_')[-1]
+        is_spot = (ptype.upper() == 'SPOT')
+        category = exchange.categorize_product(ptype)
+        epdt = adapter(pdt, ref_key=category)
+        efilenames = api.get_efilenames(category, epdt)
+        for date in dates if use_ray else tqdm(dates, desc=f'Downloading {DATA_SOURCE} {pdt} historical data by date', colour='yellow'):
+            efilename = create_efilename(epdt, date, is_spot=is_spot)
+            if efilename not in efilenames:
+                # logger.debug(f'{efilename} does not exist in {DATA_SOURCE}')
+                continue
+            if use_ray:
+                ray_tasks[pdt].append((category, pdt, date))
+            else:
+                # run ETL = Extract, Transform, Load data
+                if raw_data := api.get_data(category, epdt, date):
+                    resampled_datas = resample_raw_data(category, pdt, date, raw_data)
+                    for dtype in dtypes:
+                        etl.load_data(pdt, date, dtype, resampled_datas[dtype], data_path=data_path, use_minio=use_minio)
+                else:
+                    raise Exception(f'failed to download {DATA_SOURCE} {pdt} {date} historical data')
+
+    if use_ray:
+        import ray
+        from ray.util.queue import Queue
+
+        @ray.remote
+        def _run_task(log_queue: Queue, category: str, pdt: str, date: str):
+            if not logger.handlers:
+                logger.addHandler(QueueHandler(log_queue))
+                logger.setLevel(logging.DEBUG)
+            epdt = adapter(pdt, ref_key=category)
+            if raw_data := api.get_data(category, epdt, date):
+                resampled_datas = resample_raw_data(category, pdt, date, raw_data)
+                for dtype in dtypes:
+                    etl.load_data(pdt, date, dtype, resampled_datas[dtype], data_path=data_path, use_minio=use_minio)
+            else:
+                raise Exception(f'failed to download {DATA_SOURCE} {pdt} {date} historical data')
+
+        log_queue = Queue()
+        QueueListener(log_queue, *logger.handlers, respect_handler_level=True).start()
+        for pdt in tqdm(ray_tasks, desc=f'Downloading {DATA_SOURCE} historical data by product', colour='green'):
+            batches = [ray_tasks[pdt][i: i + batch_size] for i in range(0, len(ray_tasks[pdt]), batch_size)]
+            for batch in tqdm(batches, desc=f'Downloading {DATA_SOURCE} {pdt} historical data by batch ({batch_size=})', colour='yellow'):
+                futures = [_run_task.remote(log_queue, *task) for task in batch]
+                ray.get(futures)
+                
+    logger.warning(f'finished downloading historical data from {DATA_SOURCE} to {data_path} or MinIO if enabled')

pfeed/utils/utils.py

@@ -0,0 +1,77 @@
+import re
+import datetime
+import calendar
+import pytz
+
+
+def get_TZ_abbrev_and_UTC_offset(date: str, tz_identifier='US/Eastern'):
+    '''Returns timezone abbreviation (e.g. EST, EDT) based on the timezone identifier.
+    Useful when you want to determine if e.g. New York is in EST or EDT now.
+    Args:
+        date: e.g. 2023-11-11
+        timezone: TZ identifier from IANA timezone database
+    '''
+    date = datetime.datetime.strptime(date, '%Y-%m-%d')
+    timezone = pytz.timezone(tz_identifier)
+    local_date = timezone.localize(date)  # attach timezone to the datetime object 
+    return local_date.strftime('%Z%z')
+
+
+def get_x_days_before_in_UTC(x=0) -> str:
+    return (datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(days=x)).strftime('%Y-%m-%d')
+
+
+def get_dates_in_between(start_date: str, end_date: str) -> list[str]:
+    start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d').date()
+    end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d').date()
+    date_delta = end_date - start_date
+    return [str(start_date + datetime.timedelta(i)) for i in range(date_delta.days + 1)]
+
+
+def create_filename(pdt: str, date: str, file_extension: str) -> str:
+    return pdt + '_' + date + file_extension
+
+
+def extract_date_from_filename(filename: str) -> str | None:
+    date_pattern = r'\d{4}-\d{2}-\d{2}'
+    match = re.search(date_pattern, filename)
+    if match:
+        date = match.group(0)
+        return date
+
+
+def rollback_date_range(rollback_period: str) -> tuple[str, str]:
+    '''Returns start_date and end_date based on the rollback_period (e.g. '1w', '1M').'''
+    def _nextmonth(year, month):
+        if month == 12:
+            return year+1, 1
+        else:
+            return year, month+1
+    utcnow = datetime.datetime.now(tz=datetime.timezone.utc)
+    period = int(rollback_period[:-1])
+    if rollback_period.endswith('d'):
+        timedelta = datetime.timedelta(days=period)
+    elif rollback_period.endswith('w'):
+        timedelta = datetime.timedelta(weeks=period)
+    elif rollback_period.endswith('M'):
+        year, month = utcnow.year, utcnow.month - period
+        while month <= 0:
+            month += 12  # Rollback to the previous year
+            year -= 1
+        total_days_in_month = 0
+        while not (year == utcnow.year and month == utcnow.month):
+            year, month = _nextmonth(year, month)
+            _, days_in_month = calendar.monthrange(year, month)
+            total_days_in_month += days_in_month
+        timedelta = datetime.timedelta(days=total_days_in_month)
+    elif rollback_period.endswith('y'):
+        year = utcnow.year - period
+        total_days_in_year = 0
+        for yr in range(year, utcnow.year):
+            total_days_in_year += 366 if calendar.isleap(yr) else 365
+        timedelta = datetime.timedelta(days=total_days_in_year)
+    else:
+        raise ValueError(f"Unsupported {rollback_period=}")
+    end_date = utcnow - datetime.timedelta(days=1)  # Previous day
+    start_date = end_date - timedelta + datetime.timedelta(days=1)
+    return start_date.strftime('%Y-%m-%d'), end_date.strftime('%Y-%m-%d')

pfeed-0.0.1.dev1.dist-info/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pfeed-0.0.1.dev1.dist-info/METADATA

@@ -0,0 +1,42 @@
+Metadata-Version: 2.1
+Name: pfeed
+Version: 0.0.1.dev1
+Summary: A Modern Data Pipeline for Algo-Trading, supporting getting both Real-Time and Historical Data.
+Author: Stephen Yau
+Author-email: softwareentrepreneer+pfeed@gmail.com
+Requires-Python: >=3.10,<3.12
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0.0)
+Requires-Dist: minio (>=7.2.3,<8.0.0)
+Requires-Dist: orjson (>=3.9.12,<4.0.0)
+Requires-Dist: pandas (>=2.2.0,<3.0.0)
+Requires-Dist: pyarrow (>=15.0.0,<16.0.0)
+Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
+Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
+Requires-Dist: ray (>=2.9.1,<3.0.0)
+Requires-Dist: requests (>=2.31.0,<3.0.0)
+Requires-Dist: rich (>=13.7.0,<14.0.0)
+Requires-Dist: tqdm (>=4.66.1,<5.0.0)
+Requires-Dist: yfinance (>=0.2.36,<0.3.0)
+Description-Content-Type: text/markdown
+
+## Setup
+```bash
+poetry install
+```
+
+## Start Processing Historical Data
+```
+python main.py -m historical -p BTC_USDT_PERP -s bybit --no-minio
+# or
+from pfeed import bybit
+bybit.run_historical(...)
+```
+
+## Start Streaming Live Data (supports PAPER trading)
+```python
+python main.py -e live -s bybit -m streaming
+```
+

pfeed-0.0.1.dev1.dist-info/RECORD

@@ -0,0 +1,27 @@
+pfeed/.DS_Store,sha256=VopWakmBjGn9kpV7l3WwYvr4ZpufKwmHA97oYMaAwn8,6148
+pfeed/__init__.py,sha256=4qzQygCmy0QyU8lJuaKbs0bCnZVvQf1UiYIYCSuf62U,376
+pfeed/config/logging.yml,sha256=9pOePCcMAQsK2urN_Km73wzH4xdtnBobntXlJa5olY0,1591
+pfeed/const/commons.py,sha256=Mh-hYF_RRpbSB-WbAE_7e3WnVWxGkOXm-cTQ9wNir9o,125
+pfeed/const/paths.py,sha256=IZyl4SK_RQ2KgwqctZury4BvBjpSv_GVffDbMZJLAgQ,293
+pfeed/datastore.py,sha256=Uku0v061Os05J2fqFVt0__JzK4V4Dh5C_eR-Lz8ABZM,3469
+pfeed/feeds/__init__.py,sha256=LgBB4xo_6TC630RLT1gmrVLbbL6jezqXgaiB9JGAbjE,104
+pfeed/feeds/base_feed.py,sha256=UNlhS3grajkuyZWrsknLxHbgHq2hl74A41CJg4dH2EE,79
+pfeed/feeds/bybit_feed.py,sha256=2MsrrEDVxbIbRZCTJLBtJ1wTYsF9gw3j90YCbulaSsM,3815
+pfeed/feeds/custom_csv_feed.py,sha256=qTkanCebga7Vcrz9Gus0YXX6ZZkhxwiXw02ID9zmZ5M,281
+pfeed/feeds/yahoo_finance_feed.py,sha256=yCNopqZ9lBdl_gVDnvglB1l8ReO74Hy9c5gINNNSeZ4,3326
+pfeed/filepath.py,sha256=eCsz7VcgUr01r0pU-SI41SNh2OVxFT0lakXIGSsLqvg,1324
+pfeed/main.py,sha256=NKYiad7zcHUbJAg1IAwZd3WL6arRvQe9Z4VxBt8pzjs,3168
+pfeed/sources/.DS_Store,sha256=bprxU9J9w5pEM8T_WyRU2_WKbdzscUWgfmLwHfkMw90,6148
+pfeed/sources/__init__.py,sha256=jzafyaYJFj8G2NTuJNqKShqpYjS8NGgb4BskvEH6r6Q,32
+pfeed/sources/bybit/__init__.py,sha256=4LT5I1gsoAD0oJWEbTGq2_ZSakgv0RxcVAVDsfpD5XY,111
+pfeed/sources/bybit/api.py,sha256=hE67WGRG9GbqXbwngazyQlsbYXijh93nTbiNX-PUrq8,2111
+pfeed/sources/bybit/config.yml,sha256=WI2F5Eh96PU_m-mW5pj7g9rkb-nSs-xgug8IxX0-WIA,47
+pfeed/sources/bybit/const.py,sha256=dyhtcOnVsO2iJSqOBn2G6s2xhWATr-ZOcgj1aSimQOE,1398
+pfeed/sources/bybit/eda.ipynb,sha256=4CTHgT1VjGHdQOyDBUSd63Uq0mxxyCARTTLdmwC0wys,4405251
+pfeed/sources/bybit/etl.py,sha256=YVNkxGidNpTKQJ5FMTSO3tCeDSkkS8LALmSi9Nwpq3M,5823
+pfeed/sources/bybit/historical.py,sha256=dP8LBsdTDbw9eX-_vMO15g_bbLfY765wX2fXMgOLjPQ,6879
+pfeed/utils/utils.py,sha256=B0JjbPxJNbMEjv5lGhwOlpEEUT9eV3p28CpTt4TjiVY,3122
+pfeed-0.0.1.dev1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+pfeed-0.0.1.dev1.dist-info/METADATA,sha256=P3lwrBdScqxszoZNqUcZiUsdytjPK1B_2xQmJ5xdTEA,1257
+pfeed-0.0.1.dev1.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+pfeed-0.0.1.dev1.dist-info/RECORD,,

pfeed-0.0.1.dev1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.8.1
+Root-Is-Purelib: true
+Tag: py3-none-any