pfeed 0.0.1.dev1__py3-none-any.whl

pfeed/__init__.py

@@ -0,0 +1,20 @@
+import multiprocessing
+
+from dotenv import load_dotenv, find_dotenv
+from rich.console import Console
+
+from pfeed.sources import bybit
+from pfeed.feeds import YahooFinanceFeed, BybitFeed
+
+
+cprint = Console().print
+load_dotenv(find_dotenv())
+multiprocessing.set_start_method('fork', force=True)
+
+
+__all__ = (
+    'bybit',
+    'YahooFinanceFeed',
+    'BybitFeed',
+    'cprint',
+)

pfeed/config/logging.yml

@@ -0,0 +1,47 @@
+version: 1
+filename_format: '%Y-%m-%d_UTC%z'
+loggers:
+  root:
+    level: 'DEBUG'
+    handlers: ['compressed_timed_rotating_file_handler', 'stream_handler']
+    propagate: False
+  pfeed:
+    level: 'DEBUG'
+    handlers: ['compressed_timed_rotating_file_handler', 'stream_handler']
+    propagate: False
+  minio:
+    level: 'DEBUG'
+    handlers: ['compressed_timed_rotating_file_handler', 'stream_handler']
+    propagate: False
+  yahoo_finance:
+    level: 'DEBUG'
+    handlers: ['compressed_timed_rotating_file_handler', 'stream_handler']
+    propagate: False
+  bybit:
+    level: 'DEBUG'
+    handlers: ['compressed_timed_rotating_file_handler', 'stream_handler']
+    propagate: False
+handlers:
+  file_handler:
+    class: 'logging.FileHandler'
+    level: 'DEBUG'
+    formatter: 'file'
+  compressed_timed_rotating_file_handler:
+    class: 'pfund.logging.handlers.CompressedTimedRotatingFileHandler'
+    level: 'DEBUG'
+    formatter: 'file'
+    kwargs: {'when': 'midnight', 'backupCount': 7, 'utc': True, 'encoding': 'utf-8'}
+  stream_handler:
+    class: 'logging.StreamHandler'
+    level: 'DEBUG'
+    formatter: 'console'
+formatters:
+  path:
+    format: '%(asctime)s.%(msecs)03d | %(levelname)s | %(name)s | %(message)s | %(shortpath)s fn:%(funcName)s ln:%(lineno)d'
+    datefmt: '%H:%M:%S%z'
+  file:
+    format: '%(asctime)s.%(msecs)03d | %(levelname)s | %(message)s | %(filename)s fn:%(funcName)s ln:%(lineno)d'
+    datefmt: '%H:%M:%S%z'
+  console:
+    format: '%(asctime)s.%(msecs)03d | %(levelname)s | %(name)s | %(message)s | fn:%(funcName)s ln:%(lineno)d'
+    datefmt: '%Y-%m-%d %H:%M:%S%z'

pfeed/const/commons.py

@@ -0,0 +1,2 @@
+SUPPORTED_DATA_TYPES = ['raw', 'tick', 'second', 'minute', 'hour', 'daily']
+SUPPORTED_DATA_FEEDS = ['YAHOO_FINANCE', 'BYBIT']

pfeed/const/paths.py

@@ -0,0 +1,9 @@
+from pathlib import Path
+
+
+PROJ_NAME = Path(__file__).resolve().parents[2].name
+MAIN_PATH = Path(__file__).resolve().parents[3]
+PROJ_PATH = MAIN_PATH / PROJ_NAME / PROJ_NAME
+CONFIG_PATH = PROJ_PATH / 'config'
+LOG_PATH = MAIN_PATH / PROJ_NAME / 'logs'
+DATA_PATH = MAIN_PATH / PROJ_NAME / 'data'

pfeed/datastore.py

@@ -0,0 +1,117 @@
+import os
+import io
+import logging
+
+from typing import Generator
+
+from minio import Minio, S3Error
+from minio.api import ObjectWriteResult
+
+from pfeed.const.paths import PROJ_NAME
+
+
+logger = logging.getLogger('minio')
+
+
+# EXTEND, currently only consider using MinIO
+class Datastore:
+    DATA_PART_SIZE = 5 * (1024 ** 2)  # part size for S3, 5 MB
+    BUCKET_NAME = PROJ_NAME + '-' + os.getenv('PFEED_ENV', 'DEV').lower()
+    
+    def __init__(self, **kwargs):
+        self.minio = Minio(
+            endpoint=os.getenv('MINIO_ENDPOINT'),
+            access_key=os.getenv('MINIO_ACCESS_KEY'),
+            secret_key=os.getenv('MINIO_SECRET_KEY'),
+            # turn off TLS, i.e. not using HTTPS
+            secure=True if os.getenv('PFEED_ENV', 'DEV').upper() == 'PRD' else False,
+            **kwargs,
+        )
+
+    def __getattr__(self, attr):
+        '''gets triggered only when the attribute is not found'''
+        return getattr(self.minio, attr)
+    
+    def get_object(self, object_name: str) -> bytes | None:
+        try:
+            bucket_name = self.BUCKET_NAME
+            res = self.minio.get_object(bucket_name, object_name)
+            if res.status == 200:
+                return res.data
+            else:
+                logger.error(f'Unhandled MinIO response status {res.status}')
+        except S3Error as err:
+            # logger.warning(f'MinIO S3Error {object_name=} {err=}')
+            pass
+
+    def list_objects(self, prefix) -> list | None:
+        '''
+            Args:
+                prefix: e.g. live/bybit/historical/raw/BTC_USDT_PERP/
+        '''
+        bucket_name = self.BUCKET_NAME
+        objects: Generator = self.minio.list_objects(bucket_name, prefix=prefix)
+        return list(objects)
+    
+    def put_object(self, object_name: str, data: bytes, **kwargs) -> ObjectWriteResult:
+        bucket_name = self.BUCKET_NAME
+        if not self.minio.bucket_exists(bucket_name):
+            self.minio.make_bucket(bucket_name)
+        return self.minio.put_object(
+            bucket_name,
+            object_name,
+            data=io.BytesIO(data),
+            part_size=self.DATA_PART_SIZE,
+            length=-1,
+            **kwargs
+        )
+        
+            
+if __name__ == '__main__':
+    datastore = Datastore()
+    # list buckets
+    # buckets = datastore.list_buckets()
+    # for bucket in buckets:
+    #     print(bucket.name, bucket.creation_date)
+
+    # list objects
+    objects = datastore.list_objects()
+    for obj in objects:
+        print(obj.object_name)
+    
+    # get object
+    # data = datastore.get_object(
+    #     object_name="live/bybit/historical/raw/BTC_USDT_PERP/BTC_USDT_PERP_2023-11-01.csv.gz"
+    # )
+    
+    # put object
+    # datastore.put_object(
+    #     bucket_name='test',
+    #     object_name='test_prefix/test',
+    #     data=b'test',
+    #     part_size=1024**2 * 5,
+    # )
+    
+    # upload a file
+    # datastore.fput_object(
+    #     bucket_name="dev",
+    #     object_name="test_prefix/test",
+    #     file_path=f"{PROJ_PATH}/test_data/test.txt"
+    # )
+
+    # get object info
+    # res = datastore.stat_object(
+    #     bucket_name="dev",
+    #     object_name="test_prefix/test"
+    # )
+
+
+    # copy an object from one prefix to another
+    # res = datastore.copy_object(
+    #     bucket_name="dev",
+    #     object_name="new_prefix/test",
+    #     source=CopySource(
+    #         bucket_name='dev',
+    #         object_name='test_prefix/test'
+    #     )
+    # )

pfeed/feeds/__init__.py

@@ -0,0 +1,2 @@
+from pfeed.feeds.yahoo_finance_feed import YahooFinanceFeed
+from pfeed.feeds.bybit_feed import BybitFeed

pfeed/feeds/base_feed.py

@@ -0,0 +1,3 @@
+class BaseFeed:
+    def __init__(self, name):
+        self.name = name.upper()

pfeed/feeds/bybit_feed.py

@@ -0,0 +1,93 @@
+import io
+import datetime
+
+import pandas as pd
+
+from pfeed.feeds.base_feed import BaseFeed
+from pfeed.sources.bybit import api
+from pfeed.sources.bybit import etl
+from pfeed.sources.bybit.const import DATA_SOURCE, create_efilename
+from pfeed.const.paths import DATA_PATH
+from pfeed.utils.utils import get_dates_in_between, rollback_date_range
+
+
+__all__ = ['BybitFeed']
+
+
+class BybitFeed(BaseFeed):
+    def __init__(self):
+        super().__init__('bybit')
+    
+    @staticmethod
+    def _derive_dtype_from_resolution(resolution):
+        from pfund.datas.resolution import Resolution
+        resolution = Resolution(resolution)
+        if resolution.is_tick():
+            return 'tick'
+        elif resolution.is_second():
+            return 'second'
+        elif resolution.is_minute():
+            return 'minute'
+        elif resolution.is_hour():
+            return 'hour'
+        elif resolution.is_day():
+            return 'daily'
+        else:
+            raise Exception(f'{resolution=} is not supported')
+    
+    def get_historical_data(
+        self,
+        pdt: str,
+        rollback_period: str='1w',
+        resolution: str='1d',
+        start_date: str=None,
+        end_date: str=None,
+        data_path: str=str(DATA_PATH),
+    ) -> pd.DataFrame:
+        from pfund.exchanges.bybit.exchange import Exchange
+        
+        env = 'LIVE'  # historical data is from LIVE env
+        exchange = Exchange(env)
+        adapter = exchange.adapter
+        dtype = self._derive_dtype_from_resolution(resolution)
+        ptype = pdt.split('_')[-1]
+        is_spot = (ptype.upper() == 'SPOT')
+        category = exchange.categorize_product(ptype)
+        epdt = adapter(pdt, ref_key=category)
+        efilenames = api.get_efilenames(category, epdt)
+        
+        if start_date:
+            # default for end_date is yesterday
+            end_date: str = end_date or (datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+        else:
+            start_date, end_date = rollback_date_range(rollback_period)
+        dates: list[str] = get_dates_in_between(start_date, end_date)
+        
+        dfs = []
+        for date in dates:
+            data_str = f'{DATA_SOURCE} {pdt} {date}'
+            efilename = create_efilename(epdt, date, is_spot=is_spot)
+            if efilename not in efilenames:
+                print(f'{efilename} does not exist in {DATA_SOURCE}')
+                continue
+            if local_data := etl.extract_data(pdt, date, dtype, env=env, mode='historical', data_path=data_path):
+                # e.g. local_data could be 1m data (period always = 1), but resampled_data could be 3m data
+                resampled_data: bytes = etl.resample_data(local_data, resolution, is_tick=True if dtype == 'tick' else False, category=category)
+                print(f'loaded {data_str} local {dtype} data')
+            else:
+                print(f'Downloading {data_str} data on the fly, please consider using {DATA_SOURCE.lower()}.run_historical(...) to pre-download data to your local computer first')
+                if raw_data := api.get_data(category, epdt, date):
+                    tick_data: bytes = etl.clean_data(category, raw_data)
+                    resampled_data: bytes = etl.resample_data(tick_data, resolution, is_tick=True, category=category)
+                    print(f'resampled {data_str} data to {resolution=}')
+                else:
+                    raise Exception(f'failed to download {data_str} historical data')
+            df = pd.read_parquet(io.BytesIO(resampled_data))
+            dfs.append(df)
+        return pd.concat(dfs)
+                
+        
+if __name__ == '__main__':
+    feed = BybitFeed()
+    df = feed.get_historical_data('BCH_USDT_PERP', resolution='1d', rollback_period='2d')
+    print(df)

pfeed/feeds/custom_csv_feed.py

@@ -0,0 +1,13 @@
+from pfeed.feeds.base_feed import BaseFeed
+
+
+__all__ = ['CustomCsvFeed']
+
+
+# TODO
+class CustomCsvFeed(BaseFeed):
+    def __init__(self, name='custom_csv'):
+        super().__init__(name)
+        
+    def get_historical_data(self, *args, **kwargs):
+        raise NotImplementedError

pfeed/feeds/yahoo_finance_feed.py

@@ -0,0 +1,96 @@
+import datetime
+
+import pandas as pd
+import yfinance as yf
+
+from pfeed.feeds.base_feed import BaseFeed
+
+
+__all__ = ['YahooFinanceFeed']
+
+
+class YahooFinanceFeed(BaseFeed):
+    _ADAPTER = {
+        'timeframe': {
+            # pfund's : yfinance's
+            'M': 'mo',
+            'w': 'wk',
+        }
+    }
+    # yfinance's valid intervals: [1m, 2m, 5m, 15m, 30m, 60m, 90m, 1h, 1d, 5d, 1wk, 1mo, 3mo]
+    SUPPORTED_TIMEFRAMES_AND_PERIODS = {
+        'm': [1, 2, 5, 15, 30, 60, 90],
+        'h': [1],
+        'd': [1, 5],
+        'w': [1],
+        'M': [1, 3],
+    }
+    
+    def __init__(self):
+        super().__init__('yahoo_finance')
+    
+    def get_ticker(self, symbol):
+        return yf.Ticker(symbol.upper())
+    
+    def get_historical_data(
+        self, 
+        symbol: str,
+        rollback_period: str='1M', 
+        resolution: str='1d', 
+        start_date: str=None,
+        end_date: str=None, 
+        **kwargs
+    ) -> pd.DataFrame:
+        """Simple Wrapper of yfinance history().
+        For the details of args and kwargs, please refer to https://github.com/ranaroussi/yfinance
+        """
+        from pfund.datas.resolution import Resolution
+        
+        # convert pfund's rollback_period format to yfinance's period
+        rollback_period = Resolution(rollback_period)
+        timeframe = repr(rollback_period.timeframe)
+        etimeframe = self._ADAPTER['timeframe'].get(timeframe, timeframe)
+        erollback_period = str(rollback_period.period) + etimeframe
+        # if user is directly using yfinance variable `period`, use it
+        if 'period' in kwargs:
+            period = kwargs['period']
+            del kwargs['period']
+        else:
+            period = erollback_period
+            
+        # convert pfund's resolution format to yfinance's interval
+        resolution = Resolution(resolution)
+        timeframe = repr(resolution.timeframe)
+        etimeframe = self._ADAPTER['timeframe'].get(timeframe, timeframe)
+        eresolution = str(resolution.period) + etimeframe
+        # if user is directly using yfinance variable `interval`, use it
+        if 'interval' in kwargs:
+            interval = kwargs['interval']
+            del kwargs['interval']
+        else:
+            interval = eresolution
+        
+        if start_date:
+            # default for end_date is today
+            end_date = end_date or datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%d')
+        else:
+            start_date = end_date = None
+        
+        ticker = self.get_ticker(symbol)
+        df = ticker.history(period=period, interval=interval, start=start_date, end=end_date, **kwargs)
+        df.rename_axis('ts', inplace=True)  # rename index 'Date' to 'ts'
+        df.columns = df.columns.str.lower()
+        # if there are spaces in column names, they will be turned into some weird names like "_10" 
+        # during "for row in df.itertuples()"
+        df = df.rename(columns={'stock splits': 'stock_splits'})
+        # convert to UTC
+        df.index = df.index.tz_convert('UTC')
+        # convert to UTC and remove +hh:mm from YYYY-MM-DD hh:mm:ss+hh:mm
+        # df.index = df.index.tz_convert('UTC').tz_localize(None)
+        return df
+    
+    
+if __name__ == '__main__':
+    feed = YahooFinanceFeed()
+    df = feed.get_historical_data('TSLA')
+    print(df)

pfeed/filepath.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+
+from typing import Literal
+
+from pfeed.utils.utils import create_filename
+
+
+class FilePath:
+    '''Simple wrapper for file path to extract info faster'''
+    def __init__(
+            self, 
+            data_path: str, 
+            env: Literal['PAPER', 'LIVE'], 
+            data_source: str, 
+            data_type: Literal['raw', 'tick', 'second', 'minute', 'hour', 'daily'], 
+            mode: Literal['historical', 'streaming'],
+            pdt: str,
+            date: str,
+            file_extension: str,
+        ):
+        self.data_path = self.dpath = data_path
+        self.data_Path = self.dPath = Path(data_path)
+        self.env = env.lower()
+        self.data_source = data_source.lower()
+        self.data_type = self.dtype = data_type.lower()
+        self.mode = mode
+        self.pdt = pdt.lower()
+        self.date = date
+        self.file_extension = file_extension
+        self.filename = create_filename(pdt.upper(), date, file_extension)
+        self.storage_Path = self.sPath = Path(self.env) / self.data_source / self.mode / self.dtype / pdt.upper() / self.filename
+        self.storage_path = self.spath = str(self.storage_Path)
+        self.file_Path = self.fPath = self.data_Path / self.storage_Path
+        self.file_path = self.fpath = str(self.file_Path)
+        
+    

pfeed/main.py

@@ -0,0 +1,72 @@
+import os
+import sys
+import logging
+import importlib
+from types import TracebackType
+from argparse import ArgumentParser
+
+
+from pfeed.const.paths import LOG_PATH, DATA_PATH
+
+
+ALIASES = {
+    'yf': 'yahoo_finance',
+}
+
+
+def _custom_excepthook(exception_class: type[BaseException], exception: BaseException, traceback: TracebackType):
+    '''Catches any uncaught exceptions and logs them'''
+    # sys.__excepthook__(exception_class, exception, traceback)
+    try:
+        raise exception
+    except:
+        logging.getLogger('pfeed').exception('Uncaught exception:')
+
+
+if __name__ == '__main__':
+    sys.excepthook = _custom_excepthook
+    
+    parser = ArgumentParser()
+    parser.add_argument('-e', '--env', choices=['PAPER', 'LIVE'], default='LIVE', required=False)
+    parser.add_argument('--data-path', dest='data_path', default=str(DATA_PATH), required=False)
+    parser.add_argument('--log-path', dest='log_path', default=str(LOG_PATH), required=False)
+    parser.add_argument('-m', '--mode', default='historical', help='historical=historical data processing; streaming=live data streaming', choices=['historical', 'streaming'], required=False)
+    parser.add_argument('-b', '--start-date', dest='start_date', help='Start date in YYYY-MM-DD format', required=False)
+    parser.add_argument('-n', '--end-date', dest='end_date', help='End date in YYYY-MM-DD format', required=False)
+    parser.add_argument('--ptypes', nargs='+', help='List of product types, e.g. PERP = get all perpetuals', required=False)
+    parser.add_argument('-p', '--pdts', nargs='+', help='List of trading products', required=False)
+    parser.add_argument('--dtypes', nargs='+', help='List of data types, e.g. raw, tick, second, minute, hour, daily', required=False)
+    parser.add_argument(
+        '-s', '--source', required=True,
+        choices=[
+            'bybit',
+        ],
+    )
+    parser.add_argument('-z', '--batch-size', dest='batch_size', default=8, help='batch size for Ray tasks', type=int, required=False)
+    parser.add_argument('--no-ray', dest='no_ray', action='store_true', required=False)
+    parser.add_argument('--no-minio', dest='no_minio', action='store_true', required=False)
+    
+    # handle arguments
+    args = parser.parse_args()
+    print("Ray is disabled") if args.no_ray else print("Ray is enabled")
+    print("MinIO is disabled") if args.no_minio or os.getenv('MINIO_ENDPOINT') is None else print("MinIO is enabled")
+    env, source, mode = args.env.upper(), args.source.lower(), args.mode.lower()
+
+    source = ALIASES.get(source, source)
+    pipeline = importlib.import_module(f'pfeed.sources.{source}.{mode}')
+    if mode == 'historical':
+        pipeline.run(
+            dtypes=args.dtypes,
+            ptypes=args.ptypes,
+            pdts=args.pdts,
+            start_date=args.start_date,
+            end_date=args.end_date,
+            log_path=args.log_path,
+            data_path=args.data_path,
+            batch_size=args.batch_size,
+            use_ray=not args.no_ray,
+            use_minio=not args.no_minio,
+        )
+    else:
+        # TODO: implement streaming mode
+        raise NotImplementedError(f'{mode} is not implemented yet')

pfeed/sources/__init__.py

@@ -0,0 +1 @@
+from pfeed.sources import bybit

pfeed/sources/bybit/__init__.py

@@ -0,0 +1,2 @@
+from pfeed.sources.bybit.historical import run as run_historical
+# from pfeed.sources.bybit.streaming import *

pfeed/sources/bybit/api.py

@@ -0,0 +1,63 @@
+import re
+import time
+import logging
+
+import requests
+
+from bs4 import BeautifulSoup
+from pfeed.sources.bybit.const import DATA_SOURCE, DATA_SOURCE_URLS, DATA_NAMING_REGEX_PATTERNS, create_efilename
+
+
+logger = logging.getLogger(DATA_SOURCE.lower())
+
+
+def get(url, handle_func, frequency=1, num_retry=3):
+    '''
+    Handles general requests.get with control on frequency and number of retry
+    Args:
+        handle_func: specific logic for handling response
+    '''
+    logger.debug(f'calling {url}')
+    while num_retry:
+        res = requests.get(url)
+        if res.status_code == 200:
+            return handle_func(res)
+        elif res.status_code == 404:
+            logger.error(f'File not found {url=} {res.status_code=} {res.text=}')
+            break
+        else:
+            logger.warning(f'{res.status_code=} {res.text=}')
+        time.sleep(frequency)
+    else:
+        logger.error(f'failed to call {url}')
+
+
+def get_efilenames(category: str, epdt: str):
+    '''
+    Get efilenames (e.g. BTCUSDT2022-10-04.csv.gz)
+    '''
+    def _handle_response(res):
+        soup = BeautifulSoup(res.text, 'html.parser')
+        efilenames = [node.get('href') for node in soup.find_all('a')]
+        return efilenames
+    url = '/'.join([DATA_SOURCE_URLS[category], epdt])
+    return get(url, _handle_response, frequency=1, num_retry=3)
+    
+
+def get_epdts(category: str, ptype: str):
+    def _handle_response(res):
+        soup = BeautifulSoup(res.text, 'html.parser')
+        epdts = [node.get('href').replace('/', '') for node in soup.find_all('a') if pattern.search(node.get('href'))]
+        return epdts
+    pattern = re.compile(DATA_NAMING_REGEX_PATTERNS[ptype])
+    url = DATA_SOURCE_URLS[category]
+    return get(url, _handle_response, frequency=1, num_retry=3)
+
+
+def get_data(category: str, epdt: str, date: str):
+    def _handle_response(res):
+        data = res.content
+        return data
+    efilename = create_efilename(epdt, date, is_spot=(category.upper()=='SPOT'))
+    url = f"{DATA_SOURCE_URLS[category]}/{epdt}/{efilename}"
+    return get(url, _handle_response, frequency=1, num_retry=3)

pfeed/sources/bybit/config.yml

@@ -0,0 +1,4 @@
+ptypes: []
+pdts: []
+# start_date: 
+# end_date: 

pfeed/sources/bybit/const.py

@@ -0,0 +1,42 @@
+DATA_SOURCE = 'BYBIT'
+# EXTEND: support FUT and IFUT
+# SUPPORTED_CRYPTO_PRODUCT_TYPES = ['SPOT', 'PERP', 'IPERP', 'FUT', 'IFUT']
+SUPPORTED_CRYPTO_PRODUCT_TYPES = ['SPOT', 'PERP', 'IPERP']
+RAW_DATA_TYPE = 'tick'
+# do not need to be precise
+DATA_START_DATE = '2020-01-01'
+DATA_SOURCE_URLS = {
+    'linear': 'https://public.bybit.com/trading',
+    'inverse': 'https://public.bybit.com/trading',
+    'spot': 'https://public.bybit.com/spot',
+}
+DATA_NAMING_REGEX_PATTERNS = {
+    'PERP': '(USDT\/|PERP\/)$',  # USDT perp or USDC perp;
+    'FUT': '-\d{2}[A-Z]{3}\d{2}\/$',  # USDC futures e.g. BTC-10NOV23/
+    'IPERP': 'USD\/$',  # inverse perps;
+    'IFUT': 'USD[A-Z]\d{2}\/$',  # inverse futures e.g. BTCUSDH24/
+    # match everything since everything from https://public.bybit.com/spot is spot
+    'SPOT': '.*',
+}
+SELECTED_RAW_COLS = {
+    'linear': ['timestamp', 'side', 'size', 'price'],
+    'inverse': ['timestamp', 'side', 'size', 'price'],
+    'spot': ['timestamp', 'side', 'volume', 'price'],
+}
+RENAMING_COLS = {
+    'linear': {'timestamp': 'ts', 'size': 'volume'},
+    'inverse': {'timestamp': 'ts', 'size': 'volume'},
+    'spot': {'timestamp': 'ts'},
+}
+RAW_DATA_TIMESTAMP_UNITS = {
+    'linear': 's',
+    'inverse': 's',
+    'spot': 'ms'
+}
+
+
+def create_efilename(epdt, date, is_spot=False):
+    if is_spot:
+        return f'{epdt}_{date}.csv.gz'
+    else:
+        return f'{epdt}{date}.csv.gz'