pfeed 0.0.1.dev14__tar.gz → 0.0.2.dev1__tar.gz

{pfeed-0.0.1.dev14 → pfeed-0.0.2.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pfeed
-Version: 0.0.1.dev14
+Version: 0.0.2.dev1
 Summary: Data pipeline for algo-trading, getting and storing both real-time and historical data made easy.
 Home-page: https://pfund.ai
 License: Apache-2.0
@@ -23,7 +23,7 @@ Requires-Dist: fastparquet (>=2024.5.0,<2025.0.0)
 Requires-Dist: minio (>=7.2.8,<8.0.0) ; extra == "data" or extra == "all"
 Requires-Dist: pandas (>=2.2.2,<3.0.0) ; extra == "df" or extra == "all"
 Requires-Dist: pfund (>=0.0.1.dev13,<0.0.2)
-Requires-Dist: polars (>=1.6.0,<2.0.0) ; extra == "df" or extra == "all"
+Requires-Dist: polars (>=1.7.1,<2.0.0) ; extra == "df" or extra == "all"
 Requires-Dist: psutil (>=6.0.0,<7.0.0) ; extra == "data" or extra == "all"
 Requires-Dist: pyarrow (>=15.0.0,<16.0.0) ; extra == "df" or extra == "all"
 Requires-Dist: ray (>=2.35.0,<3.0.0) ; extra == "boost" or extra == "all"

{pfeed-0.0.1.dev14 → pfeed-0.0.2.dev1}/pfeed/__init__.py

@@ -20,9 +20,9 @@ def download_historical_data(
     ptypes: str | list[str] | None = None,
     start_date: str | None = None,
     end_date: str | None = None,
-    num_cpus: int = 8,
-    use_ray: bool = True,
     use_minio: bool = False,
+    use_ray: bool = True,
+    ray_num_cpus: int = 8,
 ):
     data_source = importlib.import_module(f"pfeed.sources.{data_source.lower()}")
     return data_source.download_historical_data(
@@ -31,9 +31,9 @@ def download_historical_data(
         ptypes=ptypes,
         start_date=start_date,
         end_date=end_date,
-        num_cpus=num_cpus,
-        use_ray=use_ray,
         use_minio=use_minio,
+        use_ray=use_ray,
+        ray_num_cpus=ray_num_cpus,
     )
 
 

{pfeed-0.0.1.dev14 → pfeed-0.0.2.dev1}/pfeed/cli/commands/download.py

@@ -41,7 +41,7 @@ def download(data_source, pdts, dtypes, ptypes, start_date, end_date, num_cpus,
         ptypes=ptypes,
         start_date=start_date.date().strftime('%Y-%m-%d') if start_date else start_date,
         end_date=end_date.date().strftime('%Y-%m-%d') if end_date else end_date,
-        num_cpus=num_cpus,
         use_ray=not no_ray,
+        ray_num_cpus=num_cpus,
         use_minio=use_minio,
     )

{pfeed-0.0.1.dev14 → pfeed-0.0.2.dev1}/pfeed/config_handler.py

@@ -5,9 +5,6 @@ import logging
 from types import TracebackType
 from dataclasses import dataclass
 
-import yaml
-from dotenv import find_dotenv, load_dotenv
-
 from pfeed.const.paths import PROJ_NAME, MAIN_PATH, LOG_PATH, DATA_PATH, USER_CONFIG_FILE_PATH
 
 
@@ -48,6 +45,8 @@ class ConfigHandler:
     
     @classmethod
     def load_config(cls):
+        import yaml
+
         '''Loads user's config file and returns a ConfigHandler object'''
         config_file_path = USER_CONFIG_FILE_PATH
         if config_file_path.is_file():
@@ -77,9 +76,15 @@ class ConfigHandler:
         self.load_env_file(self.env_file_path)
         
         if self.debug:
-            self.enable_debug_mode()
+            is_loggers_set_up = bool(logging.getLogger('pfeed').handlers)
+            if is_loggers_set_up:
+                print('loggers are already set up, ignoring enabling debug mode')
+            else:
+                self.enable_debug_mode()
     
     def load_env_file(self, env_file_path: str | None):
+        from dotenv import find_dotenv, load_dotenv
+        
         if not env_file_path:
             found_env_file_path = find_dotenv(usecwd=True, raise_error_if_not_found=False)
             if found_env_file_path:

pfeed-0.0.2.dev1/pfeed/data_tools/data_tool_pandas.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from pfeed.resolution import ExtendedResolution
+    from pfeed.types.common_literals import tSUPPORTED_STORAGES
+
+import os
+import io
+
+import s3fs
+import pandas as pd
+
+from pfeed.const.common import SUPPORTED_STORAGES
+
+
+name = 'pandas'
+
+
+def read_parquet(paths_or_obj: list[str] | str | bytes, *args, storage: tSUPPORTED_STORAGES='local', **kwargs) -> pd.DataFrame:
+    assert storage in SUPPORTED_STORAGES, f'{storage=} not in {SUPPORTED_STORAGES}'
+    if isinstance(paths_or_obj, bytes):
+        obj = io.BytesIO(paths_or_obj)
+        return pd.read_parquet(obj, *args, **kwargs)
+    else:
+        if storage == 'minio':
+            if 'filesystem' not in kwargs:
+                fs = s3fs.S3FileSystem(
+                    endpoint_url="http://"+os.getenv('MINIO_HOST', 'localhost')+':'+os.getenv('MINIO_PORT', '9000'),
+                    key=os.getenv('MINIO_ROOT_USER', 'pfunder'),
+                    secret=os.getenv('MINIO_ROOT_PASSWORD', 'password'),
+                )
+                kwargs['filesystem'] = fs
+        paths = paths_or_obj if isinstance(paths_or_obj, list) else [paths_or_obj]
+        return pd.read_parquet(paths, *args, **kwargs)
+
+
+def estimate_memory_usage(df: pd.DataFrame) -> float:
+    """Estimate the memory usage of a pandas DataFrame in GB."""
+    return df.memory_usage(deep=True).sum() / (1024 ** 3)
+
+    
+def organize_time_series_columns(
+    pdt: str, 
+    resolution: str | ExtendedResolution, 
+    df: pd.DataFrame,
+    override_resolution: bool=False,
+) -> pd.DataFrame:
+    """Standardize the columns of a pandas DataFrame.
+    Moving 'ts', 'product', 'resolution' to the leftmost side.
+    """
+    from pfeed.resolution import ExtendedResolution
+    assert 'ts' in df.columns, "'ts' column not found"
+    if isinstance(resolution, str):
+        resolution = ExtendedResolution(resolution)
+    if 'product' not in df.columns:
+        df['product'] = pdt
+    if 'resolution' not in df.columns or override_resolution:
+        df['resolution'] = repr(resolution)
+    left_cols = ['ts', 'product', 'resolution']
+    df = df.reindex(left_cols + [col for col in df.columns if col not in left_cols], axis=1)
+    return df

{pfeed-0.0.1.dev14 → pfeed-0.0.2.dev1}/pfeed/data_tools/data_tool_polars.py

@@ -14,30 +14,26 @@ from pfeed.const.common import SUPPORTED_STORAGES
 name = 'polars'
 
 
-def read_parquet(path_or_obj: str | bytes, *args, storage: tSUPPORTED_STORAGES='local', **kwargs) -> pl.DataFrame | pl.LazyFrame:
+def read_parquet(paths_or_obj: list[str] | str | bytes, *args, storage: tSUPPORTED_STORAGES='local', **kwargs) -> pl.DataFrame | pl.LazyFrame:
     assert storage in SUPPORTED_STORAGES, f'{storage=} not in {SUPPORTED_STORAGES}'
-    if isinstance(path_or_obj, bytes):
-        obj = path_or_obj
+    if isinstance(paths_or_obj, bytes):
+        obj = paths_or_obj
         return pl.read_parquet(obj, *args, **kwargs)
     else:
-        path = path_or_obj
+        paths = paths_or_obj if isinstance(paths_or_obj, list) else [paths_or_obj]
         if storage == 'local':
-            return pl.scan_parquet(path, *args, **kwargs)
+            return pl.scan_parquet(paths, *args, **kwargs)
         elif storage == 'minio':
             storage_options = {
                 "endpoint_url": "http://"+os.getenv('MINIO_HOST', 'localhost')+':'+os.getenv('MINIO_PORT', '9000'),
                 "access_key_id": os.getenv('MINIO_ROOT_USER', 'pfunder'),
                 "secret_access_key": os.getenv('MINIO_ROOT_PASSWORD', 'password'),
             }
-            return pl.scan_parquet(path, *args, storage_options=storage_options, **kwargs)
+            return pl.scan_parquet(paths, *args, storage_options=storage_options, **kwargs)
         else:
             raise NotImplementedError(f'{storage=}')
 
 
-def concat(dfs: list[pl.DataFrame | pl.LazyFrame], *args, **kwargs) -> pl.DataFrame | pl.LazyFrame:
-    return pl.concat(dfs, *args, **kwargs)
-
-        
 def estimate_memory_usage(df: pl.DataFrame | pl.LazyFrame) -> float:
     """Estimate the memory usage of a polars DataFrame in GB."""
     if isinstance(df, pl.LazyFrame):
@@ -45,21 +41,28 @@ def estimate_memory_usage(df: pl.DataFrame | pl.LazyFrame) -> float:
     return df.estimated_size(unit='gb')
 
 
-def organize_time_series_columns(pdt: str, resolution: str | ExtendedResolution, df: pl.DataFrame | pl.LazyFrame) -> pl.DataFrame | pl.LazyFrame:
+def organize_time_series_columns(
+    pdt: str, 
+    resolution: str | ExtendedResolution, 
+    df: pl.DataFrame | pl.LazyFramem,
+    override_resolution: bool=False,
+) -> pl.DataFrame | pl.LazyFrame:
     from pfeed.resolution import ExtendedResolution
     if isinstance(df, pl.LazyFrame):
         cols = df.collect_schema().names()
     else:
         cols = df.columns
     assert 'ts' in cols, "'ts' column not found"
-    assert 'product' not in cols, "'product' column already exists"
-    assert 'resolution' not in cols, "'resolution' column already exists"
     if isinstance(resolution, str):
         resolution = ExtendedResolution(resolution)
-    df = df.with_columns(
-        pl.lit(pdt).alias('product'),
-        pl.lit(repr(resolution)).alias('resolution')
-    )
+    if 'product' not in cols:
+        df = df.with_columns(
+            pl.lit(pdt).alias('product'),
+        )
+    if 'resolution' not in cols or override_resolution:
+        df = df.with_columns(
+            pl.lit(repr(resolution)).alias('resolution')
+        )
     left_cols = ['ts', 'product', 'resolution']
-    df = df.select(left_cols + [col for col in df.collect_schema().names() if col not in left_cols])
+    df = df.select(left_cols + [col for col in cols if col not in left_cols])
     return df

{pfeed-0.0.1.dev14 → pfeed-0.0.2.dev1}/pfeed/datastore.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     try:
-        from minio.api import ObjectWriteResult
+        from minio.api import ObjectWriteResult, Tags
     except ImportError:
         pass
     from typing import Generator
@@ -12,10 +12,9 @@ import io
 import logging
 
 
-def assert_if_minio_running():
+def check_if_minio_running():
     import requests
     from requests.exceptions import RequestException, ReadTimeout
-    from minio.error import MinioException
 
     endpoint = os.getenv('MINIO_HOST', 'localhost')+':'+os.getenv('MINIO_PORT', '9000')
     if not endpoint.startswith('http'):
@@ -26,13 +25,15 @@ def assert_if_minio_running():
     try:
         response = requests.get(f'{endpoint}/minio/health/live', timeout=3)
         if response.status_code != 200:
-            raise MinioException(f"Unhandled response: {response.status_code=} {response.content} {response}")
+            print(f"Unhandled response from MinIO: {response.status_code=} {response.content} {response}")
+            return False
     except (ReadTimeout, RequestException) as e:
-        raise MinioException(f"MinIO is not running or not detected on {endpoint}: {e}, please use 'pfeed docker-compose up -d' to start MinIO")
+        return False
+    return True
 
 
 class Datastore:
-    DATA_PART_SIZE = 5 * (1024 ** 2)  # part size for S3, 5 MB
+    # DATA_PART_SIZE = 5 * (1024 ** 2)  # part size for S3, 5 MB
     BUCKET_NAME = 'pfeed'
 
     # EXTEND, currently only consider using MinIO
@@ -40,9 +41,9 @@ class Datastore:
     def initialize_store(cls, name: str, **kwargs):
         if name == 'minio':
             from minio import Minio
-            assert_if_minio_running()
+            endpoint = os.getenv('MINIO_HOST', 'localhost')+':'+os.getenv('MINIO_PORT', '9000')
             cls.minio = Minio(
-                endpoint=os.getenv('MINIO_HOST', 'localhost')+':'+os.getenv('MINIO_PORT', '9000'),
+                endpoint=endpoint,
                 access_key=os.getenv('MINIO_ROOT_USER', 'pfunder'),
                 secret_key=os.getenv('MINIO_ROOT_PASSWORD', 'password'),
                 # turn off TLS, i.e. not using HTTPS
@@ -74,7 +75,16 @@ class Datastore:
                 self.logger.error(f'Unhandled MinIO response status {res.status}')
         except S3Error as err:
             # logger.warning(f'MinIO S3Error {object_name=} {err=}')
-            pass
+            return None
+
+    def exist_object(self, object_name: str) -> bool:
+        from minio import S3Error
+        try:
+            res: Tags | None = self.minio.get_object_tags(self.BUCKET_NAME, object_name)
+            return True
+        except S3Error as err:
+            # self.logger.warning(f'MinIO S3Error {object_name=} {err=}')
+            return False
 
     def list_objects(self, prefix) -> list | None:
         '''
@@ -89,8 +99,9 @@ class Datastore:
             self.BUCKET_NAME,
             object_name,
             data=io.BytesIO(data),
-            part_size=self.DATA_PART_SIZE,
-            length=-1,
+            # part_size=self.DATA_PART_SIZE,
+            length=len(data),
+            content_type='application/parquet',
             **kwargs
         )
         

{pfeed-0.0.1.dev14 → pfeed-0.0.2.dev1}/pfeed/etl.py

@@ -2,7 +2,7 @@
 Except extracting and loading data, this module uses "pandas" for data transformation.
 '''
 from __future__ import annotations
-from typing import TYPE_CHECKING, Literal
+from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from pfeed.types.common_literals import (
         tSUPPORTED_ENVIRONMENTS,
@@ -10,8 +10,6 @@ if TYPE_CHECKING:
         tSUPPORTED_STORAGES, 
         tSUPPORTED_DATA_TOOLS,
     )
-    from pfeed.resolution import ExtendedResolution
-    tOUTPUT_FORMATS = Literal['bytes'] | tSUPPORTED_DATA_TOOLS
 
 import logging
 import importlib
@@ -22,7 +20,8 @@ try:
 except ImportError:
     pass
 
-from pfeed.datastore import Datastore
+from pfeed.resolution import ExtendedResolution
+from pfeed.datastore import Datastore, check_if_minio_running
 from pfeed.filepath import FilePath
 from pfeed.config_handler import get_config
 from pfeed.const.common import (
@@ -31,17 +30,14 @@ from pfeed.const.common import (
     SUPPORTED_DOWNLOAD_DATA_SOURCES, 
     SUPPORTED_DATA_TOOLS,
 )
-from pfeed.types.common_literals import tSUPPORTED_DATA_TOOLS
 from pfeed.utils.utils import derive_trading_venue
-from pfeed.utils.file_format import read_raw_data
-
+from pfeed.utils.file_formats import read_raw_data
 try:
     from pfeed.utils.monitor import print_disk_usage
 except ImportError:
     print_disk_usage = None
 
 
-OUTPUT_FORMATS = ['bytes'] + SUPPORTED_DATA_TOOLS
 DataFrame = pd.DataFrame | pl.DataFrame | pl.LazyFrame
 
 
@@ -51,7 +47,6 @@ __all__ = [
     'transform_data',
     'load_data',
     'clean_raw_data',
-    'standardize_raw_data',
     'resample_data',
 ]
 
@@ -59,39 +54,36 @@ __all__ = [
 def get_data(
     env: tSUPPORTED_ENVIRONMENTS,
     data_source: tSUPPORTED_DOWNLOAD_DATA_SOURCES,
-    resolution: str | ExtendedResolution,
     pdt: str,
-    date: str,
+    resolution: str | ExtendedResolution,
+    dates: list[str],
+    storages: list[tSUPPORTED_STORAGES] | None = None,
     trading_venue: str='',
-    output_format: tOUTPUT_FORMATS='pandas',
-) -> bytes | DataFrame | None:
+    output_format: tSUPPORTED_DATA_TOOLS='pandas',
+) -> DataFrame | None:
     """Extract data without specifying the data origin. 
     This function will try to extract data from all supported data origins.
 
     Args:
         env: trading environment, e.g. 'PAPER' | 'LIVE'.
         data_source (Literal['BYBIT']): The data source to extract data from.
+        pdt (str): product, e.g. BTC_USDT_PERP.
         resolution: Data resolution. e.g. '1m' = 1 minute as the unit of each data bar/candle.
             Also supports raw resolution such as 'r1m', where 'r' stands for raw.            
             Default is '1d' = 1 day.
-        pdt (str): product, e.g. BTC_USDT_PERP.
-        date (str): The date of the data to extract.
+        dates (list[str]): The dates of the data to extract.
+        storages: origins of data to search from, default is all supported storages
         trading_venue (str): trading venue's name, e.g. exchange's name or dapp's name
         output_format: The format of the output data. Default is 'pandas'.
-    Returns:
-        bytes | DataFrame | None: The extracted data as bytes, or None if the data is not found.
     """    
-    try:
-        from minio.error import MinioException
-    except ImportError:
-        MinioException = Exception
-    
-    trading_venue = trading_venue or derive_trading_venue(data_source)
-    for storage in SUPPORTED_STORAGES:
-        try:
-            data: bytes | pd.DataFrame | None = extract_data(env, storage, data_source, trading_venue, resolution, pdt, date, output_format=output_format)
-        except MinioException:
-            data = None
+    logger = logging.getLogger(data_source.lower() + '_data')
+    storages = storages or SUPPORTED_STORAGES
+    for storage in storages:
+        if storage == 'minio':
+            if not check_if_minio_running():
+                continue
+        logger.debug(f'searching {storage=} for {data_source} {pdt} {resolution} data from {dates[0]} to {dates[-1]}')
+        data: DataFrame | None = extract_data(env, storage, data_source, pdt, resolution, dates, trading_venue=trading_venue, output_format=output_format)
         if data is not None:
             return data
 
@@ -100,84 +92,72 @@ def extract_data(
     env: tSUPPORTED_ENVIRONMENTS,
     storage: tSUPPORTED_STORAGES,
     data_source: tSUPPORTED_DOWNLOAD_DATA_SOURCES,
-    trading_venue: str,
-    resolution: str | ExtendedResolution,
     pdt: str,
-    date: str,
-    output_format: tOUTPUT_FORMATS='pandas',
-) -> bytes | DataFrame | None:
+    resolution: str | ExtendedResolution,
+    dates: list[str],
+    trading_venue: str='',
+    output_format: tSUPPORTED_DATA_TOOLS='pandas',
+) -> DataFrame | None:
     """
     Extracts data from a specified data source and returns it as bytes.
 
     Args:
         env: trading environment, e.g. 'PAPER' | 'LIVE'.
-        storage: The origin of the data (local or minio).
+        storage: The origin of the data (e.g. local or minio).
         data_source: The source of the data.
-        trading_venue: trading venue's name, e.g. exchange's name or dapp's name
+        pdt (str): product, e.g. BTC_USDT_PERP.
         resolution: Data resolution. e.g. '1m' = 1 minute as the unit of each data bar/candle.
             Also supports raw resolution such as 'r1m', where 'r' stands for raw.            
             Default is '1d' = 1 day.
-        pdt (str): product, e.g. BTC_USDT_PERP.
-        date (str): The date of the data.
+        dates (list[str]): The dates of the data.
+        trading_venue: trading venue's name, e.g. exchange's name or dapp's name
         output_format: The format of the output data. Default is 'pandas'.
-    Returns:
-        bytes | DataFrame | None: The extracted data as bytes, or None if extraction fails.
-
-    Raises:
-        AssertionError: If any of the input parameters are invalid.
-        NotImplementedError: If the data origin is not supported.
-        MinioException: If MinIO is not running / set up correctly.
     """
-    from pfeed.resolution import ExtendedResolution
-    
     logger = logging.getLogger(data_source.lower() + '_data')
-    
     env, storage, data_source, pdt, output_format = env.upper(), storage.lower(), data_source.upper(), pdt.upper(), output_format.lower()
+    trading_venue = trading_venue or derive_trading_venue(data_source)
+    trading_venue = trading_venue.upper()
     assert env in SUPPORTED_ENVIRONMENTS, f'Invalid {env=}, {SUPPORTED_ENVIRONMENTS=}'
     assert storage in SUPPORTED_STORAGES, f'Invalid {storage=}, {SUPPORTED_STORAGES=}'
     assert data_source in SUPPORTED_DOWNLOAD_DATA_SOURCES, f'Invalid {data_source=}, SUPPORTED DATA SOURCES={SUPPORTED_DOWNLOAD_DATA_SOURCES}'
-    assert output_format in OUTPUT_FORMATS, f'Invalid {output_format=}, {OUTPUT_FORMATS=}'
+    assert output_format in SUPPORTED_DATA_TOOLS, f'Invalid {output_format=}, valid options: {SUPPORTED_DATA_TOOLS}'
     if isinstance(resolution, str):
         resolution = ExtendedResolution(resolution)
     if output_format != 'bytes':
         data_tool = importlib.import_module(f'pfeed.data_tools.data_tool_{output_format.lower()}')
     config = get_config()
-    fp = FilePath(env, data_source, trading_venue, pdt, resolution, date, file_extension='.parquet', data_path=config.data_path)
-    if storage == 'local':
-        if fp.exists():
-            if output_format == 'bytes':
-                with open(fp.file_path, 'rb') as f:
-                    data: bytes = f.read()
-            else:
-                data: DataFrame = data_tool.read_parquet(fp.file_path)
-            logger.debug(f'extracted {data_source} {pdt} {date} {resolution} data from local path {fp.file_path}')
-            return data
+    filepaths = [FilePath(env, data_source, trading_venue, pdt, resolution, date, file_extension='.parquet', data_path=config.data_path) for date in dates]
+    try:
+        df = None
+        if storage == 'local':
+            if all(fp.exists() for fp in filepaths):
+                df: DataFrame = data_tool.read_parquet([fp.file_path for fp in filepaths])
+        elif storage == 'minio':
+            datastore = Datastore(storage)
+            object_names = [fp.storage_path for fp in filepaths]
+            if all(datastore.exist_object(object_name) for object_name in object_names):
+                paths = ["s3://" + datastore.BUCKET_NAME + "/" + object_name for object_name in object_names]
+                df: DataFrame = data_tool.read_parquet(paths, storage='minio')
         else:
-            logger.debug(f'failed to extract {data_source} {pdt} {date} {resolution} data from local path {fp.file_path}')
-    elif storage == 'minio':
-        datastore = Datastore(storage)
-        object_name = fp.storage_path
-        data: bytes | None = datastore.get_object(object_name)
-        if data:
-            if output_format != 'bytes':
-                file_path = "s3://" + datastore.BUCKET_NAME + "/" + object_name
-                data: DataFrame = data_tool.read_parquet(file_path, storage='minio')
-            logger.debug(f'extracted {data_source} {pdt} {date} {resolution} data from MinIO object {object_name}')
+            raise NotImplementedError(f'{storage=}')
+        
+        if df is not None:
+            logger.debug(f'extracted {data_source} {pdt} {resolution} data from {dates[0]} to {dates[-1]} from {storage}')
         else:
-            logger.debug(f'failed to extract {data_source} {pdt} {date} {resolution} data from MinIO object {object_name}')
-        return data
-    else:
-        raise NotImplementedError(f'{storage=}')
+            logger.debug(f'failed to extract {data_source} {pdt} {resolution} data from {dates[0]} to {dates[-1]} from {storage}')
+        return df
+    except Exception as err:
+        logger.exception(f'failed to extract {data_source} {pdt} {resolution} data from {dates[0]} to {dates[-1]} from {storage}, {err=}')
     
     
 def transform_data(
     data_source: tSUPPORTED_DOWNLOAD_DATA_SOURCES,
+    pdt: str,
     data: bytes | pd.DataFrame | pl.LazyFrame,
     data_resolution: str | ExtendedResolution,
     target_resolution: str | ExtendedResolution,
 ) -> bytes | pd.DataFrame | pl.LazyFrame:
     """Transforms data to a target resolution"""
-    from pfeed.resolution import ExtendedResolution
     if isinstance(data_resolution, str):
         data_resolution = ExtendedResolution(data_resolution)
     if isinstance(target_resolution, str):
@@ -192,22 +172,23 @@ def transform_data(
     elif data_resolution.is_raw() and target_resolution.is_raw():  # e.g. 'r1t' -> 'r1m
         raise Exception(f'{data_resolution=} and {target_resolution=} are both raw resolutions')
     else:
-        data: bytes | pd.DataFrame | pl.LazyFrame = standardize_raw_data(data, data_resolution.is_tick())
-        if target_resolution.is_tick():
-            return data
-        else:
-            return resample_data(data, target_resolution)
+        df: pd.DataFrame = _convert_data_to_pandas_df(data)
+        df = _standardize_columns(df, data_resolution.is_tick())
+        if not target_resolution.is_tick():
+            df = resample_data(df, target_resolution)
+        df = _organize_columns(df, pdt, target_resolution)
+        return _handle_result(data, df)
 
 
 def load_data(
     env: tSUPPORTED_ENVIRONMENTS,
     storage: tSUPPORTED_STORAGES,
     data_source: tSUPPORTED_DOWNLOAD_DATA_SOURCES,
-    trading_venue: str,
     data: bytes,
-    resolution: str | ExtendedResolution,
     pdt: str,
+    resolution: str | ExtendedResolution,
     date: str,
+    trading_venue: str='',
     **kwargs
 ) -> None:
     """
@@ -218,28 +199,23 @@ def load_data(
         storage: The destination where the data will be loaded. 
             It can be either 'local' or 'minio'.
         data_source: The source of the data.
-        trading_venue: trading venue's name, e.g. exchange's name or dapp's name
         data (bytes): The data to be loaded.
+        pdt (str): product, e.g. BTC_USDT_PERP.
         resolution: Data resolution. e.g. '1m' = 1 minute as the unit of each data bar/candle.
             Also supports raw resolution such as 'r1m', where 'r' stands for raw.            
             Default is '1d' = 1 day.
-        pdt (str): product, e.g. BTC_USDT_PERP.
         date (str): The date of the data.
+        trading_venue: trading venue's name, e.g. exchange's name or dapp's name
         **kwargs: Additional keyword arguments for MinIO.
 
     Returns:
         None
-        
-    Raises:
-        AssertionError: If any of the input parameters are invalid.
-        NotImplementedError: If the specified data destination is not implemented.
-        MinioException: If MinIO is not running / set up correctly.
     """
-    from pfeed.resolution import ExtendedResolution
-    
     logger = logging.getLogger(data_source.lower() + '_data')
     
     env, storage, data_source, pdt = env.upper(), storage.lower(), data_source.upper(), pdt.upper()
+    trading_venue = trading_venue or derive_trading_venue(data_source)
+    trading_venue = trading_venue.upper()
     assert env in SUPPORTED_ENVIRONMENTS, f'Invalid {env=}, {SUPPORTED_ENVIRONMENTS=}'
     assert storage in SUPPORTED_STORAGES, f'Invalid {storage=}, {SUPPORTED_STORAGES=}'
     assert data_source in SUPPORTED_DOWNLOAD_DATA_SOURCES, f'Invalid {data_source=}, SUPPORTED DATA SOURCES={SUPPORTED_DOWNLOAD_DATA_SOURCES}'
@@ -280,7 +256,6 @@ def clean_raw_data(
         bytes: The cleaned raw data.
     '''
     assert data_source in SUPPORTED_DOWNLOAD_DATA_SOURCES, f'Invalid {data_source=}, SUPPORTED DATA SOURCES={SUPPORTED_DOWNLOAD_DATA_SOURCES}'
-    
     const = importlib.import_module(f'pfeed.sources.{data_source.lower()}.const')
     utils = importlib.import_module(f'pfeed.sources.{data_source.lower()}.utils')
     
@@ -293,27 +268,6 @@ def clean_raw_data(
     return _handle_result(data, df)
 
 
-def standardize_raw_data(
-    data: bytes | pd.DataFrame | pl.LazyFrame, 
-    is_tick: bool
-) -> bytes | pd.DataFrame | pl.LazyFrame:
-    """Filter out unnecessary columns from raw data.
-
-    Args:
-        data (bytes): The raw data in bytes format.
-
-    Returns:
-        bytes | pd.DataFrame | pl.LazyFrame: The standardized data.
-    """
-    df: pd.DataFrame = _convert_data_to_pandas_df(data)
-    assert 'ts' in df.columns, 'ts column not found, please check if the raw data has been cleaned'
-    if is_tick:
-        df = df.loc[:, ['ts', 'side', 'volume', 'price']]
-    else:
-        df = df.loc[:, ['ts', 'open', 'high', 'low', 'close', 'volume']]
-    return _handle_result(data, df)
-
-
 def resample_data(
     data: bytes | pd.DataFrame | pl.LazyFrame, 
     resolution: str | ExtendedResolution, 
@@ -326,8 +280,6 @@ def resample_data(
         resolution (str | Resolution): The resolution at which the data should be resampled. 
             if string, it should be in the format of "# + unit (s/m/h/d)", e.g. "1s".
     '''
-    from pfeed.resolution import ExtendedResolution
-
     # standardize resolution by following pfund's standard, e.g. '1minute' -> '1m'
     if isinstance(resolution, str):
         resolution = ExtendedResolution(resolution)
@@ -380,6 +332,26 @@ def resample_data(
     return _handle_result(data, resampled_df)
 
 
+def _standardize_columns(df: pd.DataFrame, is_tick: bool) -> pd.DataFrame:
+    """Filter out unnecessary columns from raw data."""
+    assert 'ts' in df.columns, '"ts" column not found'
+    if is_tick:
+        df = df.loc[:, ['ts', 'side', 'volume', 'price']]
+    else:
+        df = df.loc[:, ['ts', 'open', 'high', 'low', 'close', 'volume']]
+    return df
+
+
+def _organize_columns(df: pd.DataFrame, pdt: str, resolution: ExtendedResolution) -> pd.DataFrame:
+    """Organizes the columns of a DataFrame.
+    Moving 'ts', 'product', 'resolution' to the leftmost side.
+    """
+    df['product'] = pdt
+    df['resolution'] = repr(resolution)
+    left_cols = ['ts', 'product', 'resolution']
+    return df.reindex(left_cols + [col for col in df.columns if col not in left_cols], axis=1)
+
+
 def _convert_data_to_pandas_df(data: bytes | pd.DataFrame | pl.LazyFrame) -> pd.DataFrame:
     """Converts data to pandas DataFrame."""
     if isinstance(data, bytes):