pfeed 0.0.1__py3-none-any.whl

pfeed/__init__.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pfeed.types.common_literals import tSUPPORTED_DOWNLOAD_DATA_SOURCES, tSUPPORTED_DATA_TYPES
+
+import importlib
+from importlib.metadata import version
+
+from pfeed.config_handler import configure, get_config
+from pfeed.const.common import ALIASES
+from pfeed.sources import bybit
+from pfeed.feeds import BybitFeed, YahooFinanceFeed
+
+
+def download_historical_data(
+    data_source: tSUPPORTED_DOWNLOAD_DATA_SOURCES,
+    pdts: str | list[str] | None = None,
+    dtypes: tSUPPORTED_DATA_TYPES | list[tSUPPORTED_DATA_TYPES] | None = None,
+    ptypes: str | list[str] | None = None,
+    start_date: str | None = None,
+    end_date: str | None = None,
+    num_cpus: int = 8,
+    use_ray: bool = True,
+    use_minio: bool = False,
+):
+    data_source = importlib.import_module(f"pfeed.sources.{data_source.lower()}")
+    return data_source.download_historical_data(
+        pdts=pdts,
+        dtypes=dtypes,
+        ptypes=ptypes,
+        start_date=start_date,
+        end_date=end_date,
+        num_cpus=num_cpus,
+        use_ray=use_ray,
+        use_minio=use_minio,
+    )
+
+
+# TODO
+def stream_realtime_data(data_source: tSUPPORTED_DOWNLOAD_DATA_SOURCES):
+    data_source = importlib.import_module(f"pfeed.sources.{data_source.lower()}")
+    return data_source.stream_realtime_data()
+    
+
+
+download = download_historical_data
+stream = stream_realtime_data
+
+
+__version__ = version("pfeed")
+__all__ = (
+    "__version__",
+    "configure",
+    "get_config",
+    "ALIASES",
+    "bybit",
+    "binance",
+    "YahooFinanceFeed",
+    "BybitFeed",
+    "BinanceFeed",
+)

pfeed/cli/__init__.py

@@ -0,0 +1,4 @@
+from pfeed.cli.main import pfeed_group
+
+
+__all__ = ["pfeed_group"]

pfeed/cli/commands/config.py

@@ -0,0 +1,69 @@
+import os
+import yaml
+from pathlib import Path
+from pprint import pformat
+
+import click
+
+from pfeed.const.paths import USER_CONFIG_FILE_PATH
+from pfeed.config_handler import ConfigHandler
+
+
+def save_config(config: ConfigHandler, config_file_path: str | Path):
+    if isinstance(config_file_path, str):
+        config_file_path = Path(config_file_path)
+    config_file_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(config_file_path, 'w') as f:
+        yaml.dump(config.__dict__, f, default_flow_style=False)
+        
+
+def remove_config(config_file_path: str | Path):
+    config_file_path = Path(config_file_path)
+    if config_file_path.is_file():
+        os.remove(config_file_path)
+
+
+@click.command()
+@click.pass_context
+@click.option('--data-path', type=click.Path(resolve_path=True), help='Set the data path')
+@click.option('--log-path', type=click.Path(resolve_path=True), help='Set the log path')
+@click.option('--logging-file', 'logging_config_file_path', type=click.Path(resolve_path=True, exists=True), help='Set the logging config file path')
+@click.option('--logging-config', type=dict, help='Set the logging config')
+@click.option('--use-fork-process', type=bool, help='If True, multiprocessing.set_start_method("fork")')
+@click.option('--use-custom-excepthook', type=bool, help='If True, log uncaught exceptions to file')
+@click.option('--env-file', 'env_file_path', type=click.Path(resolve_path=True, exists=True), help='Path to the .env file')
+@click.option('--debug', is_flag=True, help='if enabled, debug mode will be enabled where logs at DEBUG level will be printed')
+@click.option('--list', '-l', is_flag=True, is_eager=True, help='List all available options')
+@click.option('--reset', is_flag=True, is_eager=True, help='Reset the configuration to defaults') 
+def config(ctx, **kwargs):
+    """Configures pfeed settings."""
+    config: ConfigHandler = ctx.obj['config']
+    
+    # Filter out options that were not provided by the user
+    provided_options = {k: v for k, v in kwargs.items() if v is not None and v is not False}
+    
+    if kwargs.get('list'):  # Check if --list was used
+        del provided_options['list']
+        assert not provided_options, "No options should be provided with --list"
+        config_dict = config.__dict__
+        config_dict.update({'config_file_path': USER_CONFIG_FILE_PATH})
+        click.echo(f"PFeed's config:\n{pformat(config_dict)}")
+        return
+
+    if kwargs.get('reset'): # Check if --reset was used
+        del provided_options['reset']
+        assert not provided_options, "No options should be provided with --reset"
+        remove_config(USER_CONFIG_FILE_PATH)
+        click.echo("PFeed's config successfully reset.")
+        return
+    
+    # prints out current config if no options are provided
+    if not provided_options:
+        raise click.UsageError("No options provided. Use --list to see all available options.")
+    else:
+        for option, value in provided_options.items():
+            setattr(config, option, value)
+            click.echo(f"{option} set to: {value}")
+        
+        save_config(config, USER_CONFIG_FILE_PATH)
+        click.echo(f"config saved to {USER_CONFIG_FILE_PATH}.")

pfeed/cli/commands/docker_compose.py

@@ -0,0 +1,33 @@
+import os
+from pathlib import Path
+import importlib.resources
+import subprocess
+
+import click
+
+from pfeed.const.paths import PROJ_NAME
+
+
+@click.command(context_settings=dict(
+    ignore_unknown_options=True,
+    allow_extra_args=True,
+))
+@click.pass_context
+@click.option('--env-file', 'env_file_path', type=click.Path(exists=True), help='Path to the .env file')
+@click.option('--docker-file', 'docker_file_path', type=click.Path(exists=True), help='Path to the docker-compose.yml file')
+def docker_compose(ctx, env_file_path, docker_file_path):
+    """Forwards commands to docker-compose with the package's docker-compose.yml file if not specified."""
+    config = ctx.obj['config']
+    config.load_env_file(env_file_path)
+    os.environ['PFEED_DATA_PATH'] = config.data_path
+    
+    if not docker_file_path:
+        package_dir = Path(importlib.resources.files(PROJ_NAME)).resolve().parents[0]
+        docker_file_path = package_dir / 'docker-compose.yml'
+    else:
+        click.echo(f'loaded custom docker-compose.yml file from "{docker_file_path}"')
+    command = ['docker-compose', '-f', str(docker_file_path)] + ctx.args
+    subprocess.run(command)
+
+
+

pfeed/cli/commands/download.py

@@ -0,0 +1,47 @@
+import importlib
+
+import click
+
+import pfeed as pe
+from pfeed.const.common import (
+    ALIASES, 
+    SUPPORTED_DOWNLOAD_DATA_SOURCES, 
+    SUPPORTED_DATA_TYPES,
+    SUPPORTED_PRODUCT_TYPES,
+)
+
+
+# add aliases to supported download data sources
+SUPPORTED_DOWNLOAD_DATA_SOURCES_ALIASES_INCLUDED = SUPPORTED_DOWNLOAD_DATA_SOURCES + [k for k, v in ALIASES.items() if v in SUPPORTED_DOWNLOAD_DATA_SOURCES]
+
+# 'raw' data type is implicit since it doesn't have the timeframe specified, but still allow it for convenience
+# since for data source like bybit, there's only one raw data type, 'raw_tick', i.e. 'raw' will be converted to 'raw_tick'
+SUPPORTED_DATA_TYPES_IMPLICIT_RAW_ALLOWED = SUPPORTED_DATA_TYPES + ['raw']
+
+
+@click.command()
+@click.option('--data-source', '-d', required=True, type=click.Choice(SUPPORTED_DOWNLOAD_DATA_SOURCES_ALIASES_INCLUDED, case_sensitive=False), help='Data source')
+@click.option('--pdts', '-p', 'pdts', multiple=True, default=[], help='List of trading products')
+@click.option('--dtypes', '--dt', 'dtypes', multiple=True, default=['raw'], type=click.Choice(SUPPORTED_DATA_TYPES_IMPLICIT_RAW_ALLOWED, case_sensitive=False), help=f'{SUPPORTED_DATA_TYPES=}. How to pass in multiple values: --dt raw --dt tick')
+@click.option('--ptypes', '--pt', 'ptypes', multiple=True, default=[], type=click.Choice(SUPPORTED_PRODUCT_TYPES, case_sensitive=False), help='List of product types, e.g. PERP = get all perpetuals')
+@click.option('--start-date', '-s', type=click.DateTime(formats=["%Y-%m-%d"]), help='Start date in YYYY-MM-DD format')
+@click.option('--end-date', '-e', type=click.DateTime(formats=["%Y-%m-%d"]), help='End date in YYYY-MM-DD format')
+@click.option('--num-cpus', '-n', default=8, type=int, help="number of logical CPUs used for Ray's tasks")
+@click.option('--use-minio', '-m', is_flag=True, help='if enabled, data will be loaded into Minio')
+@click.option('--no-ray', is_flag=True, help='if enabled, Ray will not be used')
+@click.option('--env-file', 'env_file_path', type=click.Path(exists=True), help='Path to the .env file')
+@click.option('--debug', is_flag=True, help='if enabled, debug mode will be enabled where logs at DEBUG level will be printed')
+def download(data_source, pdts, dtypes, ptypes, start_date, end_date, num_cpus, no_ray, use_minio, env_file_path, debug):
+    pe.configure(env_file_path=env_file_path, debug=debug)
+    data_source = ALIASES.get(data_source, data_source)
+    pipeline = importlib.import_module(f'pfeed.sources.{data_source.lower()}.download')
+    pipeline.download_historical_data(
+        pdts=pdts,
+        dtypes=dtypes,
+        ptypes=ptypes,
+        start_date=start_date.date().strftime('%Y-%m-%d') if start_date else start_date,
+        end_date=end_date.date().strftime('%Y-%m-%d') if end_date else end_date,
+        num_cpus=num_cpus,
+        use_ray=not no_ray,
+        use_minio=use_minio,
+    )

pfeed/cli/commands/open.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+import importlib.resources
+import subprocess
+
+import click
+
+from pfeed.const.paths import PROJ_NAME, USER_CONFIG_FILE_PATH
+
+
+def open_with_vscode(file_path):
+    try:
+        subprocess.run(["code", str(file_path)], check=True)
+        click.echo(f"Opened {file_path} with VS Code")
+    except subprocess.CalledProcessError:
+        click.echo("Failed to open with VS Code. Falling back to default editor.")
+        click.edit(filename=file_path)
+    except FileNotFoundError:
+        click.echo("VS Code command 'code' not found. Falling back to default editor.")
+        click.edit(filename=file_path)
+
+
+@click.command()
+@click.option('--config-file', '-c', is_flag=True, help='Open the config file')
+@click.option('--log-file', '-l', is_flag=True, help='Open the logging.yaml file for logging config')
+@click.option('--docker-file', '-d', is_flag=True, help='Open the docker-compose.yml file')
+@click.option('--default-editor', '-e', is_flag=True, help='Use default editor')
+def open(config_file, log_file, docker_file, default_editor):
+    """Opens the log file or docker-compose.yml file."""
+    if all([config_file, log_file, docker_file]):
+        click.echo('Please specify only one file to open')
+        return
+    
+    package_dir = Path(importlib.resources.files(PROJ_NAME)).resolve().parents[0]
+    if config_file:
+        file_path = USER_CONFIG_FILE_PATH
+    elif log_file:
+        file_path = package_dir / 'logging.yml'
+    elif docker_file:
+        file_path = package_dir / 'docker-compose.yml'
+    else:
+        click.echo('Please specify a file to open')
+        return
+    
+    if default_editor:
+        click.edit(filename=file_path)
+    else:
+        open_with_vscode(file_path)

pfeed/cli/main.py

@@ -0,0 +1,24 @@
+import click
+
+from pfeed.config_handler import get_config
+from pfeed.cli.commands.docker_compose import docker_compose
+from pfeed.cli.commands.config import config
+from pfeed.cli.commands.download import download
+# from pfeed.cli.commands.stream import stream
+from pfeed.cli.commands.open import open
+
+
+@click.group(context_settings={"help_option_names": ["-h", "--help"]})
+@click.pass_context
+@click.version_option()
+def pfeed_group(ctx):
+    """PFeed's CLI"""
+    ctx.ensure_object(dict)
+    ctx.obj['config'] = get_config()
+
+
+pfeed_group.add_command(docker_compose)
+pfeed_group.add_command(config)
+pfeed_group.add_command(download)
+# pfeed_group.add_command(stream)
+pfeed_group.add_command(open)

pfeed/config_handler.py

@@ -0,0 +1,148 @@
+import os
+import sys
+import multiprocessing
+import logging
+from types import TracebackType
+from dataclasses import dataclass
+
+import yaml
+from dotenv import find_dotenv, load_dotenv
+
+from pfeed.const.paths import PROJ_NAME, MAIN_PATH, LOG_PATH, DATA_PATH, USER_CONFIG_FILE_PATH
+
+
+# Global configuration object
+_global_config = None
+__all__ = [
+    'get_config',
+    'configure',
+]
+
+
+def _custom_excepthook(exception_class: type[BaseException], exception: BaseException, traceback: TracebackType):
+    '''Catches any uncaught exceptions and logs them'''
+    # sys.__excepthook__(exception_class, exception, traceback)
+    try:
+        raise exception
+    except:
+        logging.getLogger(PROJ_NAME).exception('Uncaught exception:')
+
+
+@dataclass
+class ConfigHandler:
+    data_path: str = str(DATA_PATH)
+    log_path: str = str(LOG_PATH)
+    logging_config_file_path: str = f'{MAIN_PATH}/logging.yml'
+    logging_config: dict | None = None
+    use_fork_process: bool = True
+    use_custom_excepthook: bool = False
+    env_file_path: str | None=None
+    debug: bool = False
+    
+    @classmethod
+    def get_instance(cls):
+        global _global_config
+        if _global_config is None:
+            _global_config = cls.load_config()
+        return _global_config
+    
+    @classmethod
+    def load_config(cls):
+        '''Loads user's config file and returns a ConfigHandler object'''
+        config_file_path = USER_CONFIG_FILE_PATH
+        if config_file_path.is_file():
+            with open(config_file_path, 'r') as f:
+                config = yaml.safe_load(f) or {}
+        else:
+            config = {}
+        return cls(**config)
+    
+    def __post_init__(self):
+        self.initialize()
+    
+    def initialize(self):
+        self.logging_config = self.logging_config or {}
+        
+        for path in [self.data_path]:
+            if not os.path.exists(path):
+                os.makedirs(path)
+                print(f'created {path}')
+                
+        if self.use_fork_process and sys.platform != 'win32':
+            multiprocessing.set_start_method('fork', force=True)
+        
+        if self.use_custom_excepthook and sys.excepthook is sys.__excepthook__:
+            sys.excepthook = _custom_excepthook
+        
+        self.load_env_file(self.env_file_path)
+        
+        if self.debug:
+            self.enable_debug_mode()
+    
+    def load_env_file(self, env_file_path: str | None):
+        if not env_file_path:
+            found_env_file_path = find_dotenv(usecwd=True, raise_error_if_not_found=False)
+            if found_env_file_path:
+                print(f'.env file path is not specified, using env file in "{found_env_file_path}"')
+            else:
+                # print('.env file is not found')
+                return
+        load_dotenv(env_file_path, override=True)
+    
+    def enable_debug_mode(self):
+        '''Enables debug mode by setting the log level to DEBUG for all stream handlers'''
+        if 'handlers' not in self.logging_config:
+            self.logging_config['handlers'] = {}
+        for handler in ['stream_handler', 'stream_path_handler']:
+            if handler not in self.logging_config['handlers']:
+                self.logging_config['handlers'][handler] = {}
+            self.logging_config['handlers'][handler]['level'] = 'DEBUG'
+
+
+def configure(
+    data_path: str | None = None,
+    log_path: str | None = None,
+    logging_config_file_path: str | None = None,
+    logging_config: dict | None = None,
+    use_fork_process: bool | None = None,
+    use_custom_excepthook: bool | None = None,
+    env_file_path: str | None = None,
+    debug: bool | None = None,
+    **kwargs,
+):
+    '''Configures the global config object.
+    It will override the existing config values from the existing config file or the default values.
+    '''
+    global _global_config
+    _global_config = get_config()
+    
+    # override config values, if provided
+    if data_path is not None:
+        _global_config.data_path = data_path
+    if log_path is not None:
+        _global_config.log_path = log_path
+    if logging_config_file_path is not None:
+        _global_config.logging_config_file_path = logging_config_file_path
+    if logging_config is not None:
+        _global_config.logging_config = logging_config
+    if use_fork_process is not None:
+        _global_config.use_fork_process = use_fork_process
+    if use_custom_excepthook is not None:
+        _global_config.use_custom_excepthook = use_custom_excepthook
+    if env_file_path is not None:
+        _global_config.env_file_path = env_file_path
+    if debug is not None:
+        _global_config.debug = debug
+            
+    for k, v in kwargs.items():
+        if hasattr(_global_config, k):
+            setattr(_global_config, k, v)
+        else:
+            raise AttributeError(f'{k} is not an attribute of ConfigHandler')
+    
+    _global_config.initialize()
+    return _global_config
+
+
+def get_config() -> ConfigHandler:
+    return ConfigHandler.get_instance()

pfeed/const/common.py

@@ -0,0 +1,15 @@
+SUPPORTED_ENVIRONMENTS = ['BACKTEST', 'SANDBOX', 'PAPER', 'LIVE']
+SUPPORTED_DATA_FEEDS = ['YAHOO_FINANCE', 'BYBIT', 'BINANCE']
+SUPPORTED_STORAGES = ['local', 'minio']
+SUPPORTED_DOWNLOAD_DATA_SOURCES = ['BYBIT', 'BINANCE']
+SUPPORTED_CRYPTO_EXCHANGES = ['BYBIT', 'BINANCE']
+SUPPORTED_DATA_TOOLS = ['pandas', 'polars']
+SUPPORTED_PRODUCT_TYPES = ['SPOT', 'PERP', 'IPERP', 'FUT', 'IFUT']
+SUPPORTED_DATA_TYPES = [
+    'raw_tick', 'raw_second', 'raw_minute', 'raw_hour', 'raw_daily',
+    'tick', 'second', 'minute', 'hour', 'daily',
+]
+
+ALIASES = {
+    'YF': 'YAHOO_FINANCE',
+}

pfeed/const/paths.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+from platformdirs import user_log_dir, user_data_dir, user_config_dir
+
+
+# project paths
+PROJ_NAME = Path(__file__).resolve().parents[1].name
+MAIN_PATH = Path(__file__).resolve().parents[2]
+PROJ_PATH = MAIN_PATH / PROJ_NAME
+
+
+# user paths
+LOG_PATH = Path(user_log_dir()) / PROJ_NAME
+DATA_PATH = Path(user_data_dir()) / PROJ_NAME
+USER_CONFIG_PATH = Path(user_config_dir()) / PROJ_NAME
+USER_CONFIG_FILE_PATH = USER_CONFIG_PATH / f'{PROJ_NAME}_config.yml'

pfeed/data_tools/data_tool_pandas.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from pfeed.resolution import ExtendedResolution
+    from pfeed.types.common_literals import tSUPPORTED_STORAGES
+
+import os
+import io
+
+import s3fs
+import pandas as pd
+
+from pfeed.const.common import SUPPORTED_STORAGES
+
+
+name = 'pandas'
+
+
+def read_parquet(path_or_obj: str | bytes, *args, storage: tSUPPORTED_STORAGES='local', **kwargs) -> pd.DataFrame:
+    assert storage in SUPPORTED_STORAGES, f'{storage=} not in {SUPPORTED_STORAGES}'
+    if isinstance(path_or_obj, bytes):
+        obj = io.BytesIO(path_or_obj)
+        return pd.read_parquet(obj, *args, **kwargs)
+    else:
+        path = path_or_obj
+        if storage == 'local':
+            return pd.read_parquet(path, *args, **kwargs)
+        elif storage == 'minio':
+            fs = s3fs.S3FileSystem(
+                endpoint_url="http://"+os.getenv('MINIO_HOST', 'localhost')+':'+os.getenv('MINIO_PORT', '9000'),
+                key=os.getenv('MINIO_ROOT_USER', 'pfunder'),
+                secret=os.getenv('MINIO_ROOT_PASSWORD', 'password'),
+            )
+            return pd.read_parquet(path, *args, filesystem=fs, **kwargs)
+        else:
+            raise NotImplementedError(f'{storage=}')
+
+
+def concat(dfs: list[pd.DataFrame], *args, **kwargs) -> pd.DataFrame:
+    return pd.concat(dfs, *args, **kwargs)
+
+
+def estimate_memory_usage(df: pd.DataFrame) -> float:
+    """Estimate the memory usage of a pandas DataFrame in GB."""
+    return df.memory_usage(deep=True).sum() / (1024 ** 3)
+
+    
+def organize_time_series_columns(pdt: str, resolution: str | ExtendedResolution, df: pd.DataFrame) -> pd.DataFrame:
+    """Organize the columns of a pandas DataFrame.
+    Moving 'ts', 'product', 'resolution' to the leftmost side.
+    """
+    from pfeed.resolution import ExtendedResolution
+    assert 'ts' in df.columns, "'ts' column not found"
+    assert 'product' not in df.columns, "'product' column already exists"
+    assert 'resolution' not in df.columns, "'resolution' column already exists"
+    if isinstance(resolution, str):
+        resolution = ExtendedResolution(resolution)
+    left_cols = ['ts', 'product', 'resolution']
+    df['product'] = pdt
+    df['resolution'] = repr(resolution)
+    df = df.reindex(left_cols + [col for col in df.columns if col not in left_cols], axis=1)
+    return df

pfeed/data_tools/data_tool_polars.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from pfeed.resolution import ExtendedResolution
+    from pfeed.types.common_literals import tSUPPORTED_STORAGES
+    
+import os
+
+import polars as pl
+
+from pfeed.const.common import SUPPORTED_STORAGES
+
+
+name = 'polars'
+
+
+def read_parquet(path_or_obj: str | bytes, *args, storage: tSUPPORTED_STORAGES='local', **kwargs) -> pl.DataFrame | pl.LazyFrame:
+    assert storage in SUPPORTED_STORAGES, f'{storage=} not in {SUPPORTED_STORAGES}'
+    if isinstance(path_or_obj, bytes):
+        obj = path_or_obj
+        return pl.read_parquet(obj, *args, **kwargs)
+    else:
+        path = path_or_obj
+        if storage == 'local':
+            return pl.scan_parquet(path, *args, **kwargs)
+        elif storage == 'minio':
+            storage_options = {
+                "endpoint_url": "http://"+os.getenv('MINIO_HOST', 'localhost')+':'+os.getenv('MINIO_PORT', '9000'),
+                "access_key_id": os.getenv('MINIO_ROOT_USER', 'pfunder'),
+                "secret_access_key": os.getenv('MINIO_ROOT_PASSWORD', 'password'),
+            }
+            return pl.scan_parquet(path, *args, storage_options=storage_options, **kwargs)
+        else:
+            raise NotImplementedError(f'{storage=}')
+
+
+def concat(dfs: list[pl.DataFrame | pl.LazyFrame], *args, **kwargs) -> pl.DataFrame | pl.LazyFrame:
+    return pl.concat(dfs, *args, **kwargs)
+
+        
+def estimate_memory_usage(df: pl.DataFrame | pl.LazyFrame) -> float:
+    """Estimate the memory usage of a polars DataFrame in GB."""
+    if isinstance(df, pl.LazyFrame):
+        df = df.collect()
+    return df.estimated_size(unit='gb')
+
+
+def organize_time_series_columns(pdt: str, resolution: str | ExtendedResolution, df: pl.DataFrame | pl.LazyFrame) -> pl.DataFrame | pl.LazyFrame:
+    from pfeed.resolution import ExtendedResolution
+    if isinstance(df, pl.LazyFrame):
+        cols = df.collect_schema().names()
+    else:
+        cols = df.columns
+    assert 'ts' in cols, "'ts' column not found"
+    assert 'product' not in cols, "'product' column already exists"
+    assert 'resolution' not in cols, "'resolution' column already exists"
+    if isinstance(resolution, str):
+        resolution = ExtendedResolution(resolution)
+    df = df.with_columns(
+        pl.lit(pdt).alias('product'),
+        pl.lit(repr(resolution)).alias('resolution')
+    )
+    left_cols = ['ts', 'product', 'resolution']
+    df = df.select(left_cols + [col for col in df.collect_schema().names() if col not in left_cols])
+    return df