mainsequence 2.0.0__py3-none-any.whl

mainsequence/virtualfundbuilder/models.py

@@ -0,0 +1,282 @@
+from typing import Dict, List, Optional
+
+from pydantic import (BaseModel, validator, model_validator)
+import os
+import pandas as pd
+
+from mainsequence.client import AssetMixin, Asset, MARKETS_CONSTANTS
+import json
+from pydantic import FieldValidationInfo, field_validator, root_validator, Field
+
+from mainsequence.virtualfundbuilder.enums import (PriceTypeNames)
+
+from mainsequence.client.models_tdag import RunConfiguration
+import mainsequence.client as msc
+import yaml
+from mainsequence.virtualfundbuilder.utils import get_vfb_logger
+from mainsequence.tdag.utils import write_yaml
+from mainsequence.tdag.utils import hash_dict
+import copy
+from functools import lru_cache
+from datetime import datetime
+from typing import Any, Dict, List, Literal, Union
+
+from pydantic import (
+    BaseModel,
+    Field,
+    PositiveInt,
+    model_validator,
+)
+
+logger = get_vfb_logger()
+
+class VFBConfigBaseModel(BaseModel):
+    class Config:
+        arbitrary_types_allowed = True
+
+
+class MarketsTimeSeries(VFBConfigBaseModel):
+    """
+    MarketsTimeSeries based on their unique id. Used as the data sources for the prices.
+    Values include alpaca_1d_bars, binance_1d_bars etc.
+
+    Attributes:
+        unique_identifier (str): Identfier of the MarketsTimeSeries.
+    """
+    unique_identifier: str = "alpaca_1d_bars"
+
+class PricesConfiguration(VFBConfigBaseModel):
+    """
+    Configuration for price data handling in a portfolio.
+
+    Attributes:
+        bar_frequency_id (str): The frequency of price bars.
+        upsample_frequency_id (str): Frequency to upsample intraday data to.
+        intraday_bar_interpolation_rule (str): Rule for interpolating missing intraday bars.
+        is_live (bool): Boolean flag indicating if the price feed is live.
+        translation_table_unique_id (str): The unique identifier of the translation table used to identify the price source.
+    """
+    bar_frequency_id: str = "1d"
+    upsample_frequency_id: str = "1d"  # "15m"
+    intraday_bar_interpolation_rule: str = "ffill"
+    is_live: bool = False
+    translation_table_unique_id: str = "prices_translation_table_1d"
+    forward_fill_to_now: bool = False
+
+@lru_cache(maxsize=1028)  # Cache up to 1028 different combinations
+def cached_asset_filter(*args,**kwargs):
+    tmp_assets = Asset.filter_with_asset_class( *args,**kwargs)
+    return tmp_assets
+
+class AssetsConfiguration(VFBConfigBaseModel):
+    """
+    Configuration for assets included in a portfolio.
+
+    Attributes:
+        assets_category_unique_id (str):
+            Unique Identifier of assets category
+        price_type (PriceTypeNames): Type of price used for backtesting.
+        prices_configuration (PricesConfiguration): Configuration for price data handling.
+    """
+    assets_category_unique_id: str
+    price_type: PriceTypeNames = PriceTypeNames.CLOSE
+    prices_configuration: PricesConfiguration
+
+    def get_asset_list(self):
+        asset_category=msc.AssetCategory.get(unique_identifier=self.assets_category_unique_id)
+        assets=msc.Asset.filter(id__in=asset_category.assets)
+        return assets
+
+class BacktestingWeightsConfig(VFBConfigBaseModel):
+    """
+    Configuration for backtesting weights.
+
+    Attributes:
+        rebalance_strategy_name (str): Strategy used for rebalancing.
+        rebalance_strategy_configuration (Dict): Placeholder dict for the rebalance strategy configuration.
+        signal_weights_name (str): Type of signal weights strategy.
+        signal_weights_configuration (Dict): Placeholder dict for the signal weights configuration.
+    """
+    rebalance_strategy_name: str = "ImmediateSignal"
+    rebalance_strategy_configuration: Dict
+    signal_weights_name: str = "MarketCap"
+    signal_weights_configuration: Dict
+
+    def model_dump(self, **kwargs):
+        signal_weights_configuration = self.signal_weights_configuration
+        data = super().model_dump(**kwargs)
+        data["signal_weights_configuration"]["signal_assets_configuration"] = signal_weights_configuration[
+            "signal_assets_configuration"].model_dump(**kwargs)
+
+        return data
+
+    @model_validator(mode="before")
+    def parse_signal_weights_configuration(cls, values):
+        if isinstance(values["signal_weights_configuration"]["signal_assets_configuration"], AssetsConfiguration):
+            return values
+
+        asset_configuration = copy.deepcopy(values["signal_weights_configuration"]["signal_assets_configuration"])
+        if "prices_configuration" not in asset_configuration:
+            logger.info("No Price Configuration in Configuration - Use Default Price Configuration")
+            asset_configuration["prices_configuration"] = PricesConfiguration()
+
+        values["signal_weights_configuration"]["signal_assets_configuration"] = AssetsConfiguration(
+            **asset_configuration
+        )
+        return values
+
+
+class PortfolioExecutionConfiguration(VFBConfigBaseModel):
+    """
+    Configuration for portfolio execution.
+
+    Attributes:
+        commission_fee (float): Commission fee percentage.
+    """
+    commission_fee: float = 0.00018
+
+
+class FrontEndDetails(VFBConfigBaseModel):
+    description: str  # required field; must be provided and cannot be None
+
+    signal_name: Optional[str] = None
+    signal_description: Optional[str] = None
+    rebalance_strategy_name: Optional[str] = None
+    rebalance_strategy_description: Optional[str] = None
+
+class PortfolioMarketsConfig(VFBConfigBaseModel):
+    """
+    Configuration for Virtual Asset Management (VAM) portfolio.
+
+    Attributes:
+        portfolio_name (str): Name of the portfolio.
+        execution_configuration (VAMExecutionConfiguration): Execution configuration for VAM.
+    """
+    portfolio_name: str = "Portfolio Strategy Title"
+    front_end_details: Optional[FrontEndDetails] = None
+
+class AssetMixinOverwrite(VFBConfigBaseModel):
+    """
+    The Asset for evaluating the portfolio.
+
+    Attributes:
+        unique_identifier (str): The unique_identifier of the asset.
+    """
+    unique_identifier: str = MARKETS_CONSTANTS.UNIQUE_IDENTIFIER_USD
+
+class PortfolioBuildConfiguration(VFBConfigBaseModel):
+    """
+    Main class for configuring and building a portfolio.
+
+    This class defines the configuration parameters needed for
+    building a portfolio, including asset configurations, backtesting
+    weights, and execution parameters.
+
+    Attributes:
+        assets_configuration (AssetsConfiguration): Configuration details for assets.
+        portfolio_prices_frequency (str): Frequency to upsample portoflio. Optional.
+        backtesting_weights_configuration (BacktestingWeightsConfig): Weights configuration used for backtesting.
+        execution_configuration (PortfolioExecutionConfiguration): Execution settings for the portfolio.
+    """
+    assets_configuration: AssetsConfiguration
+    portfolio_prices_frequency: Optional[str] = "1d"
+    backtesting_weights_configuration: BacktestingWeightsConfig
+    execution_configuration: PortfolioExecutionConfiguration
+
+    def model_dump(self, **kwargs):
+        serialized_asset_config = self.assets_configuration.model_dump(**kwargs)
+        data = super().model_dump(**kwargs)
+        data["assets_configuration"] = serialized_asset_config
+
+        data["backtesting_weights_configuration"] = self.backtesting_weights_configuration.model_dump(**kwargs)
+        return data
+
+    @root_validator(pre=True)
+    def parse_assets_configuration(cls, values):
+
+        if not isinstance(values["assets_configuration"], AssetsConfiguration) and values['assets_configuration'] is not None:
+            values["assets_configuration"] = AssetsConfiguration(
+                assets_category_unique_id=values['assets_configuration']['assets_category_unique_id'],
+                price_type=PriceTypeNames(values['assets_configuration']['price_type']),
+                prices_configuration=PricesConfiguration(
+                    **values['assets_configuration']['prices_configuration'])
+            )
+
+        return values
+
+
+class PortfolioConfiguration(VFBConfigBaseModel):
+    """
+        Configuration for a complete portfolio, including build configuration,
+        TDAG updates, and VAM settings.
+
+        This class aggregates different configurations required for the
+        management and operation of a portfolio.
+
+    Attributes:
+        portfolio_build_configuration (PortfolioBuildConfiguration): Configuration for building the portfolio.
+        portfolio_markets_configuration (PortfolioMarketsConfig): VAM execution configuration.
+    """
+    portfolio_build_configuration: PortfolioBuildConfiguration
+    portfolio_markets_configuration: PortfolioMarketsConfig
+
+    @staticmethod
+    def read_portfolio_configuration_from_yaml(yaml_path: str):
+        with open(yaml_path, 'r') as file:
+            return yaml.safe_load(file)
+
+    @staticmethod
+    def parse_portfolio_configuration_from_yaml(yaml_path: str, auto_complete=False):
+        from mainsequence.virtualfundbuilder.config_handling import configuration_sanitizer
+        configuration = PortfolioConfiguration.read_portfolio_configuration_from_yaml(yaml_path)
+        return configuration_sanitizer(configuration, auto_complete=auto_complete)
+
+    @staticmethod
+    def parse_portfolio_configurations(
+        portfolio_build_configuration: dict,
+        portfolio_markets_configuration: dict,
+    ):
+        # Parse the individual components
+        backtesting_weights_configuration = BacktestingWeightsConfig(
+            rebalance_strategy_name=portfolio_build_configuration['backtesting_weights_configuration']['rebalance_strategy_name'],
+            rebalance_strategy_configuration=portfolio_build_configuration['backtesting_weights_configuration'][
+                'rebalance_strategy_configuration'],
+            signal_weights_name=portfolio_build_configuration['backtesting_weights_configuration'][
+                'signal_weights_name'],
+            signal_weights_configuration=portfolio_build_configuration['backtesting_weights_configuration'][
+                'signal_weights_configuration']
+        )
+
+        execution_configuration = PortfolioExecutionConfiguration(
+            commission_fee=portfolio_build_configuration['execution_configuration']['commission_fee']
+        )
+
+        portfolio_build_config = PortfolioBuildConfiguration(
+            assets_configuration=portfolio_build_configuration['assets_configuration'],
+            backtesting_weights_configuration=backtesting_weights_configuration,
+            execution_configuration=execution_configuration,
+            portfolio_prices_frequency=portfolio_build_configuration['portfolio_prices_frequency'],
+        )
+
+        portfolio_markets_configuration = PortfolioMarketsConfig(**portfolio_markets_configuration)
+
+        # Combine everything into the final PortfolioConfiguration
+        portfolio_config = PortfolioConfiguration(
+            portfolio_build_configuration=portfolio_build_config,
+            portfolio_markets_configuration=portfolio_markets_configuration
+        )
+
+        return portfolio_config
+
+    def build_yaml_configuration_file(self):
+        signal_type = self.portfolio_build_configuration.backtesting_weights_configuration.signal_weights_name
+        vfb_folder = os.path.join(os.path.expanduser("~"), "VirtualFundBuilder", "configurations")
+        vfb_folder = os.path.join(vfb_folder, signal_type)
+        if not os.path.exists(vfb_folder):
+            os.makedirs(vfb_folder)
+
+        config_hash = hash_dict(self.model_dump_json())
+        config_file_name = f"{vfb_folder}/{config_hash}.yaml"
+
+        write_yaml(dict_file=json.loads(self.model_dump_json()), path=config_file_name)
+        return config_file_name

mainsequence/virtualfundbuilder/notebook_handling.py

@@ -0,0 +1,42 @@
+import tempfile
+from pathlib import Path
+from nbconvert import PythonExporter
+import importlib.util
+import sys
+import inspect
+from typing import List, Type
+
+
+def convert_notebook_to_python_file(notebook_path):
+    """
+    Converts a Jupyter notebook to a Python file in a temporary directory.
+
+    Args:
+        notebook_path (str or pathlib.Path): The path to the Jupyter notebook (.ipynb) file.
+
+    Returns:
+        pathlib.Path: The path to the generated Python file in the temporary directory.
+    """
+
+    # Ensure notebook_path is a Path object
+    notebook_path = Path(notebook_path)
+
+    # Create a temporary directory
+    temp_dir = tempfile.mkdtemp()
+
+    # Convert the notebook to a Python script
+    exporter = PythonExporter()
+    (python_code, resources) = exporter.from_filename(str(notebook_path))
+
+    # Create the output file path
+    python_file_path = Path(temp_dir) / (notebook_path.stem + ".py")
+
+    # Write the Python code to the file
+    with open(python_file_path, "w", encoding="utf-8") as f:
+        f.write(python_code)
+
+    print(f"Notebook converted and saved to: {python_file_path}")
+
+    return python_file_path
+
+

mainsequence/virtualfundbuilder/portfolio_interface.py

@@ -0,0 +1,272 @@
+import copy
+from mainsequence.tdag.utils import write_yaml
+import os
+from typing import Dict, Any, List, Union, Optional, Tuple
+import yaml
+import re
+
+from .config_handling import configuration_sanitizer
+from .data_nodes import PortfolioStrategy, PortfolioFromDF
+from mainsequence.client import Asset, AssetFutureUSDM, MARKETS_CONSTANTS as CONSTANTS, Portfolio, PortfolioIndexAsset
+
+from .models import PortfolioConfiguration
+from .utils import get_vfb_logger
+
+
+class PortfolioInterface():
+    """
+    Manages the overall strategy of investing. It initializes the tree and runs it either within the scheduler or
+    directly with a full tree update.
+    """
+    def __init__(self, portfolio_config_template: dict,
+                 configuration_name: str=None,is_portfolio_from_df=False):
+        """
+        Initializes the portfolio strategy with the necessary configurations.
+        """
+        if is_portfolio_from_df == True:
+            return None
+        if configuration_name:
+            self.check_valid_configuration_name(configuration_name)
+        self.portfolio_config_template = portfolio_config_template
+        self.portfolio_config = configuration_sanitizer(portfolio_config_template)
+        self.configuration_name = configuration_name
+
+        self.portfolio_markets_config = self.portfolio_config.portfolio_markets_configuration
+        self.portfolio_build_configuration = self.portfolio_config.portfolio_build_configuration
+        self.logger = get_vfb_logger()
+        self._is_initialized = False
+
+    def __str__(self):
+        configuration_name = self.configuration_name or "-"
+        str_configuration = yaml.dump(self.portfolio_config_template, default_flow_style=False)
+        return f"Configuration Name: {configuration_name}\n{str_configuration}"
+
+    def __repr__(self):
+        return self.__str__()
+
+    @classmethod
+    def build_and_run_portfolio_from_df(cls,portfolio_node:PortfolioFromDF,
+                                        debug_mode=True,
+                                        force_update=True,
+                                        update_tree=True,
+                                        portfolio_tags: List[str] = None,
+                                        add_portfolio_to_markets_backend=False,
+                                        ):
+        assert isinstance(portfolio_node, PortfolioFromDF)
+
+        interface=cls(portfolio_config_template=None,is_portfolio_from_df=True)
+        interface._is_initialized=True
+        interface.portfolio_strategy_data_node=portfolio_node
+
+
+
+        interface.run( patch_build_configuration=False,
+            debug_mode=debug_mode,
+            force_update=force_update,
+            update_tree=update_tree,
+            portfolio_tags = portfolio_tags,
+            add_portfolio_to_markets_backend=False,)
+
+
+        ## manualely
+        target_portfolio = Portfolio.get_or_none(local_time_serie__id=portfolio_node.local_time_serie.id)
+        standard_kwargs = dict(portfolio_name=portfolio_node.portfolio_name,
+                               local_time_serie_id=portfolio_node.local_time_serie.id,
+                               signal_local_time_serie_id=None,
+                               is_active=True,
+                               calendar_name=portfolio_node.calendar_name,
+                               target_portfolio_about=dict(description=portfolio_node.target_portfolio_about,
+                                                           signal_name=None,
+                                                           signal_description=None,
+                                                           rebalance_strategy_name=None),
+                               backtest_table_price_column_name="close")
+        if target_portfolio is None:
+            target_portfolio, index_asset = Portfolio.create_from_time_series(**standard_kwargs)
+        else:
+            # patch timeserie of portfolio to guaranteed recreation
+            target_portfolio.patch(**standard_kwargs)
+
+            index_asset = PortfolioIndexAsset.get(reference_portfolio__id=target_portfolio.id)
+
+
+    def _initialize_nodes(self,patch_build_configuration=True) -> None:
+        """
+        Initializes the portfolio strategy for backtesting and for live prediction.
+        Also, forces an update of the build configuration in tdag to guarantee that assets are properly rebuilt
+        patch_build_configuration:defaults to True as we want to patch the configuration while we test but for production can be set to False
+        """
+        patch = os.environ.get("PATCH_BUILD_CONFIGURATION", "False")
+        os.environ[
+            "PATCH_BUILD_CONFIGURATION"] = "True"  if patch_build_configuration else "False" # It always needs to be true as we always want to overwrite the build
+        self.portfolio_strategy_data_node = PortfolioStrategy(
+            portfolio_build_configuration=copy.deepcopy(self.portfolio_build_configuration)
+        )
+
+        os.environ["PATCH_BUILD_CONFIGURATION"] = patch
+        self._is_initialized = True
+
+    def build_target_portfolio_in_backend(self, portfolio_tags=None) -> Tuple[Portfolio, PortfolioIndexAsset]:
+        """
+        This method creates a portfolio in VAM with configm file settings.
+
+        Returns:
+        """
+        if not self._is_initialized:
+            self._initialize_nodes()
+
+        portfolio_ts = self.portfolio_strategy_data_node
+
+        def build_markets_portfolio(ts, portfolio_tags):
+            # when is live target portfolio
+            signal_weights_ts = ts.signal_weights
+
+            # timeseries can be running in local lake so need to request the id
+            standard_kwargs = dict(local_time_serie_id=ts.local_time_serie.id,
+                                   is_active=True,
+                                   signal_local_time_serie_id=signal_weights_ts.local_time_serie.id,
+                                   )
+
+            user_kwargs = self.portfolio_markets_config.model_dump()
+            user_kwargs.pop("front_end_details", None)
+
+            standard_kwargs.update(user_kwargs)
+            standard_kwargs["calendar_name"] = self.portfolio_build_configuration.backtesting_weights_configuration.rebalance_strategy_configuration[
+                                                        "calendar"]
+
+            if portfolio_tags is not None:
+                standard_kwargs["tags"] = portfolio_tags
+                # front end details
+            standard_kwargs["target_portfolio_about"] = {
+                "description": ts.get_portfolio_about_text(),
+                "signal_name": ts.backtesting_weights_config.signal_weights_name,
+                "signal_description": ts.signal_weights.get_explanation(),
+                "rebalance_strategy_name": ts.backtesting_weights_config.rebalance_strategy_name,
+            }
+
+            standard_kwargs["backtest_table_price_column_name"] = "close"
+
+            target_portfolio = Portfolio.get_or_none(local_time_serie__id=ts.local_time_serie.id)
+            if target_portfolio is None:
+                target_portfolio, index_asset = Portfolio.create_from_time_series(**standard_kwargs)
+            else:
+                # patch timeserie of portfolio to guaranteed recreation
+                target_portfolio.patch(**standard_kwargs)
+                self.logger.debug(f"Target portfolio {target_portfolio.portfolio_ticker} for local time serie {ts.local_time_serie.update_hash} already exists in Backend")
+                index_asset = PortfolioIndexAsset.get(reference_portfolio__id=target_portfolio.id)
+
+            return target_portfolio, index_asset
+
+        target_portfolio, index_asset = build_markets_portfolio(portfolio_ts, portfolio_tags=portfolio_tags)
+
+        self.index_asset = index_asset
+        self.target_portfolio = target_portfolio
+        return target_portfolio, index_asset
+
+    def run(
+            self,
+            patch_build_configuration=True,
+            debug_mode=True,
+            force_update=True,
+            update_tree=True,
+            portfolio_tags:List[str] = None,
+            add_portfolio_to_markets_backend=False,
+            *args, **kwargs
+    ):
+
+        if not self._is_initialized or patch_build_configuration == True:
+            self._initialize_nodes(patch_build_configuration=patch_build_configuration)
+
+        self.portfolio_strategy_data_node.run(
+            debug_mode=debug_mode,
+            update_tree=update_tree,
+            force_update=force_update,
+            **kwargs
+        )
+        if add_portfolio_to_markets_backend:
+            self.build_target_portfolio_in_backend(portfolio_tags=portfolio_tags)
+
+        res = self.portfolio_strategy_data_node.get_df_between_dates()
+        if len(res) > 0:
+            res = res.sort_values("time_index")
+        return res
+
+    @classmethod
+    @property
+    def configuration_folder_path(self):
+        vfb_project_path = os.getenv("VFB_PROJECT_PATH")
+        if not vfb_project_path:
+            raise ValueError(
+                "VFB_PROJECT_PATH environment variable is not set. "
+                "Please set it before using 'configuration_path'."
+            )
+        return os.path.join(vfb_project_path, "configurations")
+
+    @staticmethod
+    def check_valid_configuration_name(s: str) -> bool:
+        if not bool(re.match(r'^[A-Za-z0-9_]+$', s)):
+            raise ValueError(f"Name {s} not valid")
+
+    def store_configuration(self, configuration_name: Optional[str] = None):
+        """
+        Stores the current configuration as a YAML file under the configuration_name
+        """
+        if configuration_name and not self.configuration_name:
+            self.configuration_name = configuration_name
+
+        if not self.configuration_name:
+            raise ValueError(
+                "No configuration name was set. Provide a `configuration_name` "
+                "argument or load/set one before storing."
+            )
+
+        config_file = os.path.join(
+            self.configuration_folder_path,
+            f"{self.configuration_name}.yaml"
+        )
+
+        write_yaml(dict_file=self.portfolio_config_template, path=config_file)
+        self.logger.info(f"Configuration stored under {config_file}")
+        return config_file
+
+
+    @classmethod
+    def load_configuration(cls, configuration_name) -> PortfolioConfiguration:
+        config_file = os.path.join(cls.configuration_folder_path, f"{configuration_name}.yaml")
+        portfolio_config = PortfolioConfiguration.read_portfolio_configuration_from_yaml(config_file)
+        return PortfolioConfiguration(**portfolio_config)
+
+    @classmethod
+    def load_from_configuration(cls, configuration_name, config_file: Optional[str] = None):
+        if config_file is None:
+            config_file = os.path.join(cls.configuration_folder_path, f"{configuration_name}.yaml")
+        if not os.path.exists(config_file):
+            raise FileNotFoundError(f"Configuration file '{config_file}' does not exist.")
+
+        portfolio_config = PortfolioConfiguration.read_portfolio_configuration_from_yaml(config_file)
+        portfolio = cls(portfolio_config_template=portfolio_config, configuration_name=configuration_name)
+        return portfolio
+
+    @classmethod
+    def list_configurations(cls):
+        """
+        Lists all YAML configuration files found in the configuration_path.
+        """
+        if not os.path.exists(cls.configuration_folder_path):
+            return []
+
+        files = os.listdir(cls.configuration_folder_path)
+        yaml_files = [f for f in files if f.endswith(".yaml")]
+        # Strip off the '.yaml' extension to return just the base names
+        return [os.path.splitext(f)[0] for f in yaml_files]
+
+    def delete_stored_configuration(self):
+        """
+        Removes a saved configuration file from the configuration folder
+        """
+        if not self.configuration_name:
+            raise ValueError("No configuration name set. Cannot delete an unnamed configuration.")
+        config_file = os.path.join(self.configuration_folder_path, f"{self.configuration_name}.yaml")
+        if not os.path.exists(config_file):
+            raise FileNotFoundError(f"Configuration file '{config_file}' does not exist.")
+        os.remove(config_file)
+        self.logger.info(f"Deleted configuration file '{config_file}'.")