mainsequence 2.0.0__py3-none-any.whl

mainsequence/virtualfundbuilder/__main__.py

@@ -0,0 +1,235 @@
+import time
+
+import fire
+
+import json
+
+import runpy
+import tempfile
+import os
+from pathlib import Path
+
+import requests
+import yaml
+from requests.structures import CaseInsensitiveDict
+
+
+def get_tdag_headers():
+    headers = CaseInsensitiveDict()
+    headers["Content-Type"] = "application/json"
+    headers["Authorization"] = "Token " + os.getenv("MAINSEQUENCE_TOKEN")
+    return headers
+
+def update_job_status(status_message):
+    url = f"{os.getenv('TDAG_ENDPOINT')}/orm/api/pods/job/job_run_status/"
+
+    payload = {
+        "status": status_message,
+        "git_hash": os.getenv("GIT_HASH"),
+        "command_id": os.getenv("COMMAND_ID")
+    }
+
+    response = requests.post(url, json=payload, headers=get_tdag_headers())
+
+    if response.status_code == 200:
+        data = response.json()
+        print("Update success:", data)
+        return data
+    else:
+        print("Error updating pod:", response.status_code, response.text)
+        return None
+
+def run_configuration(configuration_name):
+    from mainsequence.virtualfundbuilder.portfolio_interface import PortfolioInterface
+    from mainsequence.virtualfundbuilder.utils import get_vfb_logger
+    logger = get_vfb_logger()
+    logger.info(f"Run Timeseries Configuration {configuration_name}")
+    portfolio = PortfolioInterface.load_from_configuration(configuration_name)
+    res = portfolio.run()
+    print(res.head())
+
+
+def run_app(app_name, configuration):
+    from mainsequence.virtualfundbuilder.resource_factory.app_factory import APP_REGISTRY
+    from mainsequence.virtualfundbuilder.utils import get_vfb_logger
+    from mainsequence.virtualfundbuilder.config_handling import replace_none_and_empty_dict_with_python_none
+
+    logger = get_vfb_logger()
+    logger.info(f"Start App {app_name} with configuration\n{configuration}")
+    try:
+        app_cls = APP_REGISTRY[app_name]
+
+        configuration_json = yaml.load(configuration, Loader=yaml.UnsafeLoader)
+        configuration_json = replace_none_and_empty_dict_with_python_none(configuration_json)
+
+        # Pull out the dict under "configuration" (or flatten it, your choice)
+        actual_config = configuration_json.get("configuration", {})
+
+        # Now pass the unpacked dictionary:
+        pydantic_config = app_cls.configuration_class(**actual_config)
+
+        app_instance = app_cls(pydantic_config)
+        results = app_instance.run()
+    except Exception as e:
+        logger.error("Error running app", exc_info=True)
+        raise
+    logger.info(f"Finished App {app_name} run with results: {results}")
+
+def run_notebook(execution_object):
+    from mainsequence.virtualfundbuilder.notebook_handling import convert_notebook_to_python_file
+    from mainsequence.virtualfundbuilder.utils import get_vfb_logger
+    logger = get_vfb_logger()
+    logger.info(f"Run Notebook {execution_object}")
+    notebook_file_path = Path(os.getenv("VFB_PROJECT_PATH")).parent / execution_object
+    python_notebook_file = convert_notebook_to_python_file(notebook_file_path)
+    runpy.run_path(python_notebook_file, run_name="__main__")
+
+def run_script(execution_object):
+    from mainsequence.virtualfundbuilder.utils import get_vfb_logger
+    logger = get_vfb_logger()
+    logger.info(f"Run script {execution_object}")
+    python_file_path = Path(os.getenv("VFB_PROJECT_PATH")).parent / execution_object
+    runpy.run_path(python_file_path, run_name="__main__")
+
+def get_py_modules(folder_path):
+    if not os.path.isdir(folder_path): return []
+    files = os.listdir(folder_path)
+    files = [f for f in files if f[0] not in ["_", "."] and f.endswith(".py")]
+    return [f.split(".")[0] for f in files]
+
+def get_pod_configuration():
+    # TODO needs to introspect for apps in any folder?
+    print("Get pod configuration")
+
+    project_library = os.getenv("PROJECT_LIBRARY_NAME")
+    if not project_library:
+        raise RuntimeError("PROJECT_LIBRARY_NAME is not set in environment")
+
+    project_path = os.getenv("VFB_PROJECT_PATH")
+
+    # Gather all submodules in data_nodes
+    data_nodes_package = f"{project_library}.data_nodes"
+    data_nodes_modules = get_py_modules(os.path.join(project_path, "data_nodes"))
+
+    # Gather all submodules in rebalance_strategies
+    rebalance_package = f"{project_library}.rebalance_strategies"
+    rebalance_modules = get_py_modules(os.path.join(project_path, "rebalance_strategies"))
+
+    # Gather all submodules in apps
+    apps_package = f"{project_library}.apps"
+    apps_modules = get_py_modules(os.path.join(project_path, "apps"))
+
+    # Build the temporary Python script to import all files
+    script_lines = []
+
+    script_lines.append("# -- Auto-generated imports for data_nodes --")
+    for mod in data_nodes_modules:
+        script_lines.append(f"import {data_nodes_package}.{mod}")
+
+    script_lines.append("# -- Auto-generated imports for rebalance_strategies --")
+    for mod in rebalance_modules:
+        script_lines.append(f"import {rebalance_package}.{mod}")
+
+    script_lines.append("# -- Auto-generated imports for apps --")
+    for mod in apps_modules:
+        script_lines.append(f"import {apps_package}.{mod}")
+
+    script_lines.append("")
+    script_lines.append("from mainsequence.virtualfundbuilder.agent_interface import TDAGAgent")
+    script_lines.append("print('Initialize TDAGAgent')")
+    script_lines.append("tdag_agent = TDAGAgent()")
+
+    TMP_SCRIPT = "\n".join(script_lines)
+    print(f"Executing Script: \n{TMP_SCRIPT}")
+
+    # Write out to a temporary .py file and run
+    temp_dir = tempfile.mkdtemp()
+    python_file_path = Path(temp_dir) / "load_pod_configuration.py"
+    with open(python_file_path, "w", encoding="utf-8") as f:
+        f.write(TMP_SCRIPT)
+    try:
+        runpy.run_path(str(python_file_path), run_name="__main__")
+    except Exception:
+        from mainsequence.virtualfundbuilder.utils import get_vfb_logger
+        logger = get_vfb_logger()
+        logger.exception("--- ERROR: Failed to import one or more project modules. ---")
+        raise
+
+def import_project_configuration():
+    from mainsequence.client import ProjectConfiguration, Job
+    from mainsequence.virtualfundbuilder.utils import get_vfb_logger
+    logger = get_vfb_logger()
+
+    # load project configuration if exists
+    config_file = Path(os.getenv("VFB_PROJECT_PATH")).parent / "project_configuration.yaml"
+    logger.info(f"Try to import project configuration from {config_file}")
+
+    if not config_file.exists():
+        logger.info("No project configuration found")
+        return
+
+    with open(config_file, "r") as f:
+        project_configuration_raw = yaml.safe_load(f)
+    project_configuration = ProjectConfiguration(**project_configuration_raw)
+
+    logger.info(f"Create or update {project_configuration.jobs} jobs in backend")
+    for job in project_configuration.jobs:
+        Job.create_from_configuration(job_configuration=job.model_dump())
+
+
+def prerun_routines():
+    data = update_job_status("RUNNING")
+    env_update = data.get("environment_update", {})
+    for key, val in env_update.items():
+        os.environ[key] = val
+
+def postrun_routines(error_on_run: bool):
+    if error_on_run:
+        update_job_status("FAILED")
+    else:
+        update_job_status("SUCCEEDED")
+
+class VirtualFundLauncher:
+
+    def __init__(self):
+        from mainsequence.virtualfundbuilder.utils import get_vfb_logger
+        self.logger = get_vfb_logger()
+
+    def run_resource(self, execution_type, execution_object=None):
+        error_on_run = False
+
+        try:
+            prerun_routines()
+            if execution_type == "configuration":
+                run_configuration(execution_object)
+            elif execution_type == "script":
+                run_script(execution_object)
+            elif execution_type == "notebook":
+                run_notebook(execution_object)
+            elif execution_type == "system_job":
+                self.logger.info("Update project resources and import project configuration")
+                import_project_configuration()
+                # get_pod_configuration already called from import
+                pass
+            elif execution_type == "app":
+                run_app(app_name=os.getenv("APP_NAME"), configuration=os.getenv("APP_CONFIGURATION"))
+            elif execution_type == "standby":
+                sleep_seconds = int(os.getenv("STANDBY_DURATION_SECONDS"))
+                self.logger.info(f"Sleep for {sleep_seconds} seconds")
+                time.sleep(sleep_seconds)
+            else:
+                raise NotImplementedError(f"Unknown execution type {execution_type}")
+
+        except Exception as e:
+            self.logger.exception(f"Exception during job run occured {e}")
+            import traceback
+            traceback.print_exc()
+            error_on_run = True
+
+        finally:
+            postrun_routines(error_on_run)
+
+
+if __name__ == "__main__":
+    fire.Fire(VirtualFundLauncher)
+

mainsequence/virtualfundbuilder/agent_interface.py

@@ -0,0 +1,77 @@
+import copy
+import inspect
+import json
+import os
+import traceback
+from pathlib import Path
+
+import yaml
+
+from mainsequence.virtualfundbuilder.enums import ResourceType
+from mainsequence.virtualfundbuilder.portfolio_interface import PortfolioInterface
+from mainsequence.virtualfundbuilder.utils import is_jupyter_environment
+from .resource_factory.base_factory import send_resource_to_backend
+from .utils import logger
+
+
+class TDAGAgent:
+
+    def __init__(self):
+        self.logger = logger
+        self.logger.info("Setup TDAG Agent successfull")
+
+    def query_agent(self, query):
+        try:
+            from mainsequence.client.models_tdag import query_agent
+            payload = {
+                "query": query,
+            }
+            response = query_agent(json_payload=payload)
+            if response.status_code not in [200, 201]:
+                raise Exception(response.text)
+
+            answer = response.json()["agent_response"]
+        except Exception as e:
+            self.logger.warning(f"Could not get answer from Agent {e}")
+            traceback.print_exc()
+            return None
+
+        return answer
+
+    def generate_portfolio(self, cls, signal_description=None):
+        full_signal_description = f"Create me a default portfolio using the {cls.__name__} signal."
+        if signal_description is not None:
+            full_signal_description += f"\n{signal_description}"
+        else:
+            full_signal_description += f"Use NVDA, AAPL and GOOGL for the assets universe."
+
+        if is_jupyter_environment():
+            code = cls.get_source_notebook()
+        else:
+            code = inspect.getsource(cls)
+        attributes = {"code": code}
+        send_resource_to_backend(cls, attributes=attributes)
+
+        payload = {
+            "strategy_name": cls.__name__,
+            "signal_description": full_signal_description,
+        }
+        self.logger.info(f"Get configuration for {cls.__name__} ...")
+        payload = json.loads(json.dumps(payload))
+        try:
+            from mainsequence.client.models_tdag import create_configuration_for_strategy
+            response = create_configuration_for_strategy(json_payload=payload)
+            if response.status_code not in [200, 201]:
+                raise Exception(response.text)
+
+            generated_configuration = response.json()["generated_configuration"]["configuration"]["portfolio_configuration"]
+            portfolio = PortfolioInterface(generated_configuration)
+
+            self.logger.info(f"Received configuration:\n{portfolio}")
+
+        except Exception as e:
+            self.logger.warning(f"Could not get configuration from TSORM {e}")
+            traceback.print_exc()
+            return None
+
+        return portfolio

mainsequence/virtualfundbuilder/config_handling.py

@@ -0,0 +1,86 @@
+import logging
+from .models import *
+logger = get_vfb_logger()
+
+def replace_none_and_empty_dict_with_python_none(config):
+    """
+    Recursively replace all string 'None' with Python None in the given dictionary
+    and log the path where replacements occur.
+
+    Args:
+        config (dict): The configuration dictionary.
+
+    Returns:
+        dict: Updated dictionary with 'None' replaced by Python None.
+    """
+    logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+    def recursive_replace(d, path="root"):
+        if isinstance(d, dict):
+            for key, value in d.items():
+                current_path = f"{path}.{key}"
+                if isinstance(value, dict) and not value:
+                    d[key] = None
+                    logger.info(f"Replaced empty dict {{}} with Python None at: {current_path}")
+                elif isinstance(value, dict):
+                    recursive_replace(value, current_path)
+                elif isinstance(value, list):
+                    for i, item in enumerate(value):
+                        recursive_replace(item, f"{current_path}[{i}]")
+                elif isinstance(value, str) and value.lower() in ['none', 'null']:
+                    d[key] = None
+                    logger.info(f"Replaced 'None' in configuration with None at {current_path}")
+
+        elif isinstance(d, list):
+            for i, item in enumerate(d):
+                recursive_replace(item, f"{path}[{i}]")
+
+    recursive_replace(config)
+    return config
+
+def configuration_sanitizer(configuration: dict) -> PortfolioConfiguration:
+    """
+    Verifies that a configuration has all the required attributes.
+    Args:
+        configuration (dict): The configuration dictionary to sanitize.
+    Returns:
+        PortfolioConfiguration: The sanitized portfolio configuration.
+    """
+    configuration = copy.deepcopy(configuration)
+    configuration = replace_none_and_empty_dict_with_python_none(configuration)
+    portfolio_build_config = configuration["portfolio_build_configuration"]
+    for key in ["assets_configuration", "backtesting_weights_configuration", "execution_configuration"]:
+        if key not in portfolio_build_config:
+            raise KeyError(f"Missing required key {key}")
+
+    if portfolio_build_config["assets_configuration"] is not None:
+        if "prices_configuration" not in portfolio_build_config["assets_configuration"]:
+            raise Exception("Missing prices configuration in portfolio_build_config['assets_configuration']")
+
+    if "rebalance_strategy_configuration" not in portfolio_build_config["backtesting_weights_configuration"]:
+        raise Exception(
+            "Missing 'rebalance_strategy_configuration' in 'backtesting_weights_configuration'"
+        )
+
+    if "calendar" not in portfolio_build_config["backtesting_weights_configuration"]["rebalance_strategy_configuration"] or not portfolio_build_config["backtesting_weights_configuration"]["rebalance_strategy_configuration"]["calendar"]:
+        raise Exception(
+            "Missing 'calendar' in 'rebalance_strategy_configuration'"
+        )
+
+    if "signal_weights_configuration" not in portfolio_build_config["backtesting_weights_configuration"]:
+        raise Exception(
+            "Missing 'signal_weights_configuration' in 'backtesting_weights_configuration'"
+        )
+
+    if "signal_assets_configuration" not in portfolio_build_config['backtesting_weights_configuration']['signal_weights_configuration']:
+        raise Exception(
+            "Missing 'signal_weights_configuration' in 'backtesting_weights_configuration'"
+        )
+
+    if "portfolio_prices_frequency" not in portfolio_build_config:
+        raise Exception("Missing 'portfolio_prices_frequency' in 'portfolio_build_config'")
+
+    return PortfolioConfiguration.parse_portfolio_configurations(
+        portfolio_build_configuration=portfolio_build_config,
+        portfolio_markets_configuration=configuration['portfolio_markets_configuration'],
+    )

mainsequence/virtualfundbuilder/contrib/apps/__init__.py

@@ -0,0 +1,8 @@
+from .generate_report import *
+from .load_external_portfolio import *
+from .news_app import *
+from .run_portfolio import *
+from .run_named_portfolio import *
+from .portfolio_report_app import *
+from .portfolio_table import *
+from .etf_replicator_app import *

mainsequence/virtualfundbuilder/contrib/apps/etf_replicator_app.py

@@ -0,0 +1,164 @@
+import datetime
+from enum import Enum
+from typing import List, Union
+
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+from mainsequence.client import Portfolio, Asset
+from mainsequence.virtualfundbuilder.contrib.data_nodes import TrackingStrategy, TrackingStrategyConfiguration
+from mainsequence.virtualfundbuilder.models import PortfolioConfiguration
+from mainsequence.virtualfundbuilder.portfolio_interface import PortfolioInterface
+from mainsequence.virtualfundbuilder.utils import get_vfb_logger
+from pydantic import BaseModel
+from mainsequence.virtualfundbuilder.resource_factory.app_factory import (
+    HtmlApp,
+    register_app,
+)
+
+logger = get_vfb_logger()
+
+class ETFReplicatorConfiguration(BaseModel):
+    source_asset_category_identifier: str = "magnificent_7"
+    etf_to_replicate: str = "XLF"
+    in_window: int = 60
+    tracking_strategy: TrackingStrategy = TrackingStrategy.LASSO
+    tracking_strategy_configuration: TrackingStrategyConfiguration
+
+@register_app()
+class ETFReplicatorApp(HtmlApp):
+    configuration_class = ETFReplicatorConfiguration
+
+    def _build_portfolio_config(self) -> dict:
+        """
+        Loads a portfolio configuration template and customizes it for ETF replication.
+        """
+        portfolio_config = PortfolioInterface.load_configuration(configuration_name="market_cap")
+        signal_weights_configuration = {
+            "etf_ticker": self.configuration.etf_to_replicate,
+            "in_window": self.configuration.in_window,
+            "tracking_strategy": self.configuration.tracking_strategy,
+            "tracking_strategy_configuration": self.configuration.tracking_strategy_configuration,
+            "signal_assets_configuration": portfolio_config.portfolio_build_configuration.backtesting_weights_configuration.signal_weights_configuration[
+                "signal_assets_configuration"]
+        }
+        signal_weights_configuration["signal_assets_configuration"].assets_category_unique_id = self.configuration.source_asset_category_identifier
+
+        portfolio_config.portfolio_build_configuration.backtesting_weights_configuration.signal_weights_configuration = signal_weights_configuration
+        portfolio_config.portfolio_build_configuration.backtesting_weights_configuration.signal_weights_name = "ETFReplicator"
+        portfolio_config.portfolio_markets_configuration.portfolio_name = f"ETFReplicator Portfolio for {self.configuration.etf_to_replicate} using {self.configuration.source_asset_category_identifier}"
+
+        return portfolio_config.model_dump()
+
+    def _create_plot(self, df_plot_normalized: pd.DataFrame, weights_pivot: pd.DataFrame) -> str:
+        """
+        Creates a combined Plotly figure with performance and asset weight subplots.
+        """
+        fig = make_subplots(rows=2, cols=1, shared_xaxes=True, vertical_spacing=0.1,
+                            subplot_titles=("ETF Replication Performance", "Calculated Asset Weights"))
+
+        # Add performance traces to the first subplot
+        fig.add_trace(go.Scatter(
+            x=df_plot_normalized.index,
+            y=df_plot_normalized[self.configuration.etf_to_replicate],
+            mode='lines',
+            name=f"Original: {self.configuration.etf_to_replicate}"
+        ), row=1, col=1)
+
+        fig.add_trace(go.Scatter(
+            x=df_plot_normalized.index,
+            y=df_plot_normalized[f"Replicated_{self.configuration.etf_to_replicate}"],
+            mode='lines',
+            name="Replicated Portfolio"
+        ), row=1, col=1)
+
+        # Add weights traces to the second subplot as a stacked area chart
+        for asset in weights_pivot.columns:
+            fig.add_trace(go.Scatter(
+                x=weights_pivot.index,
+                y=weights_pivot[asset],
+                mode='lines',
+                name=asset,
+                showlegend=True
+            ), row=2, col=1)
+
+        fig.update_layout(
+            title_text=f"ETF Replication Analysis: {self.configuration.etf_to_replicate} vs. Replicated Portfolio",
+            legend_title="Series",
+            height=800  # Increase height to accommodate both plots
+        )
+
+        fig.update_yaxes(title_text="Normalized Performance (Indexed to 100)", row=1, col=1)
+        fig.update_yaxes(title_text="Asset Weight", row=2, col=1)
+
+        return fig.to_html(full_html=False, include_plotlyjs='cdn')
+
+    def run(self) -> str:
+        # Build Portfolio Configuration
+        portfolio_config_dump = self._build_portfolio_config()
+        portfolio = PortfolioInterface(portfolio_config_template=portfolio_config_dump)
+
+        # Run Portfolio
+        portfolio.run(add_portfolio_to_markets_backend=True)
+        self.add_output(output=portfolio.target_portfolio)
+
+        # Fetch Portfolio Results in batches to avoid timeouts
+        loop_start_date = portfolio.portfolio_strategy_data_node.OFFSET_START
+        final_end_date = datetime.datetime.now(datetime.timezone.utc)
+        all_results = []
+        current_start = loop_start_date
+        while current_start < final_end_date:
+            current_end = current_start + pd.DateOffset(months=6)
+            logger.info(f"Fetching data from {current_start.date()} to {min(current_end, final_end_date).date()}")
+            results_chunk = portfolio.portfolio_strategy_data_node.get_df_between_dates(start_date=current_start, end_date=current_end)
+            all_results.append(results_chunk)
+            current_start = current_end
+
+        results = pd.concat(all_results).drop_duplicates()
+        if results.empty:
+            return "<html><body><h1>No data available to generate ETF replication report.</h1></body></html>"
+
+        # Fetch and Process Data for Plotting
+        etf_replicator_signal = portfolio.portfolio_strategy_data_node.signal_weights
+        etf_asset = etf_replicator_signal.etf_asset
+        etf_data = etf_replicator_signal.etf_bars_ts.get_df_between_dates(
+            start_date=results.index.min(),
+            end_date=results.index.max(),
+            unique_identifier_list=[etf_asset.unique_identifier]
+        )
+
+        replicated_df = results.sort_index().reset_index()[['time_index', 'close']].rename(
+            columns={'close': f"Replicated_{self.configuration.etf_to_replicate}"}
+        )
+        original_df = etf_data.sort_index().reset_index()[['time_index', 'close']].rename(
+            columns={'close': self.configuration.etf_to_replicate}
+        )
+        df_plot = pd.concat([original_df, replicated_df]).sort_values("time_index").ffill()
+        df_plot = df_plot.set_index('time_index').dropna()
+        if df_plot.empty:
+            return "<html><body><h1>Could not align original and replicated portfolio data.</h1></body></html>"
+
+        df_plot_normalized = (df_plot / df_plot.iloc[0]) * 100
+
+        weights_df = etf_replicator_signal.get_df_between_dates(start_date=results.index.min(), end_date=results.index.max())
+        weights_df = weights_df.reset_index()
+        weight_assets = Asset.filter(unique_identifier__in=list(weights_df["unique_identifier"].unique()))
+        translation_map = {asset.unique_identifier: asset.ticker for asset in weight_assets}
+        weights_df["ticker"] = weights_df["unique_identifier"].map(translation_map)
+
+        weights_pivot = weights_df.pivot(index='time_index', columns='ticker', values='signal_weight').fillna(0)
+
+        # reduce size of plot
+        weights_pivot = weights_pivot.resample('W').last().fillna(0) # only use weekly weights
+        weights_pivot = weights_pivot.loc[:, (weights_pivot > 0.01).any(axis=0)] # filter out assets with very small weights
+
+        return self._create_plot(df_plot_normalized, weights_pivot)
+
+
+if __name__ == "__main__":
+    cfg = ETFReplicatorConfiguration(
+        tracking_strategy_configuration=TrackingStrategyConfiguration(),
+        source_asset_category_identifier="s&p500_constitutents"
+    )
+    ETFReplicatorApp(cfg).run()