mainsequence 2.0.0__py3-none-any.whl

mainsequence/virtualfundbuilder/resource_factory/app_factory.py

@@ -0,0 +1,170 @@
+import hashlib
+import json
+import os
+from abc import abstractmethod
+from typing import Any
+from pydantic import BaseModel
+import inspect
+from mainsequence.client.models_tdag import Artifact, add_created_object_to_jobrun
+from mainsequence.virtualfundbuilder.enums import ResourceType
+from mainsequence.virtualfundbuilder.resource_factory.base_factory import insert_in_registry, BaseResource
+from mainsequence.virtualfundbuilder.utils import get_vfb_logger
+
+logger = get_vfb_logger()
+
+class BaseApp(BaseResource):
+    TYPE = ResourceType.APP
+
+    def __init__(self, configuration: BaseModel):
+        self.configuration = configuration
+
+    def add_output(self, output: Any):
+        """
+        Saves the given output in the backend.
+        """
+        logger.info(f"Add object {output} to job run output")
+        job_id = os.getenv("JOB_RUN_ID", None)
+
+        if job_id:
+            add_created_object_to_jobrun(
+                model_name=output.orm_class,
+                app_label=output.get_app_label(),
+                object_id=output.id
+            )
+            logger.info("Output added successfully")
+        else:
+            logger.info("This is not a Job Run - no output can be added")
+    @staticmethod
+    def hash_pydantic_object(obj: Any,digest_size: int = 16) -> str:
+        """
+        Generate a unique SHA-256 hash for any Pydantic object (including nested dependencies),
+        ensuring that lists of objects are deterministically ordered.
+
+        Args:
+            obj: A Pydantic BaseModel instance or any JSON-serializable structure.
+
+        Returns:
+            A hex string representing the SHA-256 hash of the canonical JSON representation.
+        """
+
+        def serialize(item: Any) -> Any:
+            if isinstance(item, BaseModel):
+                return serialize(item.dict(by_alias=True, exclude_unset=True))
+            elif isinstance(item, dict):
+                return {str(k): serialize(v) for k, v in sorted(item.items())}
+            elif isinstance(item, (list, tuple, set)):
+                serialized_items = [serialize(v) for v in item]
+                try:
+                    return sorted(
+                        serialized_items,
+                        key=lambda x: json.dumps(x, sort_keys=True, separators=(",", ":"))
+                    )
+                except (TypeError, ValueError):
+                    return serialized_items
+            else:
+                if hasattr(item, "isoformat"):
+                    try:
+                        return item.isoformat()
+                    except Exception:
+                        pass
+                return item
+
+        json_str = json.dumps(serialize(obj), sort_keys=True, separators=(",", ":"))
+        h = hashlib.blake2b(digest_size=digest_size)
+        h.update(json_str.encode("utf-8"))
+        return h.hexdigest()
+
+
+APP_REGISTRY = APP_REGISTRY if 'APP_REGISTRY' in globals() else {}
+def register_app(name=None, register_in_agent=True):
+    """
+    Decorator to register a model class in the factory.
+    If `name` is not provided, the class's name is used as the key.
+    """
+    def decorator(cls):
+        return insert_in_registry(APP_REGISTRY, cls, register_in_agent, name)
+    return decorator
+
+class HtmlApp(BaseApp):
+    """
+    A base class for apps that generate HTML output.
+    """
+    TYPE = ResourceType.HTML_APP
+
+    def __init__(self, *args, **kwargs):
+        self.created_artifacts = []
+        super().__init__(*args, **kwargs)
+
+    def _get_hash_from_configuration(self):
+        try:
+            return hashlib.sha256(
+                json.dumps(self.configuration.__dict__, sort_keys=True, default=str).encode()
+            ).hexdigest()[:8]
+        except Exception as e:
+            logger.warning(f"[{self.__name__}] Could not hash configuration: {e}")
+
+    def add_html_output(self, html_content, output_name=None):
+        """
+        Saves the given HTML content to a file, uploads it as an artifact,
+        and stores the artifact reference.
+        If output_name is not provided, a sequential name (e.g., ClassName_1.html) is generated.
+        """
+        if not isinstance(html_content, str):
+            raise TypeError(f"The 'add_html_output' method of {self.__class__.__name__} must be called with a string of HTML content.")
+
+        if output_name is None:
+            output_name = len(self.created_artifacts)
+
+        output_name = f"{self.__class__.__name__}_{output_name}.html"
+
+        try:
+            with open(output_name, "w", encoding="utf-8") as f:
+                f.write(html_content)
+
+            logger.info(f"[{self.__class__.__name__}] Successfully saved HTML to: {output_name}")
+        except IOError as e:
+            logger.error(f"[{self.__class__.__name__}] Error saving file: {e}")
+            raise
+
+        job_id = os.getenv("JOB_ID", None)
+        if job_id:
+            html_artifact = None
+            try:
+                html_artifact = Artifact.upload_file(
+                    filepath=output_name,
+                    name=output_name,
+                    created_by_resource_name=self.__class__.__name__,
+                    bucket_name="HTMLOutput"
+                )
+                if html_artifact:
+                    self.created_artifacts.append(html_artifact)
+                    self.add_output(html_artifact)
+                    logger.info(f"Artifact uploaded successfully: {html_artifact.id}")
+                else:
+                    logger.info("Artifact upload failed")
+            except Exception as e:
+                logger.info(f"Error uploading artifact: {e}")
+
+
+    def __init_subclass__(cls, **kwargs):
+        """
+        Wraps the subclass's `run` method to add validation and saving logic.
+        """
+        super().__init_subclass__(**kwargs)
+        original_run = cls.run
+
+        def run_wrapper(self, *args, **kwargs) -> str:
+            html_content = original_run(self, *args, **kwargs)
+
+            if html_content:
+                self.add_html_output(html_content)
+
+        cls.run = run_wrapper
+
+    @abstractmethod
+    def run(self) -> str:
+        """
+        This method should be implemented by subclasses to return HTML content as a string.
+        The base class will handle saving the output.
+        """
+        raise NotImplementedError("Subclasses of HtmlApp must implement the 'run' method.")

mainsequence/virtualfundbuilder/resource_factory/base_factory.py

@@ -0,0 +1,238 @@
+import json
+import os
+import inspect
+import importlib.util
+from threading import Thread
+
+import yaml
+
+from mainsequence.client.models_tdag import DynamicResource
+from mainsequence.tdag import DataNode
+from mainsequence.virtualfundbuilder.utils import get_vfb_logger, create_schema_from_signature
+from typing import get_type_hints, List, Optional, Union
+from pydantic import BaseModel
+from enum import Enum
+from pathlib import Path
+import sys
+import ast
+
+logger = get_vfb_logger()
+from mainsequence.virtualfundbuilder.enums import ResourceType
+from mainsequence.virtualfundbuilder.utils import runs_in_main_process
+
+class BaseResource():
+    @classmethod
+    def get_source_notebook(cls):
+        """Retrieve the exact source code of the class from notebook cells."""
+        from IPython import get_ipython
+        ipython_shell = get_ipython()
+        history = ipython_shell.history_manager.get_range()
+
+        for _, _, cell_content in history:
+            try:
+                # Parse the cell content as Python code
+                parsed = ast.parse(cell_content)
+
+                # Look for the class definition in the AST (Abstract Syntax Tree)
+                for node in ast.walk(parsed):
+                    if isinstance(node, ast.ClassDef) and node.name == cls.__name__:
+                        # Extract the start and end lines of the class
+                        start_line = node.lineno - 1
+                        end_line = max(
+                            [child.lineno for child in ast.walk(node) if hasattr(child, "lineno")]
+                        )
+                        lines = cell_content.splitlines()
+                        return "\n".join(lines[start_line:end_line])
+            except Exception as e:
+                print(e)
+                continue
+
+        return "Class definition not found in notebook history."
+
+    @classmethod
+    def build_and_parse_from_configuration(cls, **kwargs) -> 'WeightsBase':
+        type_hints = get_type_hints(cls.__init__)
+
+        def parse_value_into_hint(value, hint):
+            """
+            Recursively parse `value` according to `hint`.
+            Handles:
+              - Pydantic models
+              - Enums
+              - Lists of Pydantic models
+              - Optional[...] / Union[..., NoneType]
+            """
+            if value is None:
+                return None
+
+            from typing import get_origin, get_args, Union
+            origin = get_origin(hint)
+            args = get_args(hint)
+
+            # Handle Optional/Union
+            # e.g. Optional[SomeModel] => Union[SomeModel, NoneType]
+            if origin is Union and len(args) == 2 and type(None) in args:
+                # Identify the non-None type
+                non_none_type = args[0] if args[1] == type(None) else args[1]
+                return parse_value_into_hint(value, non_none_type)
+
+            # Handle single Pydantic model
+            if inspect.isclass(hint) and issubclass(hint, BaseModel):
+                if not isinstance(value, hint):
+                    return hint(**value)
+                return value
+
+            # Handle single Enum
+            if inspect.isclass(hint) and issubclass(hint, Enum):
+                if not isinstance(value, hint):
+                    return hint(value)
+                return value
+
+            # Handle List[...] of Pydantic models or other types
+            if origin is list:
+                inner_type = args[0]
+                # If the list elements are Pydantic models
+                if inspect.isclass(inner_type) and issubclass(inner_type, BaseModel):
+                    return [
+                        inner_type(**item) if not isinstance(item, inner_type) else item
+                        for item in value
+                    ]
+                # If the list elements are Enums, or other known transformations, handle similarly
+                if inspect.isclass(inner_type) and issubclass(inner_type, Enum):
+                    return [
+                        inner_type(item) if not isinstance(item, inner_type) else item
+                        for item in value
+                    ]
+                # Otherwise, just return the list as is
+                return value
+
+            # If none of the above, just return the value unchanged.
+            return value
+
+        # Now loop through each argument in kwargs and parse
+        for arg, value in kwargs.items():
+            if arg in type_hints:
+                hint = type_hints[arg]
+                kwargs[arg] = parse_value_into_hint(value, hint)
+
+        return cls(**kwargs)
+
+SKIP_REGISTRATION = os.getenv("SKIP_REGISTRATION", "").lower() == "true"
+def insert_in_registry(registry, cls, register_in_agent, name=None, attributes: Optional[dict]=None):
+    """ helper for strategy decorators """
+    key = name or cls.__name__  # Use the given name or the class name as the key
+
+    if key in registry and register_in_agent:
+        logger.debug(f"{cls.TYPE} '{key}' is already registered.")
+        return cls
+
+    registry[key] = cls
+    logger.debug(f"Registered {cls.TYPE} class '{key}': {cls}")
+
+    if register_in_agent and not SKIP_REGISTRATION and runs_in_main_process():
+        # send_resource_to_backend(cls, attributes)
+        Thread(
+            target=send_resource_to_backend,
+            args=(cls, attributes),
+        ).start()
+
+    return cls
+
+
+class BaseFactory:
+    @staticmethod
+    def import_module(strategy_name):
+        VFB_PROJECT_PATH = os.environ.get("VFB_PROJECT_PATH", None)
+        assert VFB_PROJECT_PATH, "There is no signals folder variable specified"
+
+        project_path = Path(VFB_PROJECT_PATH)
+
+        strategy_folder_path = project_path / strategy_name
+        logger.debug(f"Registering signals from {strategy_folder_path}")
+        package_name = f"{project_path.name}.{strategy_name}"
+
+        project_root_path = project_path.parent.parent
+        if project_root_path not in sys.path:
+            sys.path.insert(0, project_root_path)
+
+        for filename in os.listdir(strategy_folder_path):
+            try:
+                if filename.endswith(".py"):
+                    # Build the full module name
+                    module_name = f"{package_name}.{filename[:-3]}"
+
+                    # Dynamically import the module
+                    module = importlib.import_module(module_name)
+            except Exception as e:
+                logger.warning(f"Error reading code in strategy {filename}: {e}")
+
+
+def send_resource_to_backend(resource_class, attributes: Optional[dict] = None):
+    """
+    Parses the __init__ signatures of the class and its parents to generate a
+    unified JSON schema and sends the resource payload to the backend.
+    """
+    merged_properties = {}
+    merged_required = set()
+    merged_definitions = {}
+
+    # Special case for BaseApp subclasses that use a configuration_class
+    if hasattr(resource_class, 'configuration_class') and inspect.isclass(resource_class.configuration_class) and issubclass(resource_class.configuration_class,
+                                                                                                                             BaseModel):
+        config_class = resource_class.configuration_class
+        config_name = config_class.__name__
+
+        # Get the full schema for the configuration class
+        config_schema = config_class.model_json_schema(ref_template="#/$defs/{model}")
+
+        # Merge any nested definitions from the config schema
+        if "$defs" in config_schema:
+            merged_definitions.update(config_schema.pop("$defs"))
+
+        # Add the configuration class's own schema to the definitions
+        merged_definitions[config_name] = config_schema
+
+        # Create a top-level "configuration" property that references the schema
+        merged_properties["configuration"] = {
+            "$ref": f"#/$defs/{config_name}",
+            "title": "Configuration"
+        }
+        # Mark the top-level "configuration" as required
+        merged_required.add("configuration")
+
+    else:
+        # Standard logic for other resource types
+        for parent_class in reversed(resource_class.__mro__):
+            if parent_class is object or not hasattr(parent_class, '__init__') or parent_class is DataNode:
+                continue
+            if "__init__" in parent_class.__dict__:
+                parent_schema = create_schema_from_signature(parent_class.__init__)
+                merged_properties.update(parent_schema.get("properties", {}))
+                merged_definitions.update(parent_schema.get("$defs", {}))
+                merged_required.update(parent_schema.get("required", []))
+
+    final_json_schema = {
+        "title": resource_class.__name__,
+        "type": "object",
+        "properties": merged_properties,
+    }
+    if merged_required:
+        schema_required = sorted([
+            field for field in merged_required
+            if 'default' not in merged_properties.get(field, {})
+        ])
+        if schema_required:
+            final_json_schema["required"] = schema_required
+
+    if merged_definitions:
+        final_json_schema["$defs"] = merged_definitions
+
+    resource_config = DynamicResource.create(
+        name=resource_class.__name__,
+        type=resource_class.TYPE.value,
+        object_signature=final_json_schema,
+        attributes=attributes,
+    )
+
+    logger.debug(f"Sending resource '{resource_class.__name__}' to backend.")
+

mainsequence/virtualfundbuilder/resource_factory/rebalance_factory.py

@@ -0,0 +1,101 @@
+import pandas as pd
+import datetime
+import logging
+import pandas_market_calendars as mcal
+
+from mainsequence.virtualfundbuilder.enums import ResourceType
+from mainsequence.virtualfundbuilder.resource_factory.base_factory import BaseFactory, BaseResource, insert_in_registry
+
+logger = logging.getLogger("virtualfundbuilder")
+
+class RebalanceStrategyBase(BaseResource):
+    TYPE = ResourceType.REBALANCE_STRATEGY
+
+    def __init__(self,
+                 calendar: str='24/7',
+                 *args, **kwargs
+    ):
+        """
+        Args:
+            calendar (str): Trading calendar. The string should must be valid calendar from the pandas_market_calendars (like '24/7' or 'NYSE')
+        """
+        self._calendar = calendar
+
+    @property
+    def calendar(self):
+        """ Workaround due to error when pickleing the calendar """
+        return mcal.get_calendar(self._calendar)
+
+    def get_explanation(self):
+        info = f"""
+        <p>{self.__class__.__name__}: Rebalance strategy class.</p>
+        """
+        return info
+
+    def calculate_rebalance_dates(
+            self,
+            start: datetime.datetime,
+            end: datetime.datetime,
+            calendar,
+            rebalance_frequency_strategy: str
+    ) -> pd.DatetimeIndex:
+        """
+        Determines the dates on which portfolio rebalancing should be executed based on the specified rebalancing strategy.
+        This calculation takes into account the start time of the rebalancing window and the execution frequency.
+
+        Args:
+            start (pd.DataFrame): A datetime containing the start time
+
+        Returns:
+            pd.DatetimeIndex: A DatetimeIndex containing all the dates when rebalancing should occur.
+        """
+        # to account for the time during the day at which the execution starts
+        if end is None:
+            raise NotImplementedError("end_date cannot be None")
+
+        if rebalance_frequency_strategy == "daily":
+            early = calendar.schedule(start_date=start.date(), end_date=end.date())
+            rebalance_dates = early.set_index("market_open")
+            rebalance_dates = rebalance_dates.index
+        elif rebalance_frequency_strategy == "EOQ":
+            # carefull to use dates from the same calendar
+            raise NotImplementedError
+        else:
+            raise NotImplementedError(f"Strategy {rebalance_frequency_strategy} not implemented")
+
+        return rebalance_dates
+
+REBALANCE_CLASS_REGISTRY = REBALANCE_CLASS_REGISTRY if 'REBALANCE_CLASS_REGISTRY' in globals() else {}
+def register_rebalance_class(name=None, register_in_agent=True):
+    """
+    Decorator to register a model class in the factory.
+    If `name` is not provided, the class's name is used as the key.
+    """
+    def decorator(cls):
+        return insert_in_registry(REBALANCE_CLASS_REGISTRY, cls, register_in_agent, name)
+    return decorator
+
+class RebalanceFactory(BaseFactory):
+
+    @staticmethod
+    def get_rebalance_strategy(rebalance_strategy_name: str):
+        if rebalance_strategy_name not in REBALANCE_CLASS_REGISTRY:
+            RebalanceFactory.get_rebalance_strategies()
+        try:
+            return REBALANCE_CLASS_REGISTRY[rebalance_strategy_name]
+        except KeyError:
+            logger.exception(f"{rebalance_strategy_name} is not registered in this project")
+
+    @staticmethod
+    def get_rebalance_strategies():
+        import mainsequence.virtualfundbuilder.contrib.rebalance_strategies # get default strategies
+        try:
+            RebalanceFactory.import_module("rebalance_strategies")
+        except FileNotFoundError:
+            logger.info("rebalance_strategies folder no present no strategies to import")
+        return REBALANCE_CLASS_REGISTRY
+
+
+
+
+

mainsequence/virtualfundbuilder/resource_factory/signal_factory.py

@@ -0,0 +1,183 @@
+import ast
+import inspect
+
+from mainsequence.tdag.data_nodes import DataNode
+from datetime import datetime, timedelta
+import numpy as np
+import pytz
+
+from mainsequence.virtualfundbuilder.enums import ResourceType
+from mainsequence.virtualfundbuilder.resource_factory.base_factory import BaseResource, BaseFactory, insert_in_registry
+from mainsequence.virtualfundbuilder.models import AssetsConfiguration
+
+import pandas as pd
+from mainsequence.client import (Asset)
+from mainsequence.virtualfundbuilder.utils import get_vfb_logger
+
+logger = get_vfb_logger()
+
+
+class WeightsBase(BaseResource):
+    TYPE = ResourceType.SIGNAL_WEIGHTS_STRATEGY
+
+    def __init__(self,
+                 signal_assets_configuration: AssetsConfiguration,
+                 *args, **kwargs):
+        """
+        Base Class for all signal weights
+
+        Attributes:
+            assets_configuration (AssetsConfiguration): Configuration details for signal assets.
+        """
+        self.assets_configuration = signal_assets_configuration
+        super().__init__()
+
+    def get_explanation(self):
+        info = f"""
+        <p>{self.__class__.__name__}: Signal weights class.</p>
+        """
+        return info
+
+    def maximum_forward_fill(self) -> timedelta:
+        raise NotImplementedError
+
+    def get_asset_uid_to_override_portfolio_price(self):
+
+        return None
+
+    def interpolate_index(self, new_index: pd.DatetimeIndex):
+        """
+        Get interpolated weights for a time index. Weights are only valid for a certain time, therefore forward fill is limited.
+        Especially needed for gaps within the weights
+        """
+        # get values between new index
+        try:
+            weights = self.get_df_between_dates(start_date=new_index.min(), end_date=new_index.max())
+        except Exception as e:
+            raise e
+        
+        # if we need more data before to interpolate first value of new_index
+        if len(weights) == 0: # or (weights.index.get_level_values("time_index").min() > new_index.min()):
+
+            unique_identifier_range_map = {a: {"start_date": d} for a, d in self.update_statistics.asset_time_statistics.items()}
+            last_observation = self.get_df_between_dates(unique_identifier_range_map=unique_identifier_range_map)
+            if last_observation is None or last_observation.empty:
+                return pd.DataFrame()
+            last_date = last_observation.index.get_level_values("time_index")[0]
+
+            if last_date < new_index.min():
+                self.logger.warning(f"No weights data at start of the portfolio at {new_index.min()}"
+                                    f" will use last available weights {last_date}")
+                weights = self.get_df_between_dates(start_date=last_date, end_date=new_index.max())
+
+        if len(weights) == 0 :
+            self.logger.warning(f"No weights data in index interpolation")
+            return pd.DataFrame()
+
+
+
+        weights_pivot = weights.reset_index().pivot(
+            index="time_index",
+            columns=[ "unique_identifier"],
+            values="signal_weight").fillna(0)
+        weights_pivot["last_weights"] = weights_pivot.index.get_level_values(level="time_index")
+
+        # combine existing index with new index
+        combined_index = weights_pivot.index.union(new_index)
+        combined_index.name = "time_index"
+        weights_reindex = weights_pivot.reindex(combined_index)
+
+        # check which dates are outside of valid forward filling range
+        weights_reindex["last_weights"] = weights_reindex["last_weights"].ffill()
+        weights_reindex["diff_to_last_weights"] = weights_reindex.index.get_level_values(level="time_index") - \
+                                                  weights_reindex["last_weights"]
+
+        invalid_forward_fills = weights_reindex["diff_to_last_weights"] >= self.maximum_forward_fill()  # source_frequency is the duration a weight is valid
+        weights_reindex.drop(columns=["last_weights", "diff_to_last_weights"], inplace=True)
+
+        # forward fill and set dates that are outside of valid range to nan
+        weights_reindex = weights_reindex.ffill()
+        weights_reindex[invalid_forward_fills] = np.nan
+
+        if weights_reindex.isna().values.any():
+            self.logger.info(f"Could not fully interpolate for signal weights")
+
+        weights_reindex = weights_reindex.loc[new_index]
+        weights_reindex.index.name = "time_index"
+
+
+
+        return weights_reindex
+
+def _get_class_source_code(cls):
+    import ast
+    import inspect
+    import sys
+
+    try:
+        # Get the source code of the module where the class is defined
+        module = sys.modules[cls.__module__]
+        source = inspect.getsource(module)
+    except Exception as e:
+        logger.warning(f"Could not get source code for module {cls.__module__}: {e}")
+        return None
+
+    # Parse the module's source code
+    try:
+        module_ast = ast.parse(source)
+        class_source_code = None
+
+        # Iterate through the module's body to find the class definition
+        for node in module_ast.body:
+            if isinstance(node, ast.ClassDef) and node.name == cls.__name__:
+                # Get the lines corresponding to the class definition
+                lines = source.splitlines()
+                # Get the lines for the class definition
+                class_source_lines = lines[node.lineno - 1: node.end_lineno]
+                class_source_code = '\n'.join(class_source_lines)
+                break
+
+        if not class_source_code:
+            logger.warning(f"Class definition for {cls.__name__} not found in module {cls.__module__}")
+            return None
+
+        return class_source_code
+
+    except Exception as e:
+        logger.warning(f"Could not parse source code for module {cls.__module__}: {e}")
+        return None
+
+
+SIGNAL_CLASS_REGISTRY = SIGNAL_CLASS_REGISTRY if 'SIGNAL_CLASS_REGISTRY' in globals() else {}
+def register_signal_class(name=None, register_in_agent=True):
+    """
+    Decorator to register a model class in the factory.
+    If `name` is not provided, the class's name is used as the key.
+    """
+    def decorator(cls):
+        code = inspect.getsource(cls)
+        attributes = {"code": code}
+        return insert_in_registry(SIGNAL_CLASS_REGISTRY, cls, register_in_agent, attributes=attributes)
+    return decorator
+
+
+class SignalWeightsFactory(BaseFactory):
+    @staticmethod
+    def get_signal_weights_strategy(signal_weights_name) -> DataNode:
+        """
+        Creates an instance of the appropriate SignalWeights class based on the provided name.
+        """
+        if signal_weights_name not in SIGNAL_CLASS_REGISTRY:
+            SignalWeightsFactory.get_signal_weights_strategies()
+
+        return SIGNAL_CLASS_REGISTRY[signal_weights_name]
+
+    @staticmethod
+    def get_signal_weights_strategies():
+        """
+        Scans the given directory for Python files, imports the classes,
+        and returns all classes that are subclasses of WeightsBase.
+        """
+        import mainsequence.virtualfundbuilder.contrib.data_nodes # get default strategies
+        SignalWeightsFactory.import_module("data_nodes")
+        return SIGNAL_CLASS_REGISTRY