sinapsis-data-analysis 0.1.0__py3-none-any.whl

sinapsis_data_analysis/helpers/excluded_models.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+excluded_linear_models = [
+    "enet_path",
+    "lars_path",
+    "lars_path_gram",
+    "lasso_path",
+    "orthogonal_mp",
+    "orthogonal_mp_gram",
+    "ridge_regression",
+    "_sgd_fast",
+]
+excluded_neighbors_models = [
+    "_ball_tree.BallTree",
+    "_ball_tree.KDTree",
+    "sort_graph_by_row_values",
+    "kneighbors_graph",
+    "radius_neighbors_graph",
+    "sort_graph_by_row_values",
+    "VALID_METRICS",
+    "VALID_METRICS_SPARSE",
+]
+excluded_tree_models = ["plot_tree", "export_text", "export_graphviz"]

sinapsis_data_analysis/helpers/model_metrics.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict
+
+
+class ModelMetrics(BaseModel):
+    """Base class for storing model metrics"""
+
+    accuracy: float | None = None
+    precision: float | None = None
+    recall: float | None = None
+    f1_score: float | None = None
+
+    r2_score: float | None = None
+    mean_squared_error: float | None = None
+    mean_absolute_error: float | None = None
+
+    mape: float | None = None
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class ModelPredictionResults(BaseModel):
+    """Class to store model predictions and metrics"""
+
+    predictions: Any
+    metrics: ModelMetrics
+    model_config = ConfigDict(arbitrary_types_allowed=True)

sinapsis_data_analysis/templates/__init__.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+import importlib
+
+from sinapsis.templates import _import_template_package
+from sinapsis_core.template_base import Template
+
+_root_lib_path: str = "sinapsis_data_analysis.templates"
+
+_ADDITIONAL_TEMPLATE_MODULES = [
+    f"{_root_lib_path}.sklearn_manifold",
+    f"{_root_lib_path}.sklearn_train",
+    f"{_root_lib_path}.xgboost_train",
+    f"{_root_lib_path}.xgboost_inference",
+]
+_template_lookup: dict = {
+    "SKLearnInference": f"{_root_lib_path}.sklearn_inference",
+    "XGBoostInference": f"{_root_lib_path}.xgboost_inference",
+}
+for t_module in _ADDITIONAL_TEMPLATE_MODULES:
+    _template_lookup |= _import_template_package(t_module)
+
+
+def __getattr__(name: str) -> Template:
+    if name in _template_lookup:
+        module = importlib.import_module(_template_lookup[name])
+        return getattr(module, name)
+
+
+__all__ = list(_template_lookup.keys())

sinapsis_data_analysis/templates/ml_base_inference.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+from abc import abstractmethod
+from typing import Any
+
+import numpy as np
+from sinapsis_core.data_containers.data_packet import DataContainer
+from sinapsis_core.template_base import TemplateAttributes
+from sinapsis_core.template_base.template import Template
+
+
+class MLBaseInference(Template):
+    """Abstract base class for machine learning model inference.
+
+    This class provides a framework for loading a trained model
+    and using it to make predictions on new data.
+    """
+
+    class AttributesBaseModel(TemplateAttributes):
+        """Attributes for the MLBaseInference template.
+
+        Attributes:
+            model_path (str): Path to the saved model file.
+            generic_field_key (str): Key of the generic field where data is stored.
+        """
+
+        model_path: str
+        generic_field_key: str
+
+    def __init__(self, attributes: TemplateAttributes) -> None:
+        super().__init__(attributes)
+        self.model = self.load_model(self.attributes.model_path)
+
+    def get_data(self, container: DataContainer) -> Any:
+        """Get the data from the data container
+
+        Args:
+            container (DataContainer): The data container with the data
+
+        Returns:
+            Any: The data from the generic field
+        """
+        return self._get_generic_data(container, self.attributes.generic_field_key)
+
+    @staticmethod
+    def data_is_valid(data: Any) -> bool:
+        """Check if the data is valid for inference
+
+        Args:
+            data (Any): The data to validate
+
+        Returns:
+            bool: True if the data is valid, False otherwise
+        """
+        return data is not None
+
+    def preprocess_data(self, data: Any) -> Any:
+        """
+        This method can be overridden by subclasses to implement
+        specific preprocessing steps
+
+        Args:
+            data (Any): The data to preprocess
+
+        Returns:
+            Any: The preprocessed data
+        """
+        try:
+            data.pop("target")
+        except (KeyError, IndexError):
+            self.logger.info("No target column")
+        return data
+
+    @abstractmethod
+    def load_model(self, model_path: str) -> Any:
+        """
+        This abstract method should be implemented by subclasses to define
+        how the model should be loaded
+
+        Args:
+            model_path (str): Path to the saved model file
+
+        Returns:
+            Any: The loaded model
+        """
+
+    def predict(self, data: np.ndarray) -> np.ndarray:
+        """Generate predictions using the loaded model
+
+        Args:
+            data (Any): The data to make predictions on
+
+        Returns:
+            np.ndarray: The model's predictions
+        """
+
+        return self.model.predict(data)
+
+    def execute(self, container: DataContainer) -> DataContainer:
+        """
+        Gets the data, validates it, preprocesses it, makes predictions,
+        and stores the results
+
+        Args:
+            container (DataContainer): The data container with the input data
+
+        Returns:
+            DataContainer: The container with added predictions
+        """
+        data = self.get_data(container)
+
+        if not self.data_is_valid(data):
+            self.logger.warning("Invalid or missing data")
+            return container
+
+        data = self.preprocess_data(data)
+        predictions = self.predict(data)
+
+        self._set_generic_data(container, predictions)
+
+        return container

sinapsis_data_analysis/templates/ml_base_training.py

@@ -0,0 +1,261 @@
+# -*- coding: utf-8 -*-
+import os
+from abc import abstractmethod
+from typing import Any
+
+import numpy as np
+from sinapsis_core.data_containers.data_packet import DataContainer
+from sinapsis_core.template_base import TemplateAttributes
+from sinapsis_core.template_base.dynamic_template import BaseDynamicWrapperTemplate
+from sinapsis_data_readers.templates.datasets_readers.dataset_splitter import TabularDatasetSplit
+from sklearn.base import is_classifier, is_regressor
+from sklearn.metrics import (
+    accuracy_score,
+    f1_score,
+    mean_absolute_error,
+    mean_squared_error,
+    precision_score,
+    r2_score,
+    recall_score,
+)
+
+from sinapsis_data_analysis.helpers.model_metrics import (
+    ModelMetrics,
+    ModelPredictionResults,
+)
+
+
+class MLBaseAttributes(TemplateAttributes):
+    """Base attributes for machine learning model templates.
+
+    Attributes:
+        generic_field_key (str): Key of the generic field where datasets are stored.
+        model_save_path (str): Path where the trained model will be saved.
+    """
+
+    generic_field_key: str
+    model_save_path: str
+
+
+class MLBaseTraining(BaseDynamicWrapperTemplate):
+    """
+    This abstract class provides common functionality for loading data,
+    training models, making predictions, calculating metrics, and saving
+    models.
+
+
+    """
+
+    AttributesBaseModel = MLBaseAttributes
+
+    def __init__(self, attributes: TemplateAttributes) -> None:
+        """Initialize the MLBase template.
+
+        Args:
+            attributes (TemplateAttributes): The attributes for this template.
+        """
+        super().__init__(attributes)
+        self.model = self.wrapped_callable
+        self.trained_model = None
+
+    def get_dataset(self, container: DataContainer) -> Any:
+        """Get the dataset from the data container.
+
+        Args:
+            container (DataContainer): The data container with the dataset.
+
+        Returns:
+            Any: The dataset from the generic field.
+        """
+        return self._get_generic_data(container, self.attributes.generic_field_key)
+
+    @staticmethod
+    def dataset_is_valid(dataset: Any) -> bool:
+        """Check if the dataset is valid
+
+        Args:
+            dataset (Any): The dataset to validate.
+
+        Returns:
+            bool: True if the dataset is valid, False otherwise.
+        """
+        return dataset is not None
+
+    def process_dataset(self, dataset: TabularDatasetSplit) -> tuple | None:
+        """
+        Extracts x_train, y_train, x_test, y_test from the dataset
+
+        Args:
+            dataset (Any): The dataset to process
+
+        Returns:
+            tuple | None: A tuple containing (x_train, y_train, x_test, y_test)
+                or None if the dataset doesn't have the expected attributes
+        """
+        try:
+            x_train = dataset.x_train
+            y_train = dataset.y_train
+            x_test = dataset.x_test
+            y_test = dataset.y_test
+
+            return x_train, y_train, x_test, y_test
+        except AttributeError:
+            self.logger.warning("Dataset doesn't have the expected attributes")
+            return None
+
+    def train_model(self, x_train: Any, y_train: Any) -> None:
+        """Train the model using the training data
+
+        Args:
+            x_train (Any): The training features
+            y_train (Any): The training targets
+        """
+        self.trained_model = self.model.fit(x_train, y_train)
+
+    @staticmethod
+    def calculate_classification_metrics(y_true: np.ndarray, y_pred: np.ndarray) -> ModelMetrics:
+        """Calculate metrics specific to classification models
+
+        Args:
+            y_true (np.ndarray): The ground truth labels
+            y_pred (np.ndarray): The predicted labels
+
+        Returns:
+            ModelMetrics: Object containing classification metrics
+        """
+        metrics = ModelMetrics()
+        metrics.accuracy = float(accuracy_score(y_true, y_pred))
+        metrics.precision = float(precision_score(y_true, y_pred, average="weighted", zero_division=0))
+        metrics.recall = float(recall_score(y_true, y_pred, average="weighted", zero_division=0))
+        metrics.f1_score = float(f1_score(y_true, y_pred, average="weighted", zero_division=0))
+        return metrics
+
+    @staticmethod
+    def calculate_regression_metrics(y_true: np.ndarray, y_pred: np.ndarray) -> ModelMetrics:
+        """Calculate metrics specific to regression models
+
+        Args:
+            y_true (np.ndarray): The ground truth values
+            y_pred (np.ndarray): The predicted values
+
+        Returns:
+            ModelMetrics: Object containing regression metrics
+        """
+        metrics = ModelMetrics()
+        metrics.r2_score = float(r2_score(y_true, y_pred))
+        metrics.mean_squared_error = float(mean_squared_error(y_true, y_pred))
+        metrics.mean_absolute_error = float(mean_absolute_error(y_true, y_pred))
+
+        return metrics
+
+    def calculate_metrics(self, y_true: np.ndarray, y_pred: np.ndarray) -> ModelMetrics:
+        """
+        Detects whether the model is a classifier or regressor and calculates
+        the appropriate metrics
+
+        Args:
+            y_true (np.ndarray): The ground truth values/labels
+            y_pred (np.ndarray): The predicted values/labels
+
+        Returns:
+            ModelMetrics: Object containing the appropriate metrics
+        """
+        if self.trained_model is not None:
+            if is_classifier(self.trained_model):
+                return self.calculate_classification_metrics(y_true, y_pred)
+            elif is_regressor(self.trained_model):
+                return self.calculate_regression_metrics(y_true, y_pred)
+        return ModelMetrics()
+
+    def generate_predictions(self, x_test: np.ndarray, y_test: np.ndarray) -> ModelPredictionResults | None:
+        """
+        Uses the trained model to make predictions on the test data
+        and calculates the appropriate metrics
+
+        Args:
+            x_test (np.ndarray): The test features
+            y_test (np.ndarray): The test targets
+
+        Returns:
+            ModelPredictionResults: Object containing predictions and metrics
+        """
+        if self.trained_model is not None:
+            predictions = self.trained_model.predict(x_test)
+
+            metrics = self.calculate_metrics(y_test, predictions)
+
+            return ModelPredictionResults(predictions=predictions, metrics=metrics)
+        return None
+
+    def handle_model_training(self, processed_data: tuple) -> ModelPredictionResults | None:
+        """Handle the model training and prediction workflow.
+
+        Extracts data from the processed dataset, trains the model,
+        and generates predictions.
+
+        Args:
+            processed_data (tuple): Tuple containing training and testing data.
+
+        Returns:
+            ModelPredictionResults: Object containing predictions and metrics.
+        """
+        x_train, y_train, x_test, y_test = processed_data
+
+        self.train_model(x_train, y_train)
+        return self.generate_predictions(x_test, y_test)
+
+    def save_model(self) -> None:
+        """
+        Creates the necessary directories and calls the implementation-specific
+        method to save the model to the path specified in attributes.
+        If no trained model exists or an error occurs, it will be logged.
+        """
+        if self.trained_model is None:
+            self.logger.error("No model to save")
+            return
+
+        try:
+            os.makedirs(os.path.dirname(self.attributes.model_save_path), exist_ok=True)
+            self._save_model_implementation()
+            self.logger.info(f"Model saved at {self.attributes.model_save_path}")
+        except (MemoryError, TypeError) as e:
+            self.logger.error(f"Error saving model: {e}")
+
+    @abstractmethod
+    def _save_model_implementation(self) -> None:
+        """Save the trained model using an implementation-specific method.
+
+        This abstract method should be implemented by subclasses to define
+        how the model should be serialized and saved to disk.
+        """
+
+    def execute(self, container: DataContainer) -> DataContainer:
+        """
+        Gets the dataset, validates it, processes it, trains the model,
+        generates predictions, and stores the results
+
+        Args:
+            container (DataContainer): The data container with the dataset
+
+        Returns:
+            DataContainer: The container with added predictions and metrics
+        """
+        dataset = self.get_dataset(container)
+
+        if not self.dataset_is_valid(dataset):
+            self.logger.warning("Invalid or missing dataset")
+            return container
+
+        processed_data = self.process_dataset(dataset)
+
+        if processed_data is None:
+            self.logger.warning("Failed to process dataset")
+            return container
+
+        results = self.handle_model_training(processed_data)
+
+        if results is not None:
+            self._set_generic_data(container, results)
+            self.save_model()
+
+        return container

sinapsis_data_analysis/templates/sklearn_inference.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+from typing import Any
+
+import joblib
+
+from sinapsis_data_analysis.templates.ml_base_inference import MLBaseInference
+
+
+class SKLearnInference(MLBaseInference):
+    """Template for inference using sklearn models.
+
+    This template loads a saved sklearn model using joblib
+    and uses it to make predictions on new data.
+    """
+
+    CATEGORY = "SKLearn"
+
+    def load_model(self, model_path: str) -> Any:
+        """
+        Uses joblib to load a previously saved sklearn model
+
+        Args:
+            model_path (str): Path to the saved model file
+
+        Returns:
+            Any: The loaded sklearn model
+        """
+        return joblib.load(model_path)

sinapsis_data_analysis/templates/sklearn_manifold.py

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+from typing import cast
+
+import numpy as np
+import pandas as pd
+from pydantic import BaseModel, ConfigDict
+from sinapsis_core.data_containers.data_packet import DataContainer
+from sinapsis_core.template_base import Template, TemplateAttributes
+from sinapsis_core.template_base.dynamic_template import (
+    BaseDynamicWrapperTemplate,
+    WrapperEntryConfig,
+)
+from sinapsis_core.template_base.dynamic_template_factory import make_dynamic_template
+from sinapsis_core.utils.env_var_keys import SINAPSIS_BUILD_DOCS
+from sinapsis_data_readers.templates.datasets_readers.dataset_splitter import (
+    TabularDatasetSplit,
+)
+from sklearn import manifold
+
+
+class ManifoldResults(BaseModel):
+    """Class to store the results of manifold learning.
+
+    Attributes:
+        labels (np.ndarray | list): The original labels.
+        x_transformed (np.ndarray): The data after dimensionality reduction.
+    """
+
+    labels: np.ndarray | list
+    x_transformed: np.ndarray
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class SKLearnManifold(BaseDynamicWrapperTemplate):
+    """
+    This template dynamically wraps sklearn's manifold module,
+    providing access to dimensionality reduction techniques like
+    TSNE, MDS, Isomap, etc.
+    """
+
+    WrapperEntry = WrapperEntryConfig(
+        wrapped_object=manifold,
+        signature_from_doc_string=True,
+        exclude_module_atts=["locally_linear_embedding"],
+        force_init_as_method=False,
+    )
+
+    CATEGORY = "SKLearn"
+
+    class AttributesBaseModel(TemplateAttributes):
+        """Attributes for the SKLearnManifold template.
+
+        Attributes:
+            generic_field_key (str): Key of the generic field
+                where the input data is stored.
+        """
+
+        generic_field_key: str
+
+    def __init__(self, attributes: TemplateAttributes) -> None:
+        super().__init__(attributes)
+        self.manifold_model = self.wrapped_callable
+
+    @staticmethod
+    def reshape_arrays(feature_arrays: pd.DataFrame) -> np.ndarray:
+        """
+        Converts a list of arrays into a 2D numpy array suitable for
+        manifold learning algorithms
+
+        Args:
+            feature_arrays (list): List of feature arrays
+
+        Returns:
+            np.ndarray: Reshaped array suitable for manifold learning
+        """
+        array_data = np.array(feature_arrays)
+        return array_data.reshape(array_data.shape[0], -1)
+
+    def get_dataset(self, container: DataContainer) -> TabularDatasetSplit | None:
+        """Get the dataset from the data container
+
+        Args:
+            container (DataContainer): The data container with the dataset
+
+        Returns:
+            TabularDatasetSplit | None: The dataset from the generic field,
+                or None if not found
+        """
+        dataset = self._get_generic_data(container, self.attributes.generic_field_key)
+        dataset = cast(TabularDatasetSplit, dataset)
+        if dataset:
+            return dataset
+        return None
+
+    def process_dataset(self, dataset: TabularDatasetSplit) -> ManifoldResults | None:
+        """
+        Extracts the training data, reshapes it, and applies the
+        manifold learning transformation
+
+        Args:
+            dataset (TabularDatasetSplit): The dataset to process
+
+        Returns:
+            ManifoldResults | None: Results of the manifold transformation,
+                or None if the dataset is empty
+        """
+        x_train = dataset.x_train
+        y_train = dataset.y_train
+
+        if x_train is None or x_train.empty:
+            return None
+
+        x_train_reshaped = self.reshape_arrays(x_train)
+        x_transformed = self.manifold_model.fit_transform(x_train_reshaped)
+
+        return ManifoldResults(labels=y_train, x_transformed=x_transformed)
+
+    def execute(self, container: DataContainer) -> DataContainer:
+        """
+        Gets the dataset, processes it using the manifold learning algorithm,
+        and stores the results
+
+        Args:
+            container (DataContainer): The data container with the dataset
+
+        Returns:
+            DataContainer: The container with added manifold learning results
+        """
+        dataset = self.get_dataset(container)
+        if not dataset:
+            self.logger.warning("There is no dataset to process")
+            return container
+
+        results = self.process_dataset(dataset)
+
+        if results is not None:
+            self._set_generic_data(container, results)
+
+        return container
+
+
+def __getattr__(name: str) -> Template:
+    """
+    Only create a template if it's imported, this avoids creating all the base models for all templates
+    and potential import errors due to not available packages.
+    """
+    if name in SKLearnManifold.WrapperEntry.module_att_names:
+        return make_dynamic_template(name, SKLearnManifold)
+    raise AttributeError(f"template `{name}` not found in {__name__}")
+
+
+__all__ = SKLearnManifold.WrapperEntry.module_att_names
+
+
+if SINAPSIS_BUILD_DOCS:
+    dynamic_templates = [__getattr__(template_name) for template_name in __all__]
+    for template in dynamic_templates:
+        globals()[template.__name__] = template
+        del template