PyIAML 1.0.0__py3-none-any.whl

iaml/plots/violin_plot.py

@@ -0,0 +1,206 @@
+"""[PLOT] Violin plot for descriptive statistics."""
+from __future__ import annotations
+
+import io
+import textwrap
+from typing import Any
+
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from ..data_type import DataType
+from ..dataset import Dataset
+from ..plot import StatisticPlot, capture
+
+
+def _is_missing(value: Any) -> bool:
+    if value is None:
+        return True
+    if isinstance(value, float) and pd.isna(value):
+        return True
+    return False
+
+
+def _column_label(column: str, base_name: str | None) -> str:
+    column_str = str(column)
+    base_name_str = str(base_name) if base_name is not None else None
+    if base_name_str:
+        if column_str == base_name_str:
+            return 'all'
+        prefix = f"{base_name_str}_"
+        if column_str.startswith(prefix):
+            return column_str[len(prefix):]
+    if column_str.endswith('_all'):
+        return 'all'
+    return column_str
+
+
+def _plot_placeholder(message: str) -> None:
+    plt.figure()
+    plt.text(0.5, 0.5, message, ha='center', va='center')
+    plt.axis('off')
+
+
+def _extract_violin_groups(
+    value: Any,
+) -> list[tuple[str, np.ndarray, np.ndarray, np.ndarray | None]]:
+    if _is_missing(value) or not isinstance(value, dict):
+        return []
+    entries: list[tuple[str, np.ndarray, np.ndarray, np.ndarray | None]] = []
+    for category, stats in value.items():
+        if not isinstance(stats, dict):
+            continue
+        density = stats.get('density')
+        support = stats.get('support')
+        if density is None or support is None:
+            continue
+        try:
+            density_arr = np.asarray(density, dtype=float).ravel()
+            support_arr = np.asarray(support, dtype=float).ravel()
+        except (TypeError, ValueError):
+            continue
+        if density_arr.size == 0 or support_arr.size == 0:
+            continue
+        if density_arr.size != support_arr.size:
+            continue
+        mask = np.isfinite(density_arr) & np.isfinite(support_arr)
+        if not np.any(mask):
+            continue
+        density_arr = density_arr[mask]
+        support_arr = support_arr[mask]
+        order = np.argsort(support_arr)
+        density_arr = density_arr[order]
+        support_arr = support_arr[order]
+
+        quartiles = None
+        if 'quartiles' in stats:
+            try:
+                q_values = np.asarray(stats['quartiles'], dtype=float).ravel()
+            except (TypeError, ValueError):
+                q_values = np.asarray([], dtype=float)
+            if q_values.size >= 3 and np.all(np.isfinite(q_values[:3])):
+                quartiles = q_values[:3]
+        entries.append((str(category), support_arr, density_arr, quartiles))
+    return entries
+
+
+def _render_violin(
+    ax: plt.Axes,
+    groups: list[tuple[str, np.ndarray, np.ndarray, np.ndarray | None]],
+) -> None:
+    if not groups:
+        ax.text(0.5, 0.5, "No violin data", ha='center', va='center')
+        ax.axis('off')
+        return
+    max_density = max(float(np.nanmax(density)) for _, _, density, _ in groups)
+    if not np.isfinite(max_density) or max_density <= 0:
+        ax.text(0.5, 0.5, "Invalid density data", ha='center', va='center')
+        ax.axis('off')
+        return
+
+    scale = 0.4 / max_density
+    for idx, (category, support, density, quartiles) in enumerate(groups):
+        width = density * scale
+        ax.fill_betweenx(
+            support,
+            idx - width,
+            idx + width,
+            alpha=0.6,
+            edgecolor='black',
+            linewidth=0.8,
+        )
+        if quartiles is not None:
+            q1, median, q3 = quartiles
+            ax.plot([idx, idx], [q1, q3], color='black', linewidth=2)
+            ax.plot([idx - 0.08, idx + 0.08], [median, median], color='black', linewidth=2)
+
+    ax.set_xticks(range(len(groups)))
+    ax.set_xticklabels([label for label, *_ in groups], rotation=30, ha='right')
+    ax.set_xlabel('Category')
+    ax.set_ylabel('Target')
+
+
+class ViolinPlot(StatisticPlot):
+    """[PLOT] Violin Plot."""
+
+    name: str = "Violin Plot"
+    _description: str = textwrap.dedent("""\
+        Violin plots show target distributions per categorical value.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        This plot renders violin distributions for categorical features using
+        precomputed density statistics for a continuous target.
+        """)
+    refs: list[dict] = []
+
+    title: str = "Violin plot"
+    description: str = textwrap.dedent("""\
+        The violin plot shows target distributions per categorical feature.
+        """)
+    group_by_feature: bool = True
+
+    def __str__(self) -> str:
+        return 'violin'
+
+    @capture
+    def compute(
+        self,
+        dataframe: pd.DataFrame,
+        base_name: str | None = None,
+        dataset: Dataset | None = None,
+        **kwargs,
+    ) -> 'ViolinPlot':
+        """Compute violin plot statistics."""
+        self._binary_image = io.BytesIO()
+
+        if dataframe.empty:
+            _plot_placeholder("No statistics available")
+            plt.savefig(self._binary_image, format='png')
+            return self
+
+        violin_key = str(self)
+        if violin_key not in dataframe.index:
+            _plot_placeholder("Violin statistics not available")
+            plt.savefig(self._binary_image, format='png')
+            return self
+
+        if dataset is not None:
+            categorical_columns = set(dataset.get_columns_names_by_type(DataType.CATEGORICAL))
+            columns_to_show = [col for col in dataframe.columns if col in categorical_columns]
+        else:
+            columns_to_show = list(dataframe.columns)
+
+        violin_row = dataframe.loc[violin_key]
+        entries: list[tuple[str, list[tuple[str, np.ndarray, np.ndarray, np.ndarray | None]]]] = []
+        for col in columns_to_show:
+            groups = _extract_violin_groups(violin_row.get(col))
+            if groups:
+                entries.append((col, groups))
+
+        if not entries:
+            _plot_placeholder("No violin statistics available")
+            plt.savefig(self._binary_image, format='png')
+            return self
+
+        n_plots = len(entries)
+        n_cols = 1 if n_plots == 1 else 2
+        n_rows = int(np.ceil(n_plots / n_cols))
+        fig, axes = plt.subplots(n_rows, n_cols, figsize=(6 * n_cols, 4 * n_rows))
+        axes_list = np.atleast_1d(axes).ravel()
+
+        for ax, (col, groups) in zip(axes_list, entries):
+            _render_violin(ax, groups)
+            ax.set_title(_column_label(col, base_name))
+
+        for ax in axes_list[len(entries):]:
+            ax.axis('off')
+
+        if base_name:
+            fig.suptitle(f"Violin plot: {base_name}")
+            plt.tight_layout(rect=(0, 0, 1, 0.95))
+        else:
+            plt.tight_layout()
+
+        plt.savefig(self._binary_image, format='png')
+        return self

iaml/predictor.py

@@ -0,0 +1,139 @@
+"""Last step of a pipeline -> can make prediction"""
+from abc import ABCMeta, abstractmethod
+from typing import Any
+import dataclasses
+import pandas as pd
+from sklearn.base import BaseEstimator
+from .actionable import Actionable
+from .decorators.runner import runner
+from .candidate import Candidate
+
+
+@dataclasses.dataclass
+class Model(metaclass=ABCMeta):
+    """Model type (use for typing purposes only)."""
+
+    @abstractmethod
+    def predict(self, X: Any, *args, **kw) -> Any:
+        """Any predict method implemented by most ML frameworks.
+        
+        :param Any X: The dataset to predict 
+        :param tuple, optional \\*args: Additional parameters.
+        :param tuple, optional \\**kwargs: Additional parameters.
+        :return: Prediction output.
+        """
+
+class Predictor(Actionable, BaseEstimator, metaclass=ABCMeta):
+    """[STEP] Abstract learning step
+    
+    Also acts as an interface with traditional scikit-learn models
+    for better integration with IAMLPipeline.
+    """
+
+    model: Model
+
+    def __init__(self):
+        super().__init__()
+        self.optimizable: bool = True
+        self.model: Model = None
+
+    @runner
+    def run(self, candidate: Candidate) -> Candidate:
+        """Run the step. In "Run" stage, predict does not "fit". Only add himself to pipeline
+
+        :param Candidate candidate: Candidate informations 
+        :return: transformed Candidate 
+        """
+        return candidate.add_to_pipeline(self)
+
+    def predict_proba(self, X: pd.DataFrame) -> list[float]:
+        """Apply prediction model on DataFrame with probability
+
+        :param pd.DataFrame X: DataFrame use to predict
+        :raise AttributeError: Unable to predict probabilities with this model
+        :return: Predicted values
+        """
+        if self.model and hasattr(self.model, 'predict_proba'):
+            return self.model.predict_proba(X)
+
+        raise AttributeError("Unable to predict probabilities with this model")
+
+    def __getattribute__(self, attr: str) -> bool:
+        """Overload getattr to allow accurate hasattr on predict_proba
+
+        :param str attr: Attribute to test.
+        :raise AttributeError: predict_proba not implemented in this model.
+        :return: Is attribute implemented ?
+        """
+        if attr == 'predict_proba' \
+            and not( \
+                self.model and hasattr(self.model, 'predict_proba') \
+            ):
+            raise AttributeError("predict_proba not implemented in this model")
+
+        return super().__getattribute__(attr)
+
+    def predict(self, X: pd.DataFrame) -> list[float]:
+        """Apply prediction model on DataFrame
+
+        :param pd.DataFrame X: DataFrame use to predict.
+        :return: Predicted values.
+        """
+        if self.model and hasattr(self.model, 'predict'):
+            results = self.model.predict(X)
+            if hasattr(self, 'label_encoder'):
+                return self.label_encoder.inverse_transform(results)
+            return results
+        return None
+
+    def predict_survival_function(self, X: pd.DataFrame) -> list[list[float]]:
+        """Apply prediction survival function model on DataFrame
+
+        :param pd.DataFrame X: DataFrame use to predict.
+        :raise AttributeError: Unable to predict survival function with this model.
+        :return: Predicted values.
+        """
+        if self.model and hasattr(self.model, 'predict_survival_function'):
+            return self.model.predict_survival_function(X)
+
+        raise AttributeError("Unable to predict survival function with this model")
+
+    def predict_cumulative_hazard_function(self, X: pd.DataFrame) -> list[list[float]]:
+        """Apply prediction survival function model on DataFrame
+
+        :param pd.DataFrame X: DataFrame use to predict.
+        :raise AttributeError: Unable to predict survival function with this model.
+        :return: Predicted values.
+        """
+        if self.model and hasattr(self.model, 'predict_cumulative_hazard_function'):
+            return self.model.predict_cumulative_hazard_function(X)
+
+        raise AttributeError("Unable to predict cumulative hazard function with this model")
+
+    @property
+    def classes_(self) -> list:
+        """Return classes of the target in fit data"""
+        return self.model.classes_
+
+    def score(self, *args, **kwargs) -> Any:
+        """Mimic Scikitlearn API
+        
+        :return: The model score
+        """
+        return self.model.score(*args, **kwargs)
+
+    def get_params(self, *args, **kwargs) -> Any | None:
+        """Mimic Scikitlearn API
+        
+        :return: Model parameters or None if no parameters
+        """
+        if self.model and hasattr(self.model, 'get_params'):
+            return self.model.get_params(*args, **kwargs)
+        return None
+
+    def __name__(self) -> str:
+        """Return the predictor formatted name
+        
+        :return: formatted name
+        """
+        return ' '.join(x.title() for x in str(self).split('_'))

iaml/reference.py

@@ -0,0 +1,65 @@
+"""
+Reference class.
+Contain all needed data to provide a reference for a step
+"""
+from typing import List, Dict
+
+
+class Reference:  # pylint: disable=too-few-public-methods
+    """Reference class.
+    Contain all needed data to provide a reference for a step
+    
+    :param Dict properties: a dictionnary of properties for a reference object
+    :param str step_name: The step_name attached to this reference
+    """
+
+    def __init__(self, properties: Dict, step_name: str) -> None:
+        """Instantiate all properties provided to the specific reference
+        such as year of publication, authors, doi ...
+        """
+        setattr(self, 'step', step_name)
+        for k, v in properties.items():
+            setattr(self, k, v)
+
+    def __str__(self) -> str:
+        """Return a simple string containing reference information
+        
+        :return: The reference representation
+        """
+        structured = ''
+        try:
+            structured = structured + ', '.join(self.authors) + '. '
+        except AttributeError:
+            pass
+        try:
+            structured = structured + str(self.name) + '\n'
+        except AttributeError:
+            pass
+        try:
+            structured = structured + str(self.publisher) + ', '
+        except AttributeError:
+            pass
+        try:
+            structured = structured + str(self.doi) + ', '
+        except AttributeError:
+            pass
+        try:
+            structured = structured + str(self.year) +'.'
+        except AttributeError:
+            pass
+        return structured
+
+    @classmethod
+    def bibliography(cls, references: List['Reference'], structured: bool) -> str | List[Dict]:
+        """Format a bibliography in a string from a list of references
+
+        :param List[Reference] references: List of References
+        :param bool structured: Wether we want a string bibliography or a list of references
+
+        :return: Bibliography in string or List format
+        """
+        if structured:
+            return [vars(r) for r in references]
+        spacing = len(str(len(references)))
+        return '\n'.join([f"[{i+1:>{spacing}}]  {str(reference)}\n" \
+            for i, reference in enumerate(references)])

iaml/shared_cache.py

@@ -0,0 +1,90 @@
+from collections import deque
+from copy import deepcopy
+from typing import Any, Tuple
+import multiprocess.managers
+from .logger import Logger
+
+
+class CacheService:
+    """Cache partagé, LRU bornée, clé=(fingerprint, df_hash)."""
+    def __init__(self, max_cache_size: int = 100) -> None:
+        self._saved = None
+        self._lru = None
+        self._max = max_cache_size
+        self._disabled = None
+        self._lock = None
+
+    def __set_backend__(self, saved, lru, disabled, lock, maxsize: int):
+        self._saved = saved
+        self._lru = lru
+        self._disabled = disabled
+        self._lock = lock
+        self._max = maxsize
+
+    # API
+    def disable(self) -> None:
+        with self._lock:
+            self._disabled.value = True
+
+    def enable(self) -> None:
+        with self._lock:
+            self._disabled.value = False
+
+    def get(self, fingerprint: str, df_hash: str) -> Any | None:
+        if self._disabled.value:
+            return None
+        key = (fingerprint, df_hash)
+            
+        with self._lock:
+            if key in self._saved:
+                try:
+                    self._lru.remove(key)
+                except ValueError:
+                    pass
+                self._lru.append(key)
+                return deepcopy(self._saved[key])
+        return None
+
+    def put(self, fingerprint: str, df_hash: str, output: Any) -> None:
+        if self._disabled.value:
+            return
+        key = (fingerprint, df_hash)
+        
+        with self._lock:
+            self._saved[key] = deepcopy(output)
+            try:
+                self._lru.remove(key)
+            except ValueError:
+                pass
+            self._lru.append(key)
+            # Éviction LRU
+            while len(self._lru) > self._max:
+                old_key = self._lru.pop(0)
+                self._saved.pop(old_key, None)
+
+
+class CacheManager(multiprocess.managers.BaseManager):
+    pass
+
+def start_cache_manager(max_cache_size: int = 100) -> tuple[CacheManager, CacheService]:
+    """
+    Démarre un process manager et retourne (manager, cache_proxy).
+    À appeler UNE FOIS dans le process parent AVANT de lancer les workers.
+    """
+    def _cache_factory():
+        from multiprocess.managers import SyncManager
+        sm = SyncManager()
+        sm.start()
+        saved = sm.dict()
+        lru = sm.list()
+        disabled = sm.Value('b', False)
+        lock = sm.RLock()
+        cache = CacheService(max_cache_size)
+        cache.__set_backend__(saved, lru, disabled, lock, max_cache_size)
+        return cache
+
+    CacheManager.register('Cache', callable=_cache_factory)
+    mgr = CacheManager()
+    mgr.start()
+    cache: CacheService = mgr.Cache()
+    return mgr, cache

iaml/sklearn_preprocessor.py

@@ -0,0 +1,74 @@
+"""A mandatory sklearn input transformer, fitted inside each candidate pipeline."""
+from __future__ import annotations
+
+from typing import Any
+
+from joblib import hash as joblib_hash
+import numpy as np
+import pandas as pd
+from sklearn.base import clone
+from sklearn.utils.validation import check_is_fitted
+
+from .dataset import Dataset
+from .step import Step
+
+
+class SklearnPreprocessor(Step):
+    """Adapt an unsupervised sklearn transformer to IAML's fit/transform protocol.
+
+    The caller's transformer is a template only. A fresh sklearn clone is fitted
+    on each supplied Dataset.X, without target values or patient-group columns.
+    ``transformer_`` is the fitted clone, retained for prediction/provenance and
+    serialization. The template and its hyperparameters participate in cache
+    keys; the learned vocabulary does not alter the pipeline configuration.
+
+    A DataFrame output is required so column names and row alignment remain
+    explicit. This step is deliberately not registered under a search tag: it
+    can only enter the search through IAML's explicit initial_preprocessor.
+    """
+
+    name = "Initial sklearn preprocessing"
+    can_be_disabled = False
+
+    def __init__(self, transformer: Any):
+        super().__init__()
+        if not callable(getattr(transformer, "fit", None)) or not callable(getattr(transformer, "transform", None)):
+            raise TypeError("initial_preprocessor must implement sklearn fit and transform")
+        self.transformer = clone(transformer)
+        self.configuration = {
+            "transformer_class": {"default": f"{type(transformer).__module__}.{type(transformer).__qualname__}"},
+            "transformer_parameters_hash": {"default": joblib_hash(self.transformer.get_params(deep=True))},
+        }
+        self.default_configuration()
+        self.is_interchangeable = False
+        self.optimizable = False
+
+    def fit(self, dataset: Dataset) -> "SklearnPreprocessor":
+        # Never retain categories learned by the generation sample or an earlier
+        # fold. sklearn.clone also removes fitted state supplied by the caller.
+        self.__dict__.pop("transformer_", None)
+        fitted = clone(self.transformer)
+        fitted.fit(dataset.X.copy(deep=True))
+        self.transformer_ = fitted
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        check_is_fitted(self, "transformer_")
+        if not isinstance(X, pd.DataFrame):
+            raise TypeError("IAML preprocessing requires a DataFrame input")
+        result = self.transformer_.transform(X.copy(deep=True))
+        if not isinstance(result, pd.DataFrame):
+            raise TypeError("initial_preprocessor must return a numeric DataFrame")
+        if len(result) != len(X) or not result.index.equals(X.index):
+            raise ValueError("initial_preprocessor must preserve row count, order and index")
+        if not result.columns.is_unique or not len(result.columns):
+            raise ValueError("initial_preprocessor must return unique, nonempty feature columns")
+        if not all(pd.api.types.is_numeric_dtype(dtype) for dtype in result.dtypes):
+            raise TypeError("initial_preprocessor output columns must all be numeric")
+        # IAML regards bool columns as categorical. Keep the explicit numeric
+        # representation, including BooleanDtype missing values, unambiguous.
+        boolean_columns = result.select_dtypes(include=["bool", "boolean"]).columns
+        if len(boolean_columns):
+            result = result.copy()
+            result[boolean_columns] = result[boolean_columns].astype(np.float32)
+        return result

iaml/splitters/__init__.py

@@ -0,0 +1,3 @@
+"""All IAML splitters"""
+from .kfold_splitter import kfold_splitter
+from .random_splitter import random_splitter

iaml/splitters/kfold_splitter.py

@@ -0,0 +1,32 @@
+"""Allow to split a dataset into n folds to compute crossvalidation"""
+from typing import Iterator
+from sklearn.model_selection import KFold as SKKFold, StratifiedKFold
+from sklearn.model_selection import StratifiedGroupKFold
+from sklearn.model_selection import GroupKFold
+from ..dataset import Dataset
+
+
+def kfold_splitter(dataset: Dataset, nb_folds: int = 5) -> Iterator[tuple['Dataset', 'Dataset']]:
+    """Allow to split a dataset into n folds to compute crossvalidation
+
+    :param Dataset dataset: The dataset to split.
+    :param int, optional nb_folds: The number of folds to create. Default to 5.
+    :return: Iterator of tuples of train/test Dataset objects
+    """
+    kwargs = {}
+
+    if dataset.type_of_target in ['binary', 'multiclass']:
+        if dataset.has_groups:
+            kfold = StratifiedGroupKFold(nb_folds)
+            kwargs['groups'] = dataset.groups
+        else:
+            kfold = StratifiedKFold(nb_folds)
+    else:
+        if dataset.has_groups:
+            kfold = GroupKFold(nb_folds)
+            kwargs['groups'] = dataset.groups
+        else:
+            kfold = SKKFold(nb_folds)
+
+    for ds_train, ds_test in dataset.split(kfold.split, **kwargs):
+        yield (ds_train, ds_test)

iaml/splitters/random_splitter.py

@@ -0,0 +1,26 @@
+"""Allow to split randomly a dataset to train/test"""
+from typing import Iterator
+from sklearn.model_selection import ShuffleSplit, GroupShuffleSplit
+from ..dataset import Dataset
+
+
+def random_splitter(
+    dataset: Dataset,
+    ratio: float = 0.2,
+    random_state: int = 42) -> Iterator[tuple['Dataset', 'Dataset']]:
+    """Allow to split randomly a dataset to train/test
+    
+    :param Dataset dataset: The dataset to split.
+    :param float, optional ratio: The train/test ratio. Default to 0.2.
+    :param int, optional random_state: The random seed used. Default to 42.
+    :return: Iterator of tuples of train/test Dataset objects
+    """
+    kwargs = {}
+    if dataset.has_groups:
+        kwargs['groups'] = dataset.groups
+        splitter = GroupShuffleSplit(1, test_size=ratio, random_state=random_state)
+    else:
+        splitter = ShuffleSplit(1, test_size=ratio, random_state=random_state)
+
+    for ds_train, ds_test in dataset.split(splitter.split, **kwargs):
+        yield (ds_train, ds_test)

iaml/stack.py

@@ -0,0 +1,39 @@
+"""
+Frozen version of Step created to be stacked in an Candidate
+"""
+
+from typing import Dict
+from .step import Step
+class Stack:
+    """
+    Frozen version of Step created to be stacked in an Candidate
+    
+    :param Step step_class: The step class used to create this stack
+    :param Dict configuration: The configuration dictionnary
+    :param int step_id: The id of the Step
+    """
+    def __init__(self, step_class: Step, configuration: Dict, step_id: int) -> None:
+        """Initialize a stack
+        """
+        self.step_class: Step = step_class
+        """The step used in this Stack"""
+
+        self.configuration: dict = configuration
+        """The step configuration"""
+
+        self.step_id: int = step_id
+        """The step id"""
+
+    def __str__(self) -> str:
+        """Return the step name as Stack representation
+        
+        :return: The step name
+        """
+        return self.step_class.name
+
+    def explain(self) -> str:
+        """Return explanation string of Step
+        
+        :return: The explanation of the Step in string format
+        """
+        return self.step_class.explain(self.configuration)