PyIAML 1.0.0__py3-none-any.whl

iaml/__init__.py

@@ -0,0 +1,56 @@
+"""Integrated AutoML for Medical Labs (IAML).
+
+IAML helps clinical research teams build, evaluate and inspect machine learning
+pipelines for tabular classification, regression and survival analysis. Modular
+preprocessing and modeling steps support automated search, while pipeline
+descriptions, evaluation metrics and SHAP explanations support review of the
+resulting models and reporting of research methods.
+"""
+from .iaml import IAML
+from .core_dispatcher import CoreDispatcher
+from .step import *
+from .metastep import MetaStep
+from .actionable import Actionable
+from .candidate import Candidate
+from .dataset import Dataset
+from .data_type import DataType
+from .metric_plot import MetricPlot
+from .metric import Metric
+from .plot import Plot, StatisticPlot
+from .statistic import Statistic
+from .cache import Cache
+from .void_step import VoidStep
+
+from .meta_ordered_step import MetaOrderedStep
+from .meta_explorer_step import MetaExplorerStep
+from .meta_partial_explorer_step import MetaPartialExplorerStep
+
+# Default Actionables
+from .actionables import *
+
+# Wrappers
+from .wrapper import *
+
+# Metrics
+from .metrics import *
+
+# Statistics
+from .statistics import *
+
+# Plots
+from .plots import *
+
+# Stack
+from .stack import Stack
+
+# Optimizer
+from .optimizers import *
+
+# Type of target
+from .type_of_target import type_of_target
+
+# Ensure star imports expose all public names, even if __all__ is set elsewhere.
+__all__ = [
+    name for name in globals()
+    if not name.startswith("_") and name != "__all__"
+]

iaml/actionable.py

@@ -0,0 +1,11 @@
+"""Classic kind of Step that transform, resample or predict from Candidate"""
+
+# -> Must be a wildcard import to help IAML to know all available the steps
+from .step import * # pylint: disable=unused-wildcard-import,wildcard-import
+from .decorators.all import is_step
+
+
+@is_step('actionable')
+class Actionable(Step):
+    """Classic kind of Step that transform, resample or predict from Candidate"""
+    _usage = "Use when you need a concrete transform/resample/predict step over a Candidate. Applicable to pipeline steps that operate directly on Candidate data. Avoid when you need meta orchestration like MetaStep or wrappers like StepWrapper."

iaml/actionables/__init__.py

@@ -0,0 +1,21 @@
+"""
+All Step with transform, resample or predict function
+"""
+from . import (
+    boosting,
+    predictors,
+    features_selection,
+    cleaning,
+    normalize,
+    features_precleaning,
+    features_preprocessing,
+    imbalance,
+)
+from .boosting import *
+from .predictors import *
+from .features_selection import *
+from .cleaning import *
+from .normalize import *
+from .features_precleaning import *
+from .features_preprocessing import *
+from .imbalance import *

iaml/actionables/boosting/__init__.py

@@ -0,0 +1,4 @@
+"""
+All boosting Actionables
+"""
+from .act_adaboost import ActAdaBoost

iaml/actionables/boosting/act_adaboost.py

@@ -0,0 +1,59 @@
+"""Apply AdaBoost on models"""
+from typing import Any
+from sklearn.ensemble import AdaBoostClassifier
+
+from ...actionable import Actionable
+from ...decorators.all import runner
+from ...candidate import Candidate
+
+
+class ActAdaBoost(Actionable):
+    """Apply Adaboost on models
+
+    Configuration:
+        * `random_state`: Random seed (defaults to 42).
+        * `n_estimator`: Number of estimators (defaults to 2000).
+    """
+
+    name: str = "AdaBoost Classifier"
+    refs: list[dict[str, Any]] = [
+        {
+            'year': 1995,
+            'name': (
+                'A desicion-theoretic generalization of on-line learning'
+                'and an application to boosting'
+            ),
+            'authors': [
+                'Yoav Freund',
+                'Robert E. Schapire'
+            ],
+            'doi': 'https://doi.org/10.1007/3-540-59119-2_166',
+            'publisher': 'Springer, Berlin, Heidelberg'
+        }
+    ]
+
+    def __init__(self):
+        self.configuration = {
+            'random_state': {
+                'description': 'random_state',
+                'default': 42
+            },
+            'n_estimator': {
+                'description': 'Number of estimators',
+                'default': 2000
+            },
+        }
+
+    @runner
+    def run(self, candidate: Candidate) -> Candidate:
+        model = AdaBoostClassifier(
+            candidate.model,
+            n_estimators=self.get_config('n_estimator'),
+            random_state=self.get_config('random_state'))
+
+        model.fit(candidate.dataset.X, candidate.dataset.y)
+
+        return candidate.to_output(None, None, model)
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        return 0.5

iaml/actionables/cleaning/__init__.py

@@ -0,0 +1,26 @@
+"""
+All cleaning actionables
+"""
+from .act_mean_column import ActMeanColumn
+from .act_drop_numerical_column import ActDropNumericalColumn
+from .act_drop_textual_column import ActDropTextualColumn
+from .act_drop_categorical_column import ActDropCategoricalColumn
+from .act_onehot import ActOnehot
+from .act_tf_idf import ActTfIdf
+from .act_drop_date_column import ActDropDateColumn
+from .act_split_date import ActSplitDate
+from .act_word2vec import ActWord2Vec
+from .act_mice import ActMICEForestImputer
+from .act_simple_imputer import ActSimpleImputer
+from .act_knn_imputer import ActKNNImputer
+from .act_categorical_imputer import ActCategoricalImputer
+from .act_rare_category_grouper import ActRareCategoryGrouper
+from .act_frequency_encoder import ActFrequencyEncoder
+from .act_target_encoder import ActTargetEncoder
+from .act_ordinal_encoder import ActOrdinalEncoder
+from .act_count_vectorizer import ActCountVectorizer
+from .act_hashing_vectorizer import ActHashingVectorizer
+from .act_text_normalizer import ActTextNormalizer
+from .act_missing_indicator import ActMissingIndicator
+from .act_missing_count_feature import ActMissingCountFeature
+from .act_drop_high_cardinality_categorical import ActDropHighCardinalityCategorical

iaml/actionables/cleaning/act_categorical_imputer.py

@@ -0,0 +1,124 @@
+"""[STEP] Impute missing categorical values."""
+import textwrap
+from typing import Any
+
+import pandas as pd
+
+from ...actionable import Actionable
+from ...candidate import Candidate
+from ...data_type import DataType
+from ...dataset import Dataset
+from ...decorators.all import is_step
+
+
+@is_step('cleaning')
+class ActCategoricalImputer(Actionable):
+    """[STEP] Impute missing categorical values."""
+
+    name: str = 'Impute missing categorical values'
+    _usage: str = 'Use when categorical columns have missing labels you want to keep (vs ActDropCategoricalColumn). Applicable to categorical/category dtype features with NA gaps. Avoid when missingness is extreme or cardinality is high; consider ActDropHighCardinalityCategorical.'
+    _description: str = textwrap.dedent('''\
+        Impute missing categorical values using the {strategy} strategy.''')
+    _description_long: str = textwrap.dedent('''\
+        Replace missing values in categorical columns with either the most frequent
+        observed category or a constant "missing" label. The label can be customized
+        via missing_label and is also used as a fallback when no mode can be computed.''')
+
+    def __init__(self) -> None:
+        self.columns: list[str] = []
+        self.fill_values: dict[str, Any] = {}
+        self._missing_stats: dict[str, tuple[int, int, float]] = {}
+
+        self.configuration = {
+            'strategy': {
+                'description': 'Imputation strategy for categorical columns.',
+                'default': 'most_frequent',
+                'categorical': ['most_frequent', 'missing']
+            },
+            'missing_label': {
+                'description': 'Label used when strategy="missing" or when no mode exists.',
+                'default': 'missing'
+            }
+        }
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns = self._select_columns(dataset)
+        self.fill_values = {}
+        self._missing_stats = {}
+        self.explanations = []
+
+        if not self.columns or dataset.X.empty:
+            return self
+
+        X_cat = dataset.X[self.columns]
+        total_rows = len(X_cat)
+        if total_rows == 0:
+            return self
+
+        missing_counts = X_cat.isna().sum()
+        strategy = self.get_config('strategy')
+        missing_label = self.get_config('missing_label')
+
+        for column in self.columns:
+            series = X_cat[column]
+            missing = int(missing_counts[column])
+            pct = (missing / total_rows * 100.0) if total_rows else 0.0
+            self._missing_stats[column] = (missing, total_rows, pct)
+
+            if strategy == 'missing':
+                fill_value = missing_label
+            else:
+                mode_series = series.mode(dropna=True)
+                if not mode_series.empty:
+                    fill_value = mode_series.iloc[0]
+                else:
+                    fill_value = missing_label
+
+            self.fill_values[column] = fill_value
+            if missing > 0:
+                self.explanations.append(
+                    f"Filled {missing} missing values in `{column}` with {fill_value!r}."
+                )
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        if not self.columns or not self.fill_values:
+            return X
+
+        for column, fill_value in self.fill_values.items():
+            if column not in X.columns:
+                continue
+            if pd.api.types.is_categorical_dtype(X[column]):
+                if fill_value not in X[column].cat.categories:
+                    X[column] = X[column].cat.add_categories([fill_value])
+            X[column] = X[column].fillna(fill_value)
+
+        return X
+
+    def suitable(self, dataset: Dataset) -> bool:
+        columns = self._select_columns(dataset)
+        if not columns or dataset.X.empty:
+            return False
+        return bool(dataset.X[columns].isna().any().any())
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        if candidate is None or candidate.dataset.X.empty:
+            return 0.0
+        columns = self._select_columns(candidate.dataset)
+        if not columns:
+            return 0.0
+        missing = candidate.dataset.X[columns].isna().sum().sum()
+        total = candidate.dataset.X[columns].size or 1
+        return min(1.0, missing / total)
+
+    def _select_columns(self, dataset: Dataset) -> list[str]:
+        columns = dataset.get_columns_names_by_type(DataType.CATEGORICAL)
+        category_columns = list(dataset.X.select_dtypes(include=['category']).columns)
+        seen = set()
+        ordered = []
+        for column in columns + category_columns:
+            if column in dataset.X.columns and column not in seen:
+                ordered.append(column)
+                seen.add(column)
+        return ordered

iaml/actionables/cleaning/act_count_vectorizer.py

@@ -0,0 +1,204 @@
+"""[STEP] Vectorize short text with CountVectorizer."""
+import textwrap
+from typing import Any
+
+import pandas as pd
+from sklearn.feature_extraction.text import CountVectorizer
+
+from ...actionable import Actionable
+from ...candidate import Candidate
+from ...data_type import DataType
+from ...dataset import Dataset
+from ...decorators.all import is_step
+
+
+@is_step('cleaning')
+class ActCountVectorizer(Actionable):
+    """[STEP] Vectorize short text with CountVectorizer."""
+
+    name: str = 'Count Vectorizer'
+    _usage: str = 'Use when short text is predictive and you want bag-of-words counts instead of ActDropTextualColumn. Applicable to short text columns with a manageable vocabulary. Avoid when text is long, extremely sparse, or you would drop text entirely (ActDropTextualColumn).'
+    _description: str = textwrap.dedent('''\
+        Vectorize short text columns into bag-of-words counts with n-grams.''')
+    _description_long: str = textwrap.dedent('''\
+        Build a vocabulary on each short text column and replace it with count features
+        for each token or n-gram observed in the training data. Adjust the n-gram range
+        and vocabulary size to control sparsity and keep runtime manageable.''')
+
+    def __init__(self) -> None:
+        self.columns: list[tuple[str, CountVectorizer]] = []
+        self.configuration = {
+            'ngram_min': {
+                'description': 'Minimum n-gram size to include.',
+                'default': 1
+            },
+            'ngram_max': {
+                'description': 'Maximum n-gram size to include.',
+                'default': 2
+            },
+            'max_features': {
+                'description': 'Maximum size of the vocabulary (None keeps all).',
+                'default': 2000
+            },
+            'min_df': {
+                'description': 'Minimum document frequency for a term to be kept.',
+                'default': 1
+            },
+            'max_df': {
+                'description': 'Maximum document frequency for a term to be kept.',
+                'default': 1.0
+            }
+        }
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns = []
+        self.explanations = []
+
+        columns = dataset.get_columns_names_by_type([DataType.SHORT_TEXT])
+        if not columns or dataset.X.empty:
+            return self
+
+        params = self._build_vectorizer_params(len(dataset.X))
+
+        for column in columns:
+            values = dataset.X[column].fillna('').astype(str)
+            vectorizer = CountVectorizer(**params)
+            try:
+                vectorizer.fit(values)
+            except ValueError:
+                continue
+
+            feature_names = vectorizer.get_feature_names_out()
+            if len(feature_names) == 0:
+                continue
+
+            self.columns.append((column, vectorizer))
+            self.explanations.append(
+                f'Encoded text column **`{column}`** into **{len(feature_names)}** count features.'
+            )
+
+        if not self.columns and columns:
+            raise RuntimeError(
+                "Count vectorizer failed: no usable vocabulary in short text columns."
+            )
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """Apply CountVectorizer to short text columns.
+
+        :param pd.DataFrame X: DataFrame to transform.
+        :return: Transformed dataset.
+        """
+        if not self.columns:
+            return X
+
+        X = X.reset_index(drop=True)
+
+        for name, vectorizer in self.columns:
+            if name not in X.columns:
+                continue
+
+            values = X[name].fillna('').astype(str)
+            transformed = vectorizer.transform(values)
+            feature_names = vectorizer.get_feature_names_out()
+            if len(feature_names) == 0:
+                X = X.drop([name], axis=1)
+                continue
+
+            new_names = [f"{name}_{token}" for token in feature_names]
+            vector_df = pd.DataFrame(transformed.toarray(), columns=new_names)
+
+            X = pd.concat([X, vector_df], axis=1).drop([name], axis=1)
+
+        return X
+
+    def suitable(self, dataset: Dataset) -> bool:
+        if dataset.X.empty:
+            return False
+        return bool(dataset.get_columns_names_by_type([DataType.SHORT_TEXT]))
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        if candidate is None or candidate.dataset.X.empty:
+            return 0.0
+
+        columns = candidate.dataset.get_columns_names_by_type([DataType.SHORT_TEXT])
+        if not columns:
+            return 0.0
+
+        total_columns = candidate.dataset.X.shape[1] or 1
+        return min(1.0, len(columns) / total_columns)
+
+    def _build_vectorizer_params(self, n_rows: int) -> dict[str, Any]:
+        ngram_min = self._coerce_int(self.get_config('ngram_min'), 1)
+        ngram_max = self._coerce_int(self.get_config('ngram_max'), max(ngram_min, 1))
+        ngram_min = max(1, ngram_min)
+        ngram_max = max(ngram_min, ngram_max)
+
+        min_df = self._coerce_df(self.get_config('min_df'), 1)
+        max_df = self._coerce_df(self.get_config('max_df'), 1.0)
+
+        if n_rows > 0:
+            min_df = self._clamp_df(min_df, n_rows)
+            max_df = self._clamp_df(max_df, n_rows)
+            if self._effective_df(min_df, n_rows) > self._effective_df(max_df, n_rows):
+                max_df = min_df
+
+        max_features = self._coerce_optional_int(self.get_config('max_features'))
+
+        params = {
+            'ngram_range': (ngram_min, ngram_max),
+            'min_df': min_df,
+            'max_df': max_df
+        }
+        if max_features is not None:
+            params['max_features'] = max_features
+
+        return params
+
+    @staticmethod
+    def _coerce_int(value: Any, default: int) -> int:
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            return default
+
+    @staticmethod
+    def _coerce_optional_int(value: Any) -> int | None:
+        if value is None:
+            return None
+        try:
+            numeric = int(value)
+        except (TypeError, ValueError):
+            return None
+        if numeric <= 0:
+            return None
+        return numeric
+
+    @staticmethod
+    def _coerce_df(value: Any, default: float | int) -> float | int:
+        if value is None or isinstance(value, bool):
+            return default
+        if isinstance(value, int):
+            return max(0, value)
+        try:
+            numeric = float(value)
+        except (TypeError, ValueError):
+            return default
+        if numeric < 0:
+            return default
+        if numeric <= 1.0:
+            return float(numeric)
+        return int(round(numeric))
+
+    @staticmethod
+    def _clamp_df(value: float | int, n_rows: int) -> float | int:
+        if isinstance(value, float):
+            return min(max(value, 0.0), 1.0)
+        return min(max(value, 0), n_rows)
+
+    @staticmethod
+    def _effective_df(value: float | int, n_rows: int) -> float:
+        if isinstance(value, float):
+            return value * n_rows
+        return float(value)

iaml/actionables/cleaning/act_drop_categorical_column.py

@@ -0,0 +1,51 @@
+"""[STEP] Drop categorical columns"""
+import textwrap
+import pandas as pd
+from ...actionable import Actionable
+from ...dataset import Dataset
+from ...data_type import DataType
+from ...candidate import Candidate
+from ...decorators.all import is_step
+
+
+@is_step('cleaning', 'baseline_cleaning')
+class ActDropCategoricalColumn(Actionable):
+    """[STEP] Drop categorical columns"""
+
+    name: str = 'Remove categorical columns'
+    _description: str = 'Remove all columns containing categorical data from the dataset'
+    _usage: str = 'Use when categorical columns must be removed for steps that cannot handle categories. Applicable to datasets with categorical or category-typed columns. Avoid when you can impute or encode categories instead (ActCategoricalImputer, ActCountVectorizer).'
+    _description_long: str = textwrap.dedent('''\
+        Remove all columns containing categorical data from the dataset.
+        This step is used to clean the dataset in order to perform other actions later on
+        that can't be applied to categorical columns.''')
+    can_be_disabled: bool = False
+
+    def __init__(self):
+        self.columns_to_drop: list[str] = None
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns_to_drop = list(set(
+            dataset.get_columns_names_by_type([DataType.CATEGORICAL]) + \
+            list(dataset.X.select_dtypes(include=['category']).columns)
+        ))
+
+        self.explanations = [
+            f'Dropped column **`{c}`**.' for c in self.columns_to_drop
+        ]
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """Drop columns.
+
+        :param pd.DataFrame X: DataFrame to transform.
+        :return: Transformed DataFrame.
+        """
+        return X.drop(self.columns_to_drop, axis=1)
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        return 0
+
+    def suitable(self, dataset: Dataset) -> bool:
+        return bool(dataset.get_columns_names_by_type([DataType.CATEGORICAL]))

iaml/actionables/cleaning/act_drop_date_column.py

@@ -0,0 +1,48 @@
+"""[STEP] Find and drop date column"""
+import textwrap
+import pandas as pd
+from ...actionable import Actionable
+from ...data_type import DataType
+from ...candidate import Candidate
+from ...dataset import Dataset
+from ...decorators.all import is_step
+
+
+@is_step('cleaning', 'baseline_cleaning')
+class ActDropDateColumn(Actionable):
+    """Finds and drops date columns."""
+
+    name: str = 'Remove date columns'
+    _description: str = 'Remove all columns containing Date from the dataset'
+    _usage: str = 'Use when date columns are irrelevant and you want to remove them; for other types use ActDropNumericalColumn or ActDropCategoricalColumn. Applicable to columns typed as DataType.DATE. Avoid when dates carry predictive signal or need feature extraction.'
+    _description_long: str = textwrap.dedent('''\
+        Remove all columns containing Data from the dataset
+        This step is used to clean the dataset in order to perform other actions later on
+        that can't be applied to date columns.''')
+    can_be_disabled: bool = False
+
+    def __init__(self):
+        self.columns_to_drop: list[str] = None
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns_to_drop = dataset.get_columns_names_by_type(DataType.DATE)
+
+        self.explanations = [
+            f'Dropped column **`{c}`**.' for c in self.columns_to_drop
+        ]
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """Drop columns.
+
+        :param pd.DataFrame X: DataFrame to transform.
+        :return: Transformed DataFrame.
+        """
+        return X.drop(self.columns_to_drop, axis=1)
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        return 0
+
+    def suitable(self, dataset: Dataset) -> bool:
+        return bool(dataset.get_columns_names_by_type(DataType.DATE))