PyIAML 1.0.0__py3-none-any.whl

iaml/actionables/cleaning/act_onehot.py

@@ -0,0 +1,65 @@
+"""[STEP] One hot encoding categorical features"""
+import textwrap
+import pandas as pd
+from sklearn.preprocessing import OneHotEncoder
+from ...actionable import Actionable
+from ...dataset import Dataset
+from ...data_type import DataType
+from ...candidate import Candidate
+from ...decorators.all import is_step
+
+
+@is_step('cleaning')
+class ActOnehot(Actionable):
+    """[STEP] One hot encoding categorical features"""
+
+    name: str = 'One hot encoding categorical features'
+    _description: str = 'Encode categorical data to numerical value using \
+        One Hot Encoding Algorithm'
+    _usage: str = 'Use when categorical columns have meaningful distinct values and you want indicator features, rather than ActDropCategoricalColumn. Applicable to nominal categorical data with low to moderate cardinality. Avoid when categories are very high-cardinality; consider ActDropHighCardinalityCategorical.'
+    _description_long: str = textwrap.dedent('''\
+        Retrieve all unique values from a column, then transform
+        those values to multiple binary columns.
+        Exemple: If a column "A" contain 3 uniques values like "coffee", "tea" and "water",
+        this step will create binary columns "A_coffee", "A_tea" and "A_water".''')
+
+    def __init__(self):
+        self.columns: list[str] = None
+        self.encoder: OneHotEncoder = None
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns = dataset.get_columns_names_by_type(DataType.CATEGORICAL)
+        values = dataset.X[self.columns]
+        self.encoder = OneHotEncoder(handle_unknown='ignore', sparse_output=False).fit(values)
+
+        # creates a dict with columns as keys and encoded categories as values
+        features = dict(zip(self.columns, self.encoder.categories_))
+
+        self.explanations = [
+            f'Encoded categorical column **`{c}`** into **{len(v)}** new columns.'
+            for c, v in features.items() if len(v) > 0
+        ]
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """Apply One Hot Encoding to dataframe.
+
+        :param pd.DataFrame x: DataFrame to transform.
+        :return: Transformed dataset.
+        """
+        # Without Reset index, the join with OHE will create NAN (index mismatch)
+        X = X.reset_index(drop=True)
+
+        transformed = self.encoder.transform(X[self.columns])
+        ohe_df = pd.DataFrame(transformed, columns=self.encoder.get_feature_names_out(self.columns))
+        X = X.drop(self.columns, axis=1)
+        df = X.join(ohe_df)
+
+        return df
+
+    def suitable(self, dataset: Dataset) -> bool:
+        return bool(dataset.get_columns_names_by_type(DataType.CATEGORICAL))
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        return 0.5

iaml/actionables/cleaning/act_ordinal_encoder.py

@@ -0,0 +1,177 @@
+"""[STEP] Ordinal encoding categorical features."""
+import textwrap
+from typing import Any
+
+import pandas as pd
+
+from ...actionable import Actionable
+from ...candidate import Candidate
+from ...data_type import DataType
+from ...dataset import Dataset
+from ...decorators.all import is_step
+
+
+@is_step('cleaning')
+class ActOrdinalEncoder(Actionable):
+    """[STEP] Ordinal encoding categorical features."""
+
+    name: str = 'Ordinal encoding'
+    _description: str = textwrap.dedent('''\
+        Encode categorical columns into integer codes with unknown handling.''')
+    _description_long: str = textwrap.dedent('''\
+        Replace categorical values with integer codes. Categories observed during training
+        are assigned consecutive integers starting at {start_value}. Missing or unseen
+        categories are encoded as {unknown_value}.''')
+    _usage: str = "Use when you need numeric codes for categorical features; ActCountVectorizer is for text-like categories. Applicable to low-cardinality categorical columns with stable labels. Avoid when categories are high-cardinality or should be dropped (ActDropHighCardinalityCategorical)."
+
+    def __init__(self) -> None:
+        self.columns: list[str] = []
+        self.categories: dict[str, list[Any]] = {}
+        self.unknown_value: int = -1
+        self.start_value: int = 0
+        self.sort_categories: bool = True
+
+        self.configuration = {
+            'unknown_value': {
+                'description': 'Value used for unseen or missing categories.',
+                'default': -1
+            },
+            'start_value': {
+                'description': 'Starting integer assigned to known categories.',
+                'default': 0
+            },
+            'sort_categories': {
+                'description': 'Sort categories before encoding for deterministic mapping.',
+                'default': True
+            }
+        }
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns = self._select_columns(dataset)
+        self.categories = {}
+        self.explanations = []
+
+        if not self.columns or dataset.X.empty:
+            return self
+
+        self.unknown_value = self._coerce_int(self.get_config('unknown_value'), -1)
+        self.start_value = self._coerce_int(self.get_config('start_value'), 0)
+        self.sort_categories = bool(self.get_config('sort_categories'))
+
+        for column in self.columns:
+            series = dataset.X[column]
+            categories = self._extract_categories(series, self.sort_categories)
+            self.categories[column] = categories
+
+            if categories:
+                self.explanations.append(
+                    f"Ordinal-encoded `{column}` with {len(categories)} categories."
+                )
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        if not self.columns or not self.categories:
+            return X
+
+        for column in self.columns:
+            if column not in X.columns:
+                continue
+            categories = self.categories.get(column, [])
+            X[column] = self._encode_series(
+                X[column],
+                categories,
+                self.start_value,
+                self.unknown_value
+            )
+
+        return X
+
+    def suitable(self, dataset: Dataset) -> bool:
+        columns = self._select_columns(dataset)
+        if not columns or dataset.X.empty:
+            return False
+        return True
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        if candidate is None or candidate.dataset.X.empty:
+            return 0.0
+
+        columns = self._select_columns(candidate.dataset)
+        if not columns:
+            return 0.0
+
+        total_rows = len(candidate.dataset.X)
+        if total_rows <= 0:
+            return 0.0
+
+        unique_counts = candidate.dataset.X[columns].nunique(dropna=True)
+        avg_cardinality = float((unique_counts / total_rows).mean())
+        low_cardinality = 1.0 - min(1.0, max(0.0, avg_cardinality))
+
+        total_columns = candidate.dataset.X.shape[1] or 1
+        cat_ratio = len(columns) / total_columns
+
+        return min(1.0, max(cat_ratio, low_cardinality))
+
+    def _select_columns(self, dataset: Dataset) -> list[str]:
+        columns = dataset.get_columns_names_by_type(DataType.CATEGORICAL)
+        category_columns = list(dataset.X.select_dtypes(include=['category']).columns)
+        seen = set()
+        ordered = []
+        for column in columns + category_columns:
+            if column in dataset.X.columns and column not in seen:
+                ordered.append(column)
+                seen.add(column)
+        return ordered
+
+    @staticmethod
+    def _coerce_int(value: Any, default: int) -> int:
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            return default
+
+    @classmethod
+    def _extract_categories(cls, series: pd.Series, sort_categories: bool) -> list[Any]:
+        if pd.api.types.is_categorical_dtype(series):
+            categories = list(series.cat.categories)
+        else:
+            categories = series.dropna().unique().tolist()
+
+        if sort_categories and categories:
+            categories = cls._safe_sorted(categories)
+
+        return categories
+
+    @staticmethod
+    def _safe_sorted(values: list[Any]) -> list[Any]:
+        try:
+            return sorted(values)
+        except TypeError:
+            return sorted(values, key=lambda item: str(item))
+
+    @staticmethod
+    def _encode_series(
+        series: pd.Series,
+        categories: list[Any],
+        start_value: int,
+        unknown_value: int
+    ) -> pd.Series:
+        if not categories:
+            return pd.Series(unknown_value, index=series.index, dtype='int64')
+
+        cat = pd.Categorical(series, categories=categories)
+        codes = pd.Series(cat.codes, index=series.index)
+        unknown_mask = codes.eq(-1)
+
+        if start_value:
+            codes = codes + start_value
+
+        if unknown_mask.any():
+            codes = codes.astype('int64', copy=False)
+            codes.loc[unknown_mask] = int(unknown_value)
+        else:
+            codes = codes.astype('int64', copy=False)
+
+        return codes

iaml/actionables/cleaning/act_rare_category_grouper.py

@@ -0,0 +1,173 @@
+"""[STEP] Group rare categories into a shared label."""
+import math
+import textwrap
+
+import pandas as pd
+
+from ...actionable import Actionable
+from ...candidate import Candidate
+from ...data_type import DataType
+from ...dataset import Dataset
+from ...decorators.all import is_step
+
+
+@is_step('cleaning')
+class ActRareCategoryGrouper(Actionable):
+    """[STEP] Group rare categories into a shared label."""
+
+    name: str = 'Group rare categories'
+    _description: str = textwrap.dedent('''\
+        Replace rare categorical values with an "other" label to limit sparsity.''')
+    _description_long: str = textwrap.dedent('''\
+        Categories whose frequency is below {min_frequency:.0%} of the non-missing values
+        or below {min_count} occurrences are grouped into {other_label!r}. This reduces
+        the number of distinct categories before downstream encoders are applied.''')
+    _usage: str = 'Use when rare labels in categorical features add sparsity and you want grouping instead of ActDropHighCardinalityCategorical. Applicable to categorical or pandas category columns before encoding. Avoid when rare labels are important or you prefer ActDropCategoricalColumn.'
+
+    def __init__(self) -> None:
+        self.configuration = {
+            'min_frequency': {
+                'description': textwrap.dedent('''\
+                    Minimum ratio of non-missing rows required to keep a category.'''),
+                'default': 0.01
+            },
+            'min_count': {
+                'description': 'Minimum absolute count required to keep a category.',
+                'default': 2
+            },
+            'other_label': {
+                'description': 'Label used to replace rare categories.',
+                'default': 'other'
+            }
+        }
+        self.columns: list[str] = []
+        self.rare_categories: dict[str, set] = {}
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns = self._select_columns(dataset)
+        self.rare_categories = {}
+        self.explanations = []
+
+        if not self.columns or dataset.X.empty:
+            return self
+
+        min_count = max(0, int(self.get_config('min_count')))
+        min_frequency = max(0.0, float(self.get_config('min_frequency')))
+        other_label = self.get_config('other_label')
+
+        for column in self.columns:
+            counts = dataset.X[column].value_counts(dropna=True)
+            total = int(counts.sum())
+            if total == 0 or counts.empty:
+                continue
+
+            threshold = self._threshold(min_count, min_frequency, total)
+            if threshold <= 0:
+                continue
+
+            rare_counts = counts[counts < threshold]
+            if rare_counts.empty:
+                continue
+
+            rare_labels = set(rare_counts.index.tolist())
+            self.rare_categories[column] = rare_labels
+            rare_total = int(rare_counts.sum())
+            self.explanations.append(
+                f"Grouped {len(rare_labels)} rare categories in `{column}` into "
+                f"{other_label!r} ({rare_total} rows)."
+            )
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        if not self.rare_categories:
+            return X
+
+        other_label = self.get_config('other_label')
+        for column, rare_values in self.rare_categories.items():
+            if column not in X.columns or not rare_values:
+                continue
+
+            series = X[column]
+            rare_mask = series.isin(rare_values)
+            if not rare_mask.any():
+                continue
+
+            if pd.api.types.is_categorical_dtype(series):
+                if other_label not in series.cat.categories:
+                    series = series.cat.add_categories([other_label])
+                series = series.where(~rare_mask, other_label)
+                X[column] = series
+            else:
+                X.loc[rare_mask, column] = other_label
+
+        return X
+
+    def suitable(self, dataset: Dataset) -> bool:
+        columns = self._select_columns(dataset)
+        if not columns or dataset.X.empty:
+            return False
+
+        min_count = max(0, int(self.get_config('min_count')))
+        min_frequency = max(0.0, float(self.get_config('min_frequency')))
+        if min_count <= 0 and min_frequency <= 0:
+            return False
+
+        for column in columns:
+            counts = dataset.X[column].value_counts(dropna=True)
+            total = int(counts.sum())
+            if total == 0 or counts.empty:
+                continue
+
+            threshold = self._threshold(min_count, min_frequency, total)
+            if threshold > 0 and (counts < threshold).any():
+                return True
+
+        return False
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        if candidate is None or candidate.dataset.X.empty:
+            return 0.0
+
+        columns = self._select_columns(candidate.dataset)
+        if not columns:
+            return 0.0
+
+        min_count = max(0, int(self.get_config('min_count')))
+        min_frequency = max(0.0, float(self.get_config('min_frequency')))
+
+        total_values = 0
+        rare_values = 0
+        for column in columns:
+            counts = candidate.dataset.X[column].value_counts(dropna=True)
+            total = int(counts.sum())
+            if total == 0 or counts.empty:
+                continue
+            threshold = self._threshold(min_count, min_frequency, total)
+            if threshold <= 0:
+                continue
+            rare_counts = counts[counts < threshold]
+            total_values += total
+            rare_values += int(rare_counts.sum())
+
+        if total_values == 0:
+            return 0.0
+
+        return min(1.0, max(0.0, rare_values / total_values))
+
+    def _threshold(self, min_count: int, min_frequency: float, total: int) -> int:
+        ratio_threshold = 0
+        if min_frequency > 0 and total > 0:
+            ratio_threshold = int(math.ceil(min_frequency * total))
+        return max(min_count, ratio_threshold)
+
+    def _select_columns(self, dataset: Dataset) -> list[str]:
+        columns = dataset.get_columns_names_by_type(DataType.CATEGORICAL)
+        category_columns = list(dataset.X.select_dtypes(include=['category']).columns)
+        seen = set()
+        ordered = []
+        for column in columns + category_columns:
+            if column in dataset.X.columns and column not in seen:
+                ordered.append(column)
+                seen.add(column)
+        return ordered

iaml/actionables/cleaning/act_simple_imputer.py

@@ -0,0 +1,109 @@
+"""[STEP] Simple imputer dedicated to minimalist pipelines."""
+import textwrap
+from typing import Any
+
+import pandas as pd
+
+from ...actionable import Actionable
+from ...dataset import Dataset
+from ...candidate import Candidate
+from ...decorators.all import is_step
+
+
+@is_step('minimal_preprocessing')
+class ActSimpleImputer(Actionable):
+    """[STEP] Lightweight imputer that fills NaNs with mean/mode values."""
+
+    name: str = 'Simple Imputer'
+    _usage: str = 'Use when NaNs must be filled quickly for minimal models; if dropping is preferred consider ActDropNumericalColumn. Applicable to mixed numeric and categorical tables with missing values. Avoid when missingness is informative or you plan richer imputation such as ActCategoricalImputer.'
+    _description: str = textwrap.dedent('''\
+        Fill missing numeric values with the column mean and categorical values with the
+        most frequent value so minimalist predictors can run without preprocessing.''')
+    _description_long: str = textwrap.dedent('''\
+        This step mirrors a classical SimpleImputer: each numeric feature is imputed with
+        its mean (fallback to 0 when undefined) while non-numeric columns are imputed with
+        their most frequent value. It is primarily used to make minimalist predictors work
+        on raw datasets that still contain NaNs.''')
+    can_be_disabled: bool = False
+
+    def __init__(self) -> None:
+        self.numeric_fill_values: dict[str, float] = {}
+        self.categorical_fill_values: dict[str, Any] = {}
+
+    def fit(self, dataset: Dataset) -> 'ActSimpleImputer':
+        X = dataset.X
+
+        self.numeric_fill_values = {}
+        self.categorical_fill_values = {}
+        self.explanations = []
+
+        numeric_cols = X.select_dtypes(include='number').columns
+        for column in numeric_cols:
+            series = X[column]
+            missing = series.isna().sum()
+
+            fill_value = series.mean()
+            if pd.isna(fill_value):
+                fill_value = 0.0
+
+            self.numeric_fill_values[column] = float(fill_value)
+            if missing > 0:
+                self.explanations.append(
+                    f"Filled {missing} numeric values in `{column}` with {fill_value:.4f}."
+                )
+
+        categorical_cols = X.select_dtypes(exclude='number').columns
+        for column in categorical_cols:
+            series = X[column]
+            missing = series.isna().sum()
+
+            mode_series = series.mode(dropna=True)
+            if not mode_series.empty:
+                fill_value = mode_series.iloc[0]
+            elif pd.api.types.is_bool_dtype(series):
+                fill_value = False
+            else:
+                fill_value = ''
+
+            self.categorical_fill_values[column] = fill_value
+            if missing > 0:
+                self.explanations.append(
+                    f"Filled {missing} categorical values in `{column}` with {fill_value!r}."
+                )
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        if self.numeric_fill_values:
+            for column, fill_value in self.numeric_fill_values.items():
+                if column in X.columns:
+                    X[column] = X[column].fillna(fill_value)
+
+        if self.categorical_fill_values:
+            for column, fill_value in self.categorical_fill_values.items():
+                if column in X.columns:
+                    X[column] = X[column].fillna(fill_value)
+
+        # As a last resort, replace any remaining NaNs to avoid downstream crashes.
+        if X.isna().any().any():
+            numeric_cols = X.select_dtypes(include='number').columns
+            if len(numeric_cols) > 0:
+                X[numeric_cols] = X[numeric_cols].fillna(0.0)
+            categorical_cols = X.select_dtypes(exclude='number').columns
+            if len(categorical_cols) > 0:
+                bool_cols = [c for c in categorical_cols if pd.api.types.is_bool_dtype(X[c])]
+                other_cols = [c for c in categorical_cols if c not in bool_cols]
+                if bool_cols:
+                    X[bool_cols] = X[bool_cols].fillna(False)
+                if other_cols:
+                    X[other_cols] = X[other_cols].fillna('')
+
+        return X
+
+    def priorize(self, candidate: Candidate = None) -> float:  # pylint: disable=unused-argument
+        if candidate is None or candidate.dataset.X.empty:
+            return 0.0
+
+        missing = candidate.dataset.X.isna().sum().sum()
+        total = candidate.dataset.X.size or 1
+        return min(1.0, missing / total)

iaml/actionables/cleaning/act_split_date.py

@@ -0,0 +1,68 @@
+"""[STEP] Transform string column to date"""
+import textwrap
+import pandas as pd
+import numpy as np
+from ...actionable import Actionable
+from ...dataset import Dataset
+from ...data_type import DataType
+from ...candidate import Candidate
+from ...decorators.all import is_step
+
+
+@is_step('cleaning')
+class ActSplitDate(Actionable):
+    """[STEP] Transform string column to date"""
+
+    name: str = 'Create Date Elements columns'
+    _usage: str = "Use when you want to expand a date column into components rather than ActDropDateColumn. Applicable to date-typed columns with usable timestamps. Avoid when dates are already split or you plan to drop them via ActDropDateColumn."
+    _descrption: str = textwrap.dedent('''\
+        Transform a textual date column into multiple columns
+        for day, month, year, hour, minute, second''')
+    _description_long: str = textwrap.dedent('''\
+        Transform a textual date column into multiple columns for 
+        day, month, year, hour, minute, second.
+        Exemple:
+        +-------------------+----------+------------+-----------+-----------+----------+----------+
+        |date               | date_day | date_month | date_year | date_hour | date_min | date_sec |
+        +-------------------+----------+------------+-----------+-----------+----------+----------+
+        |2024-01-15 12:31:27| 15       | 01         | 2024      | 12        | 31       | 27       |
+        +-------------------+----------+------------+-----------+-----------+----------+----------+
+        ''')
+
+    def __init__(self):
+        self.columns: list[str] = None
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns = dataset.get_columns_names_by_type(DataType.DATE)
+
+        self.explanations = [
+            f'Split date column **`{c}`** into year, month, weekday, hour, minute and second.'
+            for c in self.columns
+        ]
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """Split dates columns into columns (weekday, mount, year, hour, minute, second).
+
+        :param pd.DataFrame x: DataFrame to transform.
+        :return: Transformed dataset.
+        """
+        for column in self.columns:
+            # Day
+            X[column + '_weekday'] = X[column].dt.dayofweek.replace(np.NaN, -1)
+            X[column + '_month'] = X[column].dt.month.replace(np.NaN, -1)
+            X[column + '_year'] = X[column].dt.year.replace(np.NaN, -1)
+
+            # Hour
+            X[column + '_hour'] = X[column].dt.hour.replace(np.NaN, -1)
+            X[column + '_minute'] = X[column].dt.minute.replace(np.NaN, -1)
+            X[column + '_second'] = X[column].dt.second.replace(np.NaN, -1)
+
+        return X
+
+    def suitable(self, dataset: Dataset) -> bool:
+        return bool(dataset.get_columns_names_by_type(DataType.DATE))
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        return 0.5