PyIAML 1.0.0__py3-none-any.whl

iaml/actionables/cleaning/act_drop_high_cardinality_categorical.py

@@ -0,0 +1,337 @@
+"""[STEP] Drop or hash high-cardinality categorical columns."""
+import textwrap
+from typing import Any
+
+import pandas as pd
+
+from ...actionable import Actionable
+from ...candidate import Candidate
+from ...data_type import DataType
+from ...dataset import Dataset
+from ...decorators.all import is_step
+
+
+@is_step('cleaning')
+class ActDropHighCardinalityCategorical(Actionable):
+    """[STEP] Drop or hash high-cardinality categorical columns."""
+
+    name: str = 'Handle high-cardinality categorical columns'
+    _description: str = textwrap.dedent('''\
+        Drop or hash categorical columns whose cardinality meets or exceeds {max_unique}
+        or {unique_ratio_threshold:.0%} of non-missing rows.''')
+    _description_long: str = textwrap.dedent('''\
+        High-cardinality categorical columns can create large sparse encodings.
+        Columns with too many distinct values are either removed or replaced with
+        hashed integer codes (modulo {hash_bins}) depending on strategy {strategy}.
+        Columns with fewer than {min_non_null} non-missing rows are ignored.''')
+    _usage: str = "Use when categorical columns are extremely high-cardinality and you want drop/hash instead of ActCountVectorizer. Applicable to categorical features with high unique-to-row ratios. Avoid when categories are low-cardinality or predictive, or ActCategoricalImputer is enough."
+
+    def __init__(self) -> None:
+        self.configuration = {
+            'strategy': {
+                'description': 'How to handle high-cardinality columns.',
+                'default': 'drop',
+                'categorical': ['drop', 'hash']
+            },
+            'max_unique': {
+                'description': 'Maximum number of distinct values before handling.',
+                'default': 50
+            },
+            'unique_ratio_threshold': {
+                'description': 'Minimum unique/non-missing ratio to mark as high-cardinality.',
+                'default': 0.5,
+                'range': [0.0, 1.0]
+            },
+            'min_non_null': {
+                'description': 'Minimum number of non-missing rows to evaluate.',
+                'default': 10
+            },
+            'hash_bins': {
+                'description': 'Number of hash bins when strategy="hash".',
+                'default': 64
+            },
+            'hash_key': {
+                'description': 'Stable hash key used for hashing (16 chars recommended).',
+                'default': 'iaml_hashing_key'
+            },
+            'missing_value': {
+                'description': 'Numeric value used for missing categories when hashing.',
+                'default': -1
+            }
+        }
+        self.columns_to_drop: list[str] = []
+        self.columns_to_hash: list[str] = []
+        self.cardinality_stats: dict[str, dict[str, float]] = {}
+        self.strategy: str = 'drop'
+        self.hash_bins: int = 0
+        self.hash_key: str | bytes | None = None
+        self.missing_value: int = -1
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns_to_drop = []
+        self.columns_to_hash = []
+        self.cardinality_stats = {}
+        self.explanations = []
+
+        if dataset.X.empty:
+            return self
+
+        columns = self._select_columns(dataset)
+        if not columns:
+            return self
+
+        self.strategy = self._coerce_strategy(self.get_config('strategy'))
+        max_unique = self._coerce_int(self.get_config('max_unique'), 0)
+        ratio_threshold = self._coerce_ratio(self.get_config('unique_ratio_threshold'), 0.0)
+        min_non_null = self._coerce_int(self.get_config('min_non_null'), 0)
+
+        if max_unique <= 0 and ratio_threshold <= 0:
+            return self
+
+        stats = self._cardinality_stats(dataset.X[columns])
+        high_columns = self._high_cardinality_columns(
+            columns,
+            stats,
+            max_unique,
+            ratio_threshold,
+            min_non_null
+        )
+
+        if not high_columns:
+            return self
+
+        if self.strategy == 'hash':
+            self.hash_bins = self._coerce_bins(self.get_config('hash_bins'), 64)
+            self.hash_key = self._coerce_hash_key(self.get_config('hash_key'))
+            self.missing_value = self._coerce_int(self.get_config('missing_value'), -1)
+            self.columns_to_hash = high_columns
+        else:
+            self.columns_to_drop = high_columns
+
+        self.cardinality_stats = self._stats_to_dict(stats, high_columns)
+        self._build_explanations(stats, high_columns)
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        if self.columns_to_drop:
+            drop_columns = [col for col in self.columns_to_drop if col in X.columns]
+            if drop_columns:
+                X = X.drop(columns=drop_columns)
+
+        if not self.columns_to_hash:
+            return X
+
+        hash_bins = self.hash_bins or self._coerce_bins(self.get_config('hash_bins'), 64)
+        hash_key = self.hash_key if self.hash_key is not None else \
+            self._coerce_hash_key(self.get_config('hash_key'))
+        missing_value = self.missing_value
+
+        for column in self.columns_to_hash:
+            if column not in X.columns:
+                continue
+            X[column] = self._hash_series(X[column], hash_bins, hash_key, missing_value)
+
+        return X
+
+    def suitable(self, dataset: Dataset) -> bool:
+        if dataset.X.empty:
+            return False
+
+        columns = self._select_columns(dataset)
+        if not columns:
+            return False
+
+        max_unique = self._coerce_int(self.get_config('max_unique'), 0)
+        ratio_threshold = self._coerce_ratio(self.get_config('unique_ratio_threshold'), 0.0)
+        min_non_null = self._coerce_int(self.get_config('min_non_null'), 0)
+
+        if max_unique <= 0 and ratio_threshold <= 0:
+            return False
+
+        stats = self._cardinality_stats(dataset.X[columns])
+        high_columns = self._high_cardinality_columns(
+            columns,
+            stats,
+            max_unique,
+            ratio_threshold,
+            min_non_null
+        )
+        return bool(high_columns)
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        if candidate is None or candidate.dataset.X.empty:
+            return 0.0
+
+        columns = self._select_columns(candidate.dataset)
+        if not columns:
+            return 0.0
+
+        max_unique = self._coerce_int(self.get_config('max_unique'), 0)
+        ratio_threshold = self._coerce_ratio(self.get_config('unique_ratio_threshold'), 0.0)
+        min_non_null = self._coerce_int(self.get_config('min_non_null'), 0)
+
+        if max_unique <= 0 and ratio_threshold <= 0:
+            return 0.0
+
+        stats = self._cardinality_stats(candidate.dataset.X[columns])
+        high_columns = self._high_cardinality_columns(
+            columns,
+            stats,
+            max_unique,
+            ratio_threshold,
+            min_non_null
+        )
+        if not high_columns:
+            return 0.0
+
+        ratio = len(high_columns) / max(1, len(columns))
+        return min(1.5, 0.5 + ratio)
+
+    def _select_columns(self, dataset: Dataset) -> list[str]:
+        columns = dataset.get_columns_names_by_type(DataType.CATEGORICAL)
+        category_columns = list(dataset.X.select_dtypes(include=['category']).columns)
+        seen: set[str] = set()
+        ordered: list[str] = []
+        for column in columns + category_columns:
+            if column in dataset.X.columns and column not in seen:
+                ordered.append(column)
+                seen.add(column)
+        return ordered
+
+    def _cardinality_stats(self, X: pd.DataFrame) -> pd.DataFrame:
+        non_null = X.notna().sum()
+        unique = X.nunique(dropna=True)
+        ratio = unique / non_null.replace(0, pd.NA)
+        ratio = ratio.fillna(0.0)
+        return pd.DataFrame({'unique': unique, 'non_null': non_null, 'ratio': ratio})
+
+    def _high_cardinality_columns(
+        self,
+        columns: list[str],
+        stats: pd.DataFrame,
+        max_unique: int,
+        ratio_threshold: float,
+        min_non_null: int
+    ) -> list[str]:
+        eligible = stats['non_null'] >= min_non_null
+        conditions = pd.Series(False, index=stats.index)
+        if max_unique > 0:
+            conditions |= stats['unique'] >= max_unique
+        if ratio_threshold > 0:
+            conditions |= stats['ratio'] >= ratio_threshold
+        high = eligible & conditions
+        return [column for column in columns if column in high.index and bool(high[column])]
+
+    def _build_explanations(self, stats: pd.DataFrame, columns: list[str]) -> None:
+        for column in columns:
+            values = stats.loc[column]
+            unique = int(values['unique'])
+            non_null = int(values['non_null'])
+            ratio = float(values['ratio']) if non_null else 0.0
+            if self.strategy == 'hash':
+                self.explanations.append(
+                    f"Hashed `{column}` into {self.hash_bins} bins "
+                    f"(unique {unique}/{non_null}, ratio {ratio:.2%})."
+                )
+            else:
+                self.explanations.append(
+                    f"Dropped `{column}` with {unique} unique values "
+                    f"({ratio:.2%} of {non_null} non-missing)."
+                )
+
+    @staticmethod
+    def _stats_to_dict(stats: pd.DataFrame, columns: list[str]) -> dict[str, dict[str, float]]:
+        result: dict[str, dict[str, float]] = {}
+        for column in columns:
+            values = stats.loc[column]
+            result[column] = {
+                'unique': float(values['unique']),
+                'non_null': float(values['non_null']),
+                'ratio': float(values['ratio'])
+            }
+        return result
+
+    @staticmethod
+    def _hash_series(
+        series: pd.Series,
+        bins: int,
+        hash_key: str | bytes | None,
+        missing_value: int
+    ) -> pd.Series:
+        if bins <= 0:
+            return pd.Series(missing_value, index=series.index, dtype='int64')
+
+        mask = series.isna()
+        values = series.astype('object')
+        hashed = ActDropHighCardinalityCategorical._hash_values(values, hash_key)
+        hashed = (hashed % bins).astype('int64')
+
+        if missing_value is not None and mask.any():
+            hashed = hashed.where(~mask, int(missing_value))
+
+        return hashed
+
+    @staticmethod
+    def _hash_values(values: pd.Series, hash_key: str | bytes | None) -> pd.Series:
+        if hash_key is None:
+            return pd.util.hash_pandas_object(values, index=False)
+        try:
+            return pd.util.hash_pandas_object(values, index=False, hash_key=hash_key)
+        except TypeError:
+            return pd.util.hash_pandas_object(values, index=False)
+
+    @staticmethod
+    def _coerce_int(value: Any, default: int) -> int:
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            return default
+
+    @staticmethod
+    def _coerce_ratio(value: Any, default: float) -> float:
+        try:
+            ratio = float(value)
+        except (TypeError, ValueError):
+            return default
+        return min(1.0, max(0.0, ratio))
+
+    @staticmethod
+    def _coerce_bins(value: Any, default: int) -> int:
+        try:
+            bins = int(value)
+        except (TypeError, ValueError):
+            return default
+        if bins < 2:
+            return default
+        return bins
+
+    @staticmethod
+    def _coerce_strategy(value: Any) -> str:
+        if isinstance(value, str):
+            normalized = value.strip().lower()
+            if normalized in {'drop', 'hash'}:
+                return normalized
+        return 'drop'
+
+    @staticmethod
+    def _coerce_hash_key(value: Any) -> str | bytes | None:
+        if value is None:
+            return None
+        if isinstance(value, (bytes, bytearray)):
+            key = bytes(value)
+            if not key:
+                return None
+            if len(key) < 16:
+                key = key.ljust(16, b'0')
+            elif len(key) > 16:
+                key = key[:16]
+            return key
+        key = str(value)
+        if not key:
+            return None
+        if len(key) < 16:
+            key = key.ljust(16, '0')
+        elif len(key) > 16:
+            key = key[:16]
+        return key

iaml/actionables/cleaning/act_drop_numerical_column.py

@@ -0,0 +1,75 @@
+"""[STEP] Drop Numerical Column"""
+import textwrap
+import pandas as pd
+from ...actionable import Actionable
+from ...dataset import Dataset
+from ...data_type import DataType
+from ...candidate import Candidate
+from ...decorators.all import is_step
+
+
+@is_step('cleaning')
+class ActDropNumericalColumn(Actionable):
+    """[STEP] Drop Numerical Column"""
+
+    name: str = 'Remove numerical columns'
+    _description: str = textwrap.dedent('''\
+        Remove numerical columns where the proportion of empty rows
+        in the dataset is higher than {empty_threshold}.''')
+    _description_long: str = textwrap.dedent('''\
+        Remove numerical columns from the dataset where the proportion of empty
+        rows in the dataset is higher than {empty_threshold:.0%}. This ensure that every columns will
+        be relevant for the model to train on.''')
+    _usage: str = "Use when numeric columns are mostly empty and dropping is acceptable; compare ActDropCategoricalColumn for non-numeric drops. Applicable to datasets with numeric fields and high missingness. Avoid when you should impute or the numeric signal is critical."
+
+    def __init__(self):
+        self.columns_to_drop: list[str] = None
+        self.configuration = {
+            'empty_threshold': {
+                'description': textwrap.dedent('''\
+                    Column with more or equal proportion of empty row will
+                    dropped. 1 will drop all columns'''),
+                'default': 0.5
+            }
+        }
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns_to_drop = []
+        explain = []
+
+        for column in dataset.get_columns_names_by_type(DataType.NUMERIC):
+            values = dataset.X[column]
+            nan_values_count = values.isnull().sum()
+
+            if nan_values_count / len(values) >= self.get_config('empty_threshold'):
+                self.columns_to_drop.append(column)
+                explain.append((nan_values_count, len(values)))
+
+        self.explanations = [
+            f"""Dropped column **`{c}`** because **{v[0]}** values out of
+                **{v[1]}** (**{(v[0] / v[1] * 100):.2f}%**) are empty."""
+            for c, v in zip(self.columns_to_drop, explain)
+        ]
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """Drop columns.
+
+        :param pd.DataFrame X: DataFrame to transform.
+        :return: Transformed DataFrame.
+        """
+        return X.drop(self.columns_to_drop, axis=1)
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        return 0
+
+    def suitable(self, dataset: Dataset) -> bool:
+        for column in dataset.get_columns_names_by_type(DataType.NUMERIC):
+            values = dataset.X[column]
+            nan_values_count = values.isnull().sum()
+
+            if nan_values_count / len(values) >= self.get_config('empty_threshold'):
+                return True
+
+        return False

iaml/actionables/cleaning/act_drop_textual_column.py

@@ -0,0 +1,51 @@
+"""[STEP] Drop Textual Column"""
+import textwrap
+import pandas as pd
+from ...actionable import Actionable
+from ...dataset import Dataset
+from ...data_type import DataType
+from ...candidate import Candidate
+from ...decorators.all import is_step
+
+
+@is_step('cleaning', 'baseline_cleaning')
+class ActDropTextualColumn(Actionable):
+    """[STEP] Drop Textual Column"""
+
+    name: str = 'Remove textual columns'
+    _description: str = 'Remove all columns containing textual data from the dataset'
+    _usage: str = 'Use when text columns are irrelevant for modeling or downstream steps. Applicable to datasets with free-text or short-text fields detected as text/object. Avoid when text should be vectorized or cleaned; consider ActCountVectorizer or ActCategoricalImputer.'
+    _description_long: str = textwrap.dedent('''\
+        Remove all columns containing textual data from the dataset.
+        This step is used to clean the dataset in order to perform other actions later on
+        that can't be applied to textual columns.''')
+    can_be_disabled: bool = False
+
+    def __init__(self):
+        self.columns_to_drop: list[str] = None
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns_to_drop = list(set(
+            dataset.get_columns_names_by_type([DataType.TEXT, DataType.SHORT_TEXT]) + \
+            list(dataset.X.select_dtypes(include='object').columns)
+        ))
+
+        self.explanations = [
+            f'Dropped column **`{c}`**.' for c in self.columns_to_drop
+        ]
+
+        return self
+
+    def transform(self, X) -> pd.DataFrame:
+        """Drop columns.
+
+        :param pd.DataFrame X: DataFrame to transform.
+        :return: Transformed DataFrame.
+        """
+        return X.drop(self.columns_to_drop, axis=1)
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        return 0
+
+    def suitable(self, dataset: Dataset) -> bool:
+        return bool(dataset.get_columns_names_by_type([DataType.TEXT, DataType.SHORT_TEXT]))

iaml/actionables/cleaning/act_encode_target_column.py

@@ -0,0 +1,56 @@
+"""Experimental target encoder, available only through an explicit module import.
+
+This prototype transforms y rather than X, recomputes the mapping on each call,
+and cannot reverse predictions. It is incompatible with the pipeline transformer
+contract and must remain outside automatic cleaning. See docs/component_status.rst.
+"""
+
+import textwrap
+import numpy as np
+from ...actionable import Actionable
+from ...dataset import Dataset
+from ...candidate import Candidate
+from ...decorators.all import is_step
+
+
+@is_step('experimental')
+class ActCategoryStringToNumeric(Actionable):
+    """Encode categorical target column to numeric"""
+
+    name: str = 'Textual Category To Numeric Value'
+    _description: str ='Encode categorical text data column to numeric value'
+    _usage: str = 'Use when target labels are categorical strings and models require numeric y; prefer ActDropCategoricalColumn only if target is unusable. Applicable to single target columns of low-to-moderate cardinality. Avoid when target is already numeric or when ActDropHighCardinalityCategorical is more appropriate.'
+    _description_long: str = textwrap.dedent('''\
+        Retrieve all unique values from a column, then transform those values to a numeric type.
+        Exemple: If a column contain 3 uniques values like "coffee", "tea" and "water",
+        then all the coffee values will be transformed to 0, tea to 1 and water to 2.''')
+
+    def __init__(self):
+        self.column_to_encode: list[str] = None
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.column_to_encode = [dataset.y]
+        self.explanations = [
+            f'Encoded column **`{np.unique(dataset.y)}`**. \
+            Mapping of categorical values to numerical values:\n- `{value}`: {i}'
+            for i, value in enumerate(np.unique(dataset.y)) ]
+
+        return self
+
+    def transform(self, y: np.array) -> np.array:
+        """Convert categorical target columns of candidate's dataset to numeric.
+
+        :param np.array y: Target column to transform.
+        :return: Transformed target column.
+        """
+        if np.issubdtype(y.dtype, object) or np.issubdtype(y.dtype, np.bool_):
+            categories = np.unique(y)
+            encoded_target = np.zeros_like(y, dtype=int)
+            for i, category in enumerate(categories):
+                encoded_target[y == category] = i
+            return encoded_target
+
+        return y
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        return 0.4

iaml/actionables/cleaning/act_frequency_encoder.py

@@ -0,0 +1,127 @@
+"""[STEP] Encode categorical features by relative frequency."""
+import textwrap
+from typing import Any
+
+import pandas as pd
+
+from ...actionable import Actionable
+from ...candidate import Candidate
+from ...data_type import DataType
+from ...dataset import Dataset
+from ...decorators.all import is_step
+
+
+@is_step('cleaning')
+class ActFrequencyEncoder(Actionable):
+    """[STEP] Encode categorical features by relative frequency."""
+
+    name: str = 'Frequency encoding'
+    _usage: str = "Use when categorical features need numeric encoding and frequencies are stable; compare ActDropHighCardinalityCategorical. Applicable to categorical/string features with moderate cardinality. Avoid when categories are very sparse, drifting, or too few rows to estimate."
+    _description: str = textwrap.dedent('''\
+        Encode categorical columns using their relative frequency.''')
+    _description_long: str = textwrap.dedent('''\
+        Replace each category with its relative frequency observed in the training data.
+        Frequencies are computed on non-missing values and unseen or missing values are
+        mapped to {unknown_value}.''')
+
+    def __init__(self) -> None:
+        self.columns: list[str] = []
+        self.encodings: dict[str, dict[Any, float]] = {}
+        self.fallback_values: dict[str, float] = {}
+        self.unknown_value: float = 0.0
+
+        self.configuration = {
+            'unknown_value': {
+                'description': 'Value used for unseen or missing categories.',
+                'default': 0.0
+            }
+        }
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns = self._select_columns(dataset)
+        self.encodings = {}
+        self.fallback_values = {}
+        self.explanations = []
+
+        if not self.columns or dataset.X.empty:
+            return self
+
+        self.unknown_value = self._coerce_float(self.get_config('unknown_value'), 0.0)
+
+        for column in self.columns:
+            series = dataset.X[column]
+            counts = series.value_counts(dropna=True)
+            total = int(counts.sum())
+            if total <= 0 or counts.empty:
+                self.fallback_values[column] = self.unknown_value
+                continue
+
+            frequencies = (counts / total).to_dict()
+            self.encodings[column] = frequencies
+            self.fallback_values[column] = self.unknown_value
+            self.explanations.append(
+                f"Encoded `{column}` using {len(frequencies)} categories."
+            )
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        if not self.columns:
+            return X
+
+        for column in self.columns:
+            if column not in X.columns:
+                continue
+
+            mapping = self.encodings.get(column, {})
+            fallback = self.fallback_values.get(column, self.unknown_value)
+            fallback = self._coerce_float(fallback, 0.0)
+
+            encoded = X[column].map(mapping)
+            X[column] = encoded.fillna(fallback).astype(float)
+
+        return X
+
+    def suitable(self, dataset: Dataset) -> bool:
+        columns = self._select_columns(dataset)
+        if not columns or dataset.X.empty:
+            return False
+        return True
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        if candidate is None or candidate.dataset.X.empty:
+            return 0.0
+
+        columns = self._select_columns(candidate.dataset)
+        if not columns:
+            return 0.0
+
+        total_rows = len(candidate.dataset.X)
+        if total_rows <= 0:
+            return 0.0
+
+        unique_counts = candidate.dataset.X[columns].nunique(dropna=True)
+        avg_cardinality = float((unique_counts / total_rows).mean())
+
+        total_columns = candidate.dataset.X.shape[1] or 1
+        cat_ratio = len(columns) / total_columns
+
+        return min(1.0, max(cat_ratio, avg_cardinality))
+
+    def _select_columns(self, dataset: Dataset) -> list[str]:
+        columns = dataset.get_columns_names_by_type(DataType.CATEGORICAL)
+        category_columns = list(dataset.X.select_dtypes(include=['category']).columns)
+        seen = set()
+        ordered = []
+        for column in columns + category_columns:
+            if column in dataset.X.columns and column not in seen:
+                ordered.append(column)
+                seen.add(column)
+        return ordered
+
+    @staticmethod
+    def _coerce_float(value: Any, default: float) -> float:
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            return default