PyIAML 1.0.0__py3-none-any.whl

iaml/actionables/cleaning/act_hashing_vectorizer.py

@@ -0,0 +1,186 @@
+"""[STEP] Vectorize large text with HashingVectorizer."""
+import textwrap
+from typing import Any
+
+import pandas as pd
+from sklearn.feature_extraction.text import HashingVectorizer
+
+from ...actionable import Actionable
+from ...candidate import Candidate
+from ...data_type import DataType
+from ...dataset import Dataset
+from ...decorators.all import is_step
+
+
+@is_step('cleaning')
+class ActHashingVectorizer(Actionable):
+    """[STEP] Vectorize large text with HashingVectorizer."""
+
+    name: str = 'Hashing Vectorizer'
+    _description: str = textwrap.dedent('''\
+        Vectorize text columns with the hashing trick for large vocabularies.''')
+    _description_long: str = textwrap.dedent('''\
+        Convert long text columns into fixed-size hashed feature vectors without
+        building an explicit vocabulary. This keeps memory usage bounded even for
+        very large vocabularies, at the cost of possible hash collisions.''')
+    _usage: str = "Use when text columns have huge vocabularies and you need fixed-size features, especially vs ActCountVectorizer for memory bounds. Applicable to free-text columns that you want numeric n-grams from. Avoid when token interpretability or collision-free features are required."
+
+    def __init__(self) -> None:
+        self.columns: list[str] = []
+        self.vectorizer: HashingVectorizer | None = None
+        self.configuration = {
+            'n_features': {
+                'description': 'Number of hash bins (power of two recommended).',
+                'default': 4096
+            },
+            'ngram_min': {
+                'description': 'Minimum n-gram size to include.',
+                'default': 1
+            },
+            'ngram_max': {
+                'description': 'Maximum n-gram size to include.',
+                'default': 2
+            },
+            'alternate_sign': {
+                'description': 'Use alternating signs to reduce hash collisions.',
+                'default': False
+            },
+            'binary': {
+                'description': 'If True, store binary occurrences instead of counts.',
+                'default': False
+            },
+            'norm': {
+                'description': 'Vector normalization ("l1", "l2", or None).',
+                'default': 'l2'
+            },
+            'lowercase': {
+                'description': 'Lowercase text before hashing.',
+                'default': True
+            }
+        }
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns = []
+        self.vectorizer = None
+        self.explanations = []
+
+        columns = dataset.get_columns_names_by_type([DataType.TEXT])
+        if not columns or dataset.X.empty:
+            return self
+
+        params = self._build_vectorizer_params()
+        self.vectorizer = HashingVectorizer(**params)
+        self.columns = [column for column in columns if column in dataset.X.columns]
+
+        n_features = params['n_features']
+        for column in self.columns:
+            self.explanations.append(
+                f'Encoded text column **`{column}`** into **{n_features}** hashed features.'
+            )
+
+        if not self.columns:
+            self.explanations.append(
+                'Hashing vectorizer skipped: no usable text columns.'
+            )
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """Apply HashingVectorizer to text columns.
+
+        :param pd.DataFrame X: DataFrame to transform.
+        :return: Transformed dataset.
+        """
+        if not self.columns or self.vectorizer is None:
+            return X
+
+        X = X.reset_index(drop=True)
+
+        for name in self.columns:
+            if name not in X.columns:
+                continue
+
+            values = X[name].fillna('').astype(str)
+            transformed = self.vectorizer.transform(values)
+            n_features = transformed.shape[1]
+            new_names = [f"{name}_hash_{i}" for i in range(n_features)]
+            vector_df = pd.DataFrame(transformed.toarray(), columns=new_names)
+
+            X = pd.concat([X, vector_df], axis=1).drop([name], axis=1)
+
+        return X
+
+    def suitable(self, dataset: Dataset) -> bool:
+        if dataset.X.empty:
+            return False
+        return bool(dataset.get_columns_names_by_type([DataType.TEXT]))
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        if candidate is None or candidate.dataset.X.empty:
+            return 0.0
+
+        columns = candidate.dataset.get_columns_names_by_type([DataType.TEXT])
+        if not columns:
+            return 0.0
+
+        total_columns = candidate.dataset.X.shape[1] or 1
+        return min(1.0, len(columns) / total_columns)
+
+    def _build_vectorizer_params(self) -> dict[str, Any]:
+        n_features = self._coerce_positive_int(self.get_config('n_features'), 4096)
+        ngram_min = self._coerce_int(self.get_config('ngram_min'), 1)
+        ngram_max = self._coerce_int(self.get_config('ngram_max'), max(ngram_min, 1))
+        ngram_min = max(1, ngram_min)
+        ngram_max = max(ngram_min, ngram_max)
+
+        return {
+            'n_features': n_features,
+            'ngram_range': (ngram_min, ngram_max),
+            'alternate_sign': self._coerce_bool(self.get_config('alternate_sign'), False),
+            'binary': self._coerce_bool(self.get_config('binary'), False),
+            'norm': self._coerce_norm(self.get_config('norm')),
+            'lowercase': self._coerce_bool(self.get_config('lowercase'), True)
+        }
+
+    @staticmethod
+    def _coerce_int(value: Any, default: int) -> int:
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            return default
+
+    @staticmethod
+    def _coerce_positive_int(value: Any, default: int) -> int:
+        try:
+            numeric = int(value)
+        except (TypeError, ValueError):
+            return default
+        if numeric <= 0:
+            return default
+        return numeric
+
+    @staticmethod
+    def _coerce_bool(value: Any, default: bool) -> bool:
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, str):
+            normalized = value.strip().lower()
+            if normalized in {'true', '1', 'yes', 'y'}:
+                return True
+            if normalized in {'false', '0', 'no', 'n'}:
+                return False
+        if value is None:
+            return default
+        return bool(value)
+
+    @staticmethod
+    def _coerce_norm(value: Any) -> str | None:
+        if value is None:
+            return None
+        if isinstance(value, str):
+            normalized = value.strip().lower()
+            if normalized in {'none', ''}:
+                return None
+            if normalized in {'l1', 'l2'}:
+                return normalized
+        return 'l2'

iaml/actionables/cleaning/act_knn_imputer.py

@@ -0,0 +1,152 @@
+"""[STEP] KNN imputer for numeric columns."""
+import textwrap
+
+import pandas as pd
+from sklearn.impute import KNNImputer
+
+from ...actionable import Actionable
+from ...candidate import Candidate
+from ...data_type import DataType
+from ...dataset import Dataset
+from ...decorators.all import is_step
+
+
+@is_step('cleaning')
+class ActKNNImputer(Actionable):
+    """[STEP] Impute missing numeric values using KNN."""
+
+    name: str = 'Impute missing values (KNN)'
+    _usage: str = 'Use when numeric features have gaps and you want to preserve rows vs ActDropNumericalColumn. Applicable to numeric columns with enough non-missing rows; use ActCategoricalImputer for categoricals. Avoid when missingness is extreme, dataset is tiny, or you plan to drop the column.'
+    _description: str = textwrap.dedent('''\
+        Impute missing numeric values using a k-nearest neighbors strategy.''')
+    _description_long: str = textwrap.dedent('''\
+        Uses scikit-learn KNNImputer to fill missing values in numeric columns
+        by averaging the k nearest neighbors in feature space. Non-numeric
+        columns are left untouched.''')
+
+    def __init__(self):
+        self.columns: list[str] = []
+        self.knn_columns: list[str] = []
+        self.imputer: KNNImputer | None = None
+        self._all_nan_cols: list[str] = []
+        self._fallback_values: dict[str, float] = {}
+        self._nan_stats: dict[str, tuple[int, int, float]] = {}
+
+        self.configuration = {
+            'n_neighbors': {
+                'description': 'Number of neighbors used for imputing missing values.',
+                'default': 5,
+                'range': [1, 50],
+                'passthrough': False
+            },
+            'weights': {
+                'description': 'Weight function used in prediction.',
+                'default': 'uniform',
+                'categorical': ['uniform', 'distance']
+            },
+            'metric': {
+                'description': 'Distance metric to use for missing-aware KNN.',
+                'default': 'nan_euclidean',
+                'categorical': ['nan_euclidean']
+            }
+        }
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns = dataset.get_columns_names_by_type(DataType.NUMERIC)
+        self.knn_columns = []
+        self.imputer = None
+        self._all_nan_cols = []
+        self._fallback_values = {}
+        self._nan_stats = {}
+        self.explanations = []
+
+        if not self.columns:
+            return self
+
+        X_num = dataset.X[self.columns]
+        total_rows = len(X_num)
+        if total_rows == 0:
+            self.explanations = ["Skipped KNN imputation: dataset has 0 rows."]
+            return self
+
+        missing_counts = X_num.isna().sum()
+        means = X_num.mean()
+        for column in self.columns:
+            missing = int(missing_counts[column])
+            pct = (missing / total_rows * 100.0) if total_rows else 0.0
+            self._nan_stats[column] = (missing, total_rows, pct)
+
+            mean_value = means[column]
+            if pd.isna(mean_value):
+                mean_value = 0.0
+            self._fallback_values[column] = float(mean_value)
+
+        self._all_nan_cols = [
+            column for column in self.columns if missing_counts[column] == total_rows
+        ]
+        self.knn_columns = [
+            column for column in self.columns if column not in self._all_nan_cols
+        ]
+
+        if self.knn_columns and total_rows >= 2:
+            n_neighbors = min(self.get_config('n_neighbors'), total_rows - 1)
+            n_neighbors = max(1, int(n_neighbors))
+            params = self.passthrough_parameters()
+            params['n_neighbors'] = n_neighbors
+            self.imputer = KNNImputer(**params)
+            self.imputer.fit(X_num[self.knn_columns])
+
+        if self.imputer is not None:
+            self.explanations = [
+                f"Imputed missing values of column **`{c}`** using **KNN** "
+                f"(**{n}** / **{t}**; **{pct:.2f}%** missing in train data)."
+                for c, (n, t, pct) in self._nan_stats.items()
+                if n > 0 and c in self.knn_columns
+            ]
+        if self._all_nan_cols:
+            self.explanations.append(
+                "Filled all-NaN numeric columns with 0.0: " +
+                ", ".join(f"`{c}`" for c in self._all_nan_cols) +
+                "."
+            )
+        if self.imputer is None and not self.explanations:
+            if any(n > 0 for n, _, _ in self._nan_stats.values()):
+                self.explanations.append(
+                    "Skipped KNN imputation; filled numeric columns with their mean "
+                    "(0.0 when undefined)."
+                )
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        if not self.columns:
+            return X
+
+        if self.imputer is not None and self.knn_columns:
+            if all(column in X.columns for column in self.knn_columns):
+                X_knn = X[self.knn_columns].copy()
+                X_knn = X_knn.apply(pd.to_numeric, errors='coerce')
+                imputed = self.imputer.transform(X_knn)
+                X.loc[:, self.knn_columns] = imputed
+
+        for column, fill_value in self._fallback_values.items():
+            if column in X.columns:
+                X[column] = X[column].fillna(fill_value).infer_objects(copy=False)
+
+        return X
+
+    def suitable(self, dataset: Dataset) -> bool:
+        columns = dataset.get_columns_names_by_type(DataType.NUMERIC)
+        if not columns or dataset.X.empty:
+            return False
+        return bool(dataset.X[columns].isna().any().any())
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        if candidate is None or candidate.dataset.X.empty:
+            return 0.0
+        columns = candidate.dataset.get_columns_names_by_type(DataType.NUMERIC)
+        if not columns:
+            return 0.0
+        missing = candidate.dataset.X[columns].isna().sum().sum()
+        total = candidate.dataset.X[columns].size or 1
+        return min(1.0, missing / total)

iaml/actionables/cleaning/act_mean_column.py

@@ -0,0 +1,79 @@
+"""[STEP] Fill missing values with mean"""
+import textwrap
+import pandas as pd
+import numpy as np
+from ...actionable import Actionable
+from ...dataset import Dataset
+from ...candidate import Candidate
+from ...decorators.all import is_step
+from ...data_type import DataType
+
+
+@is_step('cleaning', 'baseline_cleaning')
+class ActMeanColumn(Actionable):
+    """[STEP] Fill missing values with the mean."""
+
+    name: str = 'Fill missing values'
+    _usage: str = "Use when numeric columns have some missing values and you want a quick baseline imputation over ActDropNumericalColumn. Applicable to numerical data with moderate missingness. Avoid when missingness is high or systematic, or when ActDropNumericalColumn is safer."
+    _description: str = textwrap.dedent('''\
+        Fill missing values with the mean of non-missing values
+        when the proportion of empty rows is lower than {empty_threshold:.0%}.''')
+    _description_long: str = textwrap.dedent('''\
+        Fill a column missings values with the mean of the columns
+        when the proportion of empty rows is lower than {empty_threshold}.
+        Work only for numerical columns.''')
+    can_be_disabled: bool = False
+
+    def __init__(self):
+        self.columns: list[str] = None
+        self.configuration:dict = {
+            'empty_threshold': {
+                'description': textwrap.dedent('''\
+                    Column with less or equal proportion of empty row will be
+                    fill with mean value. 1 will always fill void values'''),
+                'default': 1 # TODO Review when adding new kind of imputer
+            }
+        }
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns = []
+        explain = []
+
+        for column in dataset.get_columns_names_by_type(DataType.NUMERIC):
+            values = dataset.X[column]
+            nan_values_count = values.isnull().sum()
+
+            mean = values.mean()
+            if np.isnan(mean):
+                mean = 0
+
+            self.columns.append((column, mean))
+            explain.append((
+                nan_values_count,
+                len(values),
+                nan_values_count / len(values) * 100,
+            ))
+
+        self.explanations = [
+            f"""Filled missing values of column **`{c}`** with **{mean:.2f}**
+                (**{v[0]}** out of **{v[1]}** values (**{v[2]:.2f}**%)
+                were missing in train data)."""
+            for (c, mean), v in zip(self.columns, explain)
+            if v[0] > 0 # hide processings that affected no values
+        ]
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """Fill NA values with the mean.
+
+        :param pd.DataFrame x: DataFrame to transform.
+        :return: Transformed dataset.
+        """
+        for name, mean in self.columns:
+            X[name] = X[name].fillna(mean).infer_objects(copy=False)
+
+        return X
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        return 1 - (candidate.dataset.X.isnull().sum().min() / len(candidate.dataset.X))