PyIAML 1.0.0__py3-none-any.whl

iaml/actionables/cleaning/act_target_encoder.py

@@ -0,0 +1,274 @@
+"""[STEP] Target mean encoding with internal cross-validation."""
+import textwrap
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import KFold, StratifiedKFold
+
+from ...actionable import Actionable
+from ...candidate import Candidate
+from ...data_type import DataType
+from ...dataset import Dataset
+from ...decorators.all import is_step
+
+
+@is_step('cleaning')
+class ActTargetEncoder(Actionable):
+    """[STEP] Encode categorical features by target mean with internal CV."""
+
+    name: str = 'Target mean encoding'
+    _description: str = textwrap.dedent('''\
+        Encode categorical columns with the mean of the target using internal CV.''')
+    _description_long: str = textwrap.dedent('''\
+        Replace each category with the mean target value computed on training folds only.
+        This yields out-of-fold encodings for the training data to reduce leakage. For new
+        data, mappings learned on the full training set are used and unseen categories are
+        mapped to the global target mean. Non-numeric targets are factorized first.''')
+    _usage: str = "Use when categorical features have target signal and you want numeric encoding vs ActCountVectorizer. Applicable to supervised data with categorical columns and enough rows. Avoid when leakage risk is high or categories are too sparse; prefer ActDropHighCardinalityCategorical."
+
+    def __init__(self) -> None:
+        self.columns: list[str] = []
+        self.encodings: dict[str, dict[Any, float]] = {}
+        self.fallback_values: dict[str, float] = {}
+        self.global_mean: float = 0.0
+        self._train_encoded: pd.DataFrame | None = None
+        self._train_X: pd.DataFrame | None = None
+
+        self.configuration = {
+            'n_splits': {
+                'description': 'Number of CV folds used to compute out-of-fold encodings.',
+                'default': 5
+            },
+            'shuffle': {
+                'description': 'Shuffle rows before splitting into folds.',
+                'default': True
+            },
+            'random_state': {
+                'description': 'Random seed used when shuffling.',
+                'default': 42
+            },
+            'smoothing': {
+                'description': 'Smoothing strength towards the global mean.',
+                'default': 1.0
+            }
+        }
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns = self._select_columns(dataset)
+        self.encodings = {}
+        self.fallback_values = {}
+        self.global_mean = 0.0
+        self._train_encoded = None
+        self._train_X = None
+        self.explanations = []
+
+        if not self.columns or dataset.X.empty or dataset.y is None:
+            return self
+
+        y_series = pd.Series(dataset.y, index=dataset.X.index)
+        y_numeric = self._coerce_target(y_series)
+        if y_numeric.empty:
+            return self
+
+        prior = float(y_numeric.mean())
+        if np.isnan(prior):
+            prior = 0.0
+        self.global_mean = prior
+
+        n_rows = len(dataset.X)
+        n_splits = self._coerce_int(self.get_config('n_splits'), 5)
+        n_splits = min(max(n_splits, 2), n_rows) if n_rows > 1 else 1
+        shuffle = bool(self.get_config('shuffle'))
+        random_state = self._coerce_int(self.get_config('random_state'), 42)
+        smoothing = self._coerce_float(self.get_config('smoothing'), 1.0)
+
+        use_cv = n_rows >= 2 and n_splits >= 2
+        splits = []
+        if use_cv:
+            splits = self._build_splits(
+                dataset.X,
+                y_series,
+                dataset.type_of_target,
+                n_splits,
+                shuffle,
+                random_state
+            )
+
+        if use_cv and not splits:
+            use_cv = False
+
+        encoded_train = pd.DataFrame(index=dataset.X.index, columns=self.columns, dtype=float)
+
+        for column in self.columns:
+            series = dataset.X[column]
+            mapping_full = self._fit_mapping(series, y_numeric, prior, smoothing)
+            self.encodings[column] = mapping_full
+            self.fallback_values[column] = prior
+
+            if use_cv:
+                oof = pd.Series(index=dataset.X.index, dtype=float)
+                for train_idx, val_idx in splits:
+                    mapping_fold = self._fit_mapping(
+                        series.iloc[train_idx],
+                        y_numeric.iloc[train_idx],
+                        prior,
+                        smoothing
+                    )
+                    encoded = series.iloc[val_idx].map(mapping_fold)
+                    oof.iloc[val_idx] = encoded
+                oof = oof.fillna(prior).astype(float)
+            else:
+                oof = series.map(mapping_full).fillna(prior).astype(float)
+
+            encoded_train[column] = oof
+
+            self.explanations.append(
+                f"Target-encoded `{column}` using {len(mapping_full)} categories."
+            )
+
+        self._train_encoded = encoded_train
+        self._train_X = dataset.X
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        if not self.columns or X.empty:
+            return X
+
+        if self._train_X is not None and X is self._train_X and self._train_encoded is not None:
+            for column in self.columns:
+                if column in X.columns and column in self._train_encoded.columns:
+                    X[column] = self._train_encoded[column].reindex(X.index).astype(float)
+            return X
+
+        for column in self.columns:
+            if column not in X.columns:
+                continue
+            mapping = self.encodings.get(column, {})
+            fallback = self.fallback_values.get(column, self.global_mean)
+            fallback = self._coerce_float(fallback, self.global_mean)
+            X[column] = X[column].map(mapping).fillna(fallback).astype(float)
+
+        return X
+
+    def suitable(self, dataset: Dataset) -> bool:
+        if dataset.y is None or dataset.X.empty:
+            return False
+        if dataset.type_of_target == 'survival':
+            return False
+        return bool(self._select_columns(dataset))
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        if candidate is None or candidate.dataset.X.empty or candidate.dataset.y is None:
+            return 0.0
+
+        columns = self._select_columns(candidate.dataset)
+        if not columns:
+            return 0.0
+
+        total_rows = len(candidate.dataset.X)
+        if total_rows <= 0:
+            return 0.0
+
+        unique_counts = candidate.dataset.X[columns].nunique(dropna=True)
+        avg_cardinality = float((unique_counts / total_rows).mean())
+
+        total_columns = candidate.dataset.X.shape[1] or 1
+        cat_ratio = len(columns) / total_columns
+
+        return min(1.0, max(cat_ratio, avg_cardinality))
+
+    def _select_columns(self, dataset: Dataset) -> list[str]:
+        columns = dataset.get_columns_names_by_type(DataType.CATEGORICAL)
+        category_columns = list(dataset.X.select_dtypes(include=['category']).columns)
+        seen = set()
+        ordered = []
+        for column in columns + category_columns:
+            if column in dataset.X.columns and column not in seen:
+                ordered.append(column)
+                seen.add(column)
+        return ordered
+
+    @staticmethod
+    def _coerce_target(y: pd.Series) -> pd.Series:
+        if y.empty:
+            return y.astype(float)
+        if pd.api.types.is_bool_dtype(y):
+            return y.astype(float)
+        if pd.api.types.is_numeric_dtype(y):
+            return y.astype(float)
+        codes, _ = pd.factorize(y, sort=True)
+        numeric = pd.Series(codes, index=y.index, dtype=float)
+        numeric[codes < 0] = np.nan
+        return numeric
+
+    @staticmethod
+    def _fit_mapping(
+        series: pd.Series,
+        target: pd.Series,
+        prior: float,
+        smoothing: float
+    ) -> dict[Any, float]:
+        if series.empty:
+            return {}
+
+        series_values = series.astype(object)
+        grouped = target.groupby(series_values).agg(['mean', 'count'])
+        if grouped.empty:
+            return {}
+
+        smoothing = float(smoothing)
+        if smoothing <= 0:
+            smooth = grouped['mean']
+        else:
+            smooth = (grouped['mean'] * grouped['count'] + prior * smoothing) \
+                / (grouped['count'] + smoothing)
+
+        return smooth.to_dict()
+
+    @staticmethod
+    def _coerce_int(value: Any, default: int) -> int:
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            return default
+
+    @staticmethod
+    def _coerce_float(value: Any, default: float) -> float:
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            return default
+
+    @staticmethod
+    def _build_splits(
+        X: pd.DataFrame,
+        y: pd.Series,
+        target_type: str,
+        n_splits: int,
+        shuffle: bool,
+        random_state: int
+    ) -> list[tuple[np.ndarray, np.ndarray]]:
+        if n_splits < 2:
+            return []
+
+        is_classification = target_type not in (
+            None,
+            'continuous',
+            'continuous-multioutput',
+            'survival'
+        )
+        if is_classification:
+            try:
+                splitter = StratifiedKFold(
+                    n_splits=n_splits,
+                    shuffle=shuffle,
+                    random_state=random_state
+                )
+                return list(splitter.split(X, y))
+            except ValueError:
+                pass
+
+        splitter = KFold(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
+        return list(splitter.split(X))

iaml/actionables/cleaning/act_text_normalizer.py

@@ -0,0 +1,241 @@
+"""[STEP] Normalize text columns before vectorization."""
+from __future__ import annotations
+
+import re
+import string
+import textwrap
+
+import pandas as pd
+from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS
+
+from ...actionable import Actionable
+from ...candidate import Candidate
+from ...data_type import DataType
+from ...dataset import Dataset
+from ...decorators.all import is_step
+
+
+_PUNCTUATION_RE = re.compile(rf"[{re.escape(string.punctuation)}]+")
+_MULTISPACE_RE = re.compile(r"\s+")
+
+
+@is_step('cleaning')
+class ActTextNormalizer(Actionable):
+    """[STEP] Normalize text columns before vectorization."""
+
+    name: str = 'Text normalizer'
+    _usage: str = 'Use when normalizing TEXT/SHORT_TEXT before ActCountVectorizer. Applicable to free-form text columns needing consistent casing, punctuation, or stopword cleanup. Avoid when text must remain verbatim or ActCountVectorizer settings already cover normalization.'
+    _description: str = textwrap.dedent('''\
+        Normalize text columns with lowercasing, punctuation removal, and stopword filtering.''')
+    _description_long: str = textwrap.dedent('''\
+        Prepare text columns for downstream vectorizers by standardizing casing, removing
+        punctuation, dropping common stopwords, and cleaning up extra whitespace. This step
+        targets columns typed as TEXT or SHORT_TEXT and keeps the processing deterministic.''')
+
+    def __init__(self) -> None:
+        self.configuration = {
+            'lowercase': {
+                'description': 'Lowercase text before normalization.',
+                'default': True
+            },
+            'remove_punctuation': {
+                'description': 'Replace punctuation characters with spaces.',
+                'default': True
+            },
+            'remove_stopwords': {
+                'description': 'Remove stopwords from text.',
+                'default': True
+            },
+            'stopwords': {
+                'description': 'Stopwords to remove ("english", "none", or a list).',
+                'default': 'english'
+            },
+            'collapse_whitespace': {
+                'description': 'Collapse repeated whitespace and strip edges.',
+                'default': True
+            }
+        }
+
+        self.columns: list[str] = []
+        self.stop_words: set[str] = set()
+        self._stopword_pattern: re.Pattern | None = None
+        self._lowercase: bool = True
+        self._remove_punctuation: bool = True
+        self._remove_stopwords: bool = True
+        self._collapse_whitespace: bool = True
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns = self._select_columns(dataset)
+        self.explanations = []
+
+        self._lowercase = self._coerce_bool(self.get_config('lowercase'), True)
+        self._remove_punctuation = self._coerce_bool(self.get_config('remove_punctuation'), True)
+        self._remove_stopwords = self._coerce_bool(self.get_config('remove_stopwords'), True)
+        self._collapse_whitespace = self._coerce_bool(self.get_config('collapse_whitespace'), True)
+
+        self.stop_words = self._normalize_stopwords(self._resolve_stopwords())
+        self._stopword_pattern = self._build_stopword_pattern(self.stop_words)
+
+        if not self.columns or dataset.X.empty:
+            return self
+
+        if not self._has_enabled_operations():
+            self.columns = []
+            self.explanations.append('Text normalizer skipped: no enabled operations.')
+            return self
+
+        operations = self._operations_summary()
+        for column in self.columns:
+            self.explanations.append(
+                f"Normalized text column **`{column}`** with {', '.join(operations)}."
+            )
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        if not self.columns or not self._has_enabled_operations():
+            return X
+
+        for column in self.columns:
+            if column not in X.columns:
+                continue
+
+            series = X[column].fillna('').astype(str)
+
+            if self._lowercase:
+                series = series.str.lower()
+            if self._remove_punctuation:
+                series = series.str.replace(_PUNCTUATION_RE, ' ', regex=True)
+            if self._stopword_pattern is not None:
+                series = series.str.replace(self._stopword_pattern, ' ', regex=True)
+            if self._collapse_whitespace:
+                series = series.str.replace(_MULTISPACE_RE, ' ', regex=True).str.strip()
+
+            X[column] = series
+
+        return X
+
+    def suitable(self, dataset: Dataset) -> bool:
+        if dataset.X.empty:
+            return False
+        if not self._select_columns(dataset):
+            return False
+        return self._has_enabled_operations(configured_only=True)
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        if candidate is None or candidate.dataset.X.empty:
+            return 0.0
+
+        columns = self._select_columns(candidate.dataset)
+        if not columns or not self._has_enabled_operations(configured_only=True):
+            return 0.0
+
+        total_columns = candidate.dataset.X.shape[1] or 1
+        return min(1.0, len(columns) / total_columns)
+
+    def _select_columns(self, dataset: Dataset) -> list[str]:
+        columns = dataset.get_columns_names_by_type([DataType.TEXT, DataType.SHORT_TEXT])
+        return [column for column in columns if column in dataset.X.columns]
+
+    def _resolve_stopwords(self) -> set[str]:
+        if not self._coerce_bool(self.get_config('remove_stopwords'), True):
+            return set()
+
+        stopwords = self.get_config('stopwords')
+        if stopwords is None:
+            return set()
+
+        if isinstance(stopwords, str):
+            normalized = stopwords.strip().lower()
+            if normalized in {'', 'none', 'false', 'off', 'no'}:
+                return set()
+            if normalized in {'english', 'sklearn'}:
+                return set(ENGLISH_STOP_WORDS)
+            tokens = re.split(r'[,;\s]+', stopwords.strip())
+            return {token for token in tokens if token}
+
+        if isinstance(stopwords, (list, tuple, set, frozenset)):
+            return {
+                str(word).strip()
+                for word in stopwords
+                if word is not None and str(word).strip()
+            }
+
+        return set()
+
+    def _normalize_stopwords(self, stopwords: set[str]) -> set[str]:
+        if not stopwords:
+            return set()
+
+        normalized: set[str] = set()
+        for word in stopwords:
+            token = str(word).strip()
+            if not token:
+                continue
+            if self._lowercase:
+                token = token.lower()
+            if self._remove_punctuation:
+                token = _PUNCTUATION_RE.sub(' ', token)
+            token = _MULTISPACE_RE.sub(' ', token).strip()
+            if not token:
+                continue
+            normalized.update(token.split())
+
+        return normalized
+
+    def _build_stopword_pattern(self, stopwords: set[str]) -> re.Pattern | None:
+        if not stopwords:
+            return None
+
+        escaped = [re.escape(word) for word in stopwords if word]
+        if not escaped:
+            return None
+
+        escaped.sort(key=len, reverse=True)
+        flags = re.IGNORECASE if not self._lowercase else 0
+        pattern = r'\b(?:' + '|'.join(escaped) + r')\b'
+        return re.compile(pattern, flags=flags)
+
+    def _operations_summary(self) -> list[str]:
+        operations: list[str] = []
+        if self._lowercase:
+            operations.append('lowercasing')
+        if self._remove_punctuation:
+            operations.append('punctuation removal')
+        if self._stopword_pattern is not None:
+            operations.append('stopword filtering')
+        if self._collapse_whitespace:
+            operations.append('whitespace cleanup')
+        if not operations:
+            operations.append('normalization')
+        return operations
+
+    def _has_enabled_operations(self, configured_only: bool = False) -> bool:
+        if configured_only:
+            lowercase = self._coerce_bool(self.get_config('lowercase'), True)
+            remove_punctuation = self._coerce_bool(self.get_config('remove_punctuation'), True)
+            collapse_whitespace = self._coerce_bool(self.get_config('collapse_whitespace'), True)
+            remove_stopwords = self._coerce_bool(self.get_config('remove_stopwords'), True)
+            if lowercase or remove_punctuation or collapse_whitespace:
+                return True
+            if remove_stopwords:
+                return bool(self._resolve_stopwords())
+            return False
+
+        if self._lowercase or self._remove_punctuation or self._collapse_whitespace:
+            return True
+        return self._stopword_pattern is not None
+
+    @staticmethod
+    def _coerce_bool(value: object, default: bool) -> bool:
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, str):
+            normalized = value.strip().lower()
+            if normalized in {'true', '1', 'yes', 'y'}:
+                return True
+            if normalized in {'false', '0', 'no', 'n'}:
+                return False
+        if value is None:
+            return default
+        return bool(value)

iaml/actionables/cleaning/act_tf_idf.py

@@ -0,0 +1,80 @@
+"""[STEP] Vectorize textual columns with TF-IDF"""
+import textwrap
+from typing import Any
+import pandas as pd
+from sklearn.feature_extraction.text import TfidfVectorizer
+from ...actionable import Actionable
+from ...dataset import Dataset
+from ...candidate import Candidate
+from ...decorators.all import is_step
+from ...data_type import DataType
+
+
+@is_step('cleaning')
+class ActTfIdf(Actionable):
+    """[STEP] Vectorize textual columns with TF-IDF"""
+
+    name: str = 'TF-IDF'
+    _description: str = textwrap.dedent('''\
+        Process "Term Frequency / Inversed Document Frequency"
+        over a list of textual columns''')
+    _description_long: str = textwrap.dedent('''\
+        This algorithm is used to evaluate the importance of a word inside
+        it\'s corpus. A word with a lot of repetitions will
+        have more importance than a word appearing once.''')
+    _usage: str = 'Use when short text needs weighted term features; prefer over ActCountVectorizer for damping frequent terms. Applicable to short text columns. Avoid when you need raw counts or want to drop the column (ActCountVectorizer, ActDropCategoricalColumn).'
+    refs: list[dict[str, Any]] = [
+        {
+            'year': 1972,
+            'name': 'A STATISTICAL INTERPRETATION OF TERM SPECIFICITY AND ITS APPLICATION \
+                IN RETRIEVAL',
+            'authors': [
+                'Karen Sparck Jones'    
+            ],
+            'doi': 'https://doi.org/10.1108/eb026526',
+            'publisher': 'Journal of Documentation Vol.21, No.1, page 11--21'
+        }
+    ]
+
+    def __init__(self):
+        self.columns: list[tuple[str, TfidfVectorizer]] = None
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.columns = []
+        for column in dataset.get_columns_names_by_type([DataType.SHORT_TEXT]):
+            values = dataset.X[column].fillna('')
+            vectorizer = TfidfVectorizer().fit(values)
+            self.columns.append((column, vectorizer))
+
+        feature_names = { c: v.get_feature_names_out() for c, v in self.columns }
+        self.explanations = [
+            f'Encoded text column **`{c}`** into **{len(v)}** new columns.'
+            for c, v in feature_names.items() if len(v) > 0
+        ]
+
+        return self
+
+    def suitable(self, dataset: Dataset) -> bool:
+        return bool(dataset.get_columns_names_by_type(DataType.SHORT_TEXT))
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """Apply TF-IDF on textual columns.
+
+        :param pd.DataFrame x: DataFrame to transform.
+        :return: Transformed dataset.
+        """
+        # Without Reset index, the join with vector_df will create NAN (index mismatch)
+        X = X.reset_index(drop=True)
+
+        for name, vectorizer in self.columns:
+            transformed = vectorizer.transform(X[name].fillna(''))
+
+            new_names = list(map(lambda x: "_".join([name, x]), vectorizer.get_feature_names_out())) # pylint: disable=cell-var-from-loop
+            vector_df = pd.DataFrame(transformed.todense(), columns=new_names)
+
+            X = pd.concat([X, vector_df], axis=1).drop([name], axis=1)
+
+        return X
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        return 0.4