PyIAML 1.0.0__py3-none-any.whl

iaml/actionables/features_precleaning/act_sentinel_to_na_n.py

@@ -0,0 +1,270 @@
+"""[STEP] Replace sentinel values with NaN"""
+import textwrap
+from typing import Any
+import numpy as np
+import pandas as pd
+from pandas.api.types import is_numeric_dtype, is_object_dtype, is_string_dtype
+from ...actionable import Actionable
+from ...candidate import Candidate
+from ...data_type import DataType
+from ...dataset import Dataset
+from ...decorators.all import is_step
+
+
+@is_step('features_precleaning')
+class ActSentinelToNaN(Actionable):
+    """[STEP] Replace sentinel values with NaN"""
+
+    name: str = 'Replace sentinel values'
+    _description: str = textwrap.dedent('''\
+        Replace configured sentinel values (e.g., -999, "NA", "unknown") with NaN.''')
+    _description_long: str = textwrap.dedent('''\
+        Sentinel values are placeholders for missing data.
+        This step replaces common numeric and text sentinels with NaN so downstream
+        steps can treat missing values consistently.''')
+    _usage: str = "Use when numeric or text columns contain sentinel placeholders (e.g., -999, 'NA') and you want them treated as missing; run before ActCoerceNumericStrings. Applicable to datasets with explicit sentinel codes or empty-string markers across numeric and categorical fields. Avoid when sentinel values are meaningful domain codes or you should remove sparse fields via ActDropHighMissingColumns."
+    refs: list[dict[str, Any]] = []
+
+    def __init__(self) -> None:
+        self.configuration = {
+            'numeric_sentinels': {
+                'description': textwrap.dedent('''\
+                    Numeric sentinel values to replace with NaN.'''),
+                'default': [-999, -9999, -99999]
+            },
+            'text_sentinels': {
+                'description': textwrap.dedent('''\
+                    Text sentinel values to replace with NaN.'''),
+                'default': ['NA', 'N/A', 'NULL', 'NONE', 'UNKNOWN', 'MISSING', 'NAN']
+            },
+            'case_insensitive': {
+                'description': 'Match text sentinels ignoring case.',
+                'default': True
+            },
+            'strip_whitespace': {
+                'description': 'Trim whitespace before matching text sentinels.',
+                'default': True
+            },
+            'include_empty_string': {
+                'description': 'Treat empty strings as missing values.',
+                'default': True
+            },
+            'numeric_in_text': {
+                'description': 'Also match numeric sentinels stored as text.',
+                'default': True
+            }
+        }
+        self.numeric_columns: list[str] = []
+        self.text_columns: list[str] = []
+        self.numeric_sentinels: list[float] = []
+        self.text_sentinels: list[str] = []
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.numeric_columns, self.text_columns = self.__candidate_columns(dataset)
+        self.numeric_sentinels, self.text_sentinels = self.__normalized_sentinels()
+
+        self.explanations = []
+        if dataset.X.empty:
+            return self
+
+        if self.numeric_sentinels:
+            for column in self.numeric_columns:
+                count = int(dataset.X[column].isin(self.numeric_sentinels).sum())
+                if count:
+                    self.explanations.append(
+                        f'Replaced {count} numeric sentinel values in **`{column}`**.'
+                    )
+
+        if self.text_sentinels:
+            for column in self.text_columns:
+                mask = self.__text_sentinel_mask(dataset.X[column], self.text_sentinels)
+                count = int(mask.sum())
+                if count:
+                    self.explanations.append(
+                        f'Replaced {count} text sentinel values in **`{column}`**.'
+                    )
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """Replace sentinel values with NaN.
+
+        :param pd.DataFrame X: DataFrame to transform.
+        :return: Transformed DataFrame.
+        """
+        if X.empty:
+            return X
+
+        if self.numeric_sentinels and self.numeric_columns:
+            columns = [column for column in self.numeric_columns if column in X.columns]
+            if columns:
+                X[columns] = X[columns].replace(self.numeric_sentinels, np.nan)
+
+        if self.text_sentinels and self.text_columns:
+            for column in self.text_columns:
+                if column not in X.columns:
+                    continue
+                mask = self.__text_sentinel_mask(X[column], self.text_sentinels)
+                if mask.any():
+                    X[column] = X[column].mask(mask, np.nan)
+
+        return X
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        if candidate is None or candidate.dataset.X.empty:
+            return 0.0
+
+        numeric_sentinels, text_sentinels = self.__normalized_sentinels()
+        if not numeric_sentinels and not text_sentinels:
+            return 0.0
+
+        numeric_columns, text_columns = self.__candidate_columns(candidate.dataset)
+        total = candidate.dataset.X.size or 1
+        count = self.__count_sentinels(
+            candidate.dataset,
+            numeric_columns,
+            text_columns,
+            numeric_sentinels,
+            text_sentinels
+        )
+        return min(1.5, 0.5 + count / total)
+
+    def suitable(self, dataset: Dataset) -> bool:
+        if dataset.X.empty:
+            return False
+
+        numeric_sentinels, text_sentinels = self.__normalized_sentinels()
+        if not numeric_sentinels and not text_sentinels:
+            return False
+
+        numeric_columns, text_columns = self.__candidate_columns(dataset)
+        if numeric_sentinels:
+            for column in numeric_columns:
+                if dataset.X[column].isin(numeric_sentinels).any():
+                    return True
+
+        if text_sentinels:
+            for column in text_columns:
+                if self.__text_sentinel_mask(dataset.X[column], text_sentinels).any():
+                    return True
+
+        return False
+
+    def __candidate_columns(self, dataset: Dataset) -> tuple[list[str], list[str]]:
+        numeric_candidates = set(dataset.get_columns_names_by_type(DataType.NUMERIC))
+        text_candidates = set(dataset.get_columns_names_by_type(
+            [DataType.CATEGORICAL, DataType.TEXT, DataType.SHORT_TEXT]
+        ))
+
+        numeric_columns = []
+        text_columns = []
+
+        for column in dataset.X.columns:
+            if column in numeric_candidates:
+                numeric_columns.append(column)
+                continue
+            if column in text_candidates:
+                text_columns.append(column)
+                continue
+
+            series = dataset.X[column]
+            if is_numeric_dtype(series):
+                numeric_columns.append(column)
+            elif is_string_dtype(series) or is_object_dtype(series):
+                text_columns.append(column)
+
+        return numeric_columns, text_columns
+
+    def __normalized_sentinels(self) -> tuple[list[float], list[str]]:
+        numeric_sentinels = self.__normalize_numeric_sentinels()
+        text_sentinels = self.__normalize_text_sentinels(numeric_sentinels)
+        return numeric_sentinels, text_sentinels
+
+    def __normalize_numeric_sentinels(self) -> list[float]:
+        values = self.get_config('numeric_sentinels') or []
+        cleaned: list[float] = []
+        seen: set[float] = set()
+
+        for value in values:
+            if value is None or isinstance(value, bool):
+                continue
+            try:
+                num = float(value)
+            except (TypeError, ValueError):
+                continue
+            if np.isnan(num) or num in seen:
+                continue
+            cleaned.append(num)
+            seen.add(num)
+
+        return cleaned
+
+    def __normalize_text_sentinels(self, numeric_sentinels: list[float]) -> list[str]:
+        values = list(self.get_config('text_sentinels') or [])
+        if self.get_config('include_empty_string'):
+            values.append('')
+        if self.get_config('numeric_in_text'):
+            values.extend(self.__numeric_sentinels_as_text(numeric_sentinels))
+
+        cleaned: list[str] = []
+        seen: set[str] = set()
+
+        for value in values:
+            if value is None:
+                continue
+            text = str(value)
+            if self.get_config('strip_whitespace'):
+                text = text.strip()
+            if self.get_config('case_insensitive'):
+                text = text.lower()
+            if text == '' and not self.get_config('include_empty_string'):
+                continue
+            if text not in seen:
+                cleaned.append(text)
+                seen.add(text)
+
+        return cleaned
+
+    def __numeric_sentinels_as_text(self, numeric_sentinels: list[float]) -> list[str]:
+        values: list[str] = []
+        for value in numeric_sentinels:
+            if np.isnan(value):
+                continue
+            if float(value).is_integer():
+                int_value = int(value)
+                values.append(str(int_value))
+                values.append(str(float(int_value)))
+            else:
+                values.append(str(value))
+        return values
+
+    def __text_sentinel_mask(self, series: pd.Series, sentinels: list[str]) -> pd.Series:
+        if not sentinels or series.empty:
+            return pd.Series(False, index=series.index)
+
+        values = series.astype('string')
+        if self.get_config('strip_whitespace'):
+            values = values.str.strip()
+        if self.get_config('case_insensitive'):
+            values = values.str.lower()
+        mask = values.isin(sentinels)
+        return mask.fillna(False)
+
+    def __count_sentinels(self,
+        dataset: Dataset,
+        numeric_columns: list[str],
+        text_columns: list[str],
+        numeric_sentinels: list[float],
+        text_sentinels: list[str]
+    ) -> int:
+        count = 0
+        if numeric_sentinels:
+            for column in numeric_columns:
+                count += int(dataset.X[column].isin(numeric_sentinels).sum())
+        if text_sentinels:
+            for column in text_columns:
+                count += int(self.__text_sentinel_mask(
+                    dataset.X[column],
+                    text_sentinels
+                ).sum())
+        return count

iaml/actionables/features_precleaning/act_trim_space.py

@@ -0,0 +1,79 @@
+"""[STEP] Trim spaces on each columns"""
+import textwrap
+from pandas.api.types import is_object_dtype
+import pandas as pd
+from ...actionable import Actionable
+from ...dataset import Dataset
+from ...candidate import Candidate
+from ...decorators.all import is_step
+
+@is_step('features_precleaning')
+class ActTrimSpaces(Actionable):
+    """
+    [STEP] Trim spaces on each columns
+    """
+    name = "Trim columns spaces"
+    _description = textwrap.dedent('''\
+        This step trim spaces on each columns.
+        It helps preventing errors on the dataset when casting columns with spaces''')
+    _description_long = textwrap.dedent('''\
+        In datasets, columns with spaces can be an issue.
+        This step remove spaces in front and back of columns values.
+        This ensures that columns can be casted correctly with having spaces throwing
+        an error.''')
+    _usage = "Use when columns or string values have leading/trailing spaces; use before ActCoerceNumericStrings or ActNormalizeColumnNames. Applicable to object/string columns and column names. Avoid when spaces are meaningful or data is already clean."
+
+    refs = []
+
+    def __init__(self):
+        self.configuration = {
+            'left_trim': {
+                'description': "Trim all columns leading spaces",
+                'default': True
+            },
+            'right_trim': {
+                'description': "Trim all columns ending spaces",
+                'default': True
+            },
+        }
+
+    def fit(self, dataset: Dataset):  # pylint: disable=unused-argument
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """Apply the dropping of rows with at least 40% empty columns.
+        
+        :param pd.DataFrame X: The dataframe to clean.
+        :return: The cleaned dataframe.
+        """
+        if self.get_config('left_trim'):
+            X = X.rename(columns=lambda x: x.lstrip())
+            for col in X.columns:
+                if isinstance(X[col].dtype, str) or is_object_dtype(X[col]):
+                    X[col] = X[col].apply(
+                        lambda x: x.lstrip() if isinstance(x, str) else x,
+                        convert_dtype=False)
+
+        if self.get_config('right_trim'):
+            X = X.rename(columns=lambda x: x.rstrip())
+            for col in X.columns:
+                if isinstance(X[col].dtype, str) or is_object_dtype(X[col]):
+                    X[col] = X[col].astype(object).apply(
+                        lambda x: x.rstrip() if isinstance(x, str) else x
+                    )
+
+        return X
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        return 1.5
+
+    def suitable(self, dataset: Dataset) -> bool:
+        cond = (
+            dataset.X.apply(
+                lambda x: (
+                    x.astype(str).str.strip() if (isinstance(x, str) or is_object_dtype(x)) else x
+                ) != x
+            ).stack().any()
+            or (dataset.X.columns.str.strip() != dataset.X.columns).any()
+        )
+        return cond

iaml/actionables/features_preprocessing/__init__.py

@@ -0,0 +1,18 @@
+"""
+Usually last step before predictor, try features decomposition or aggregation.
+Example : PCA
+"""
+from .act_pca import ActPCA
+from .act_kernel_pca import ActKernelPCA
+from .act_feature_agglomeration import ActFeatureAgglomeration
+from .act_nystroem import ActNystroem
+from .act_rbf_sampler import ActRBFSampler
+from .act_select_percentile import ActSelectPercentile
+from .act_power_transformer import ActPowerTransformer
+from .act_quantile_transformer import ActQuantileTransformer
+from .act_truncated_svd import ActTruncatedSVD
+from .act_fast_ica import ActFastICA
+from .act_sparse_random_projection import ActSparseRandomProjection
+from .act_k_bins_discretizer import ActKBinsDiscretizer
+from .act_log_transformer import ActLogTransformer
+from .act_k_means_features import ActKMeansFeatures

iaml/actionables/features_preprocessing/act_cyclical_date_encoding.py

@@ -0,0 +1,212 @@
+"""Manual cyclical date encoding, available only through an explicit module import.
+
+Date conversion and cleaning precede automatic feature preprocessing, so this
+component needs an explicit position while datetime columns are still available.
+See docs/component_status.rst.
+"""
+import textwrap
+import numpy as np
+import pandas as pd
+from ...actionable import Actionable
+from ...candidate import Candidate
+from ...dataset import Dataset
+from ...data_type import DataType
+from ...decorators.all import is_step
+
+
+@is_step('experimental')
+class ActCyclicalDateEncoding(Actionable):
+    """[STEP] Encode date columns with cyclical sine/cosine features."""
+
+    name: str = "Cyclical Date Encoding"
+    _description: str = "Encode date columns into sine/cosine features to capture cyclicity"
+    _usage: str = "Use when datetime columns have cyclic parts (month/weekday/hour) and models need numeric features. Applicable to datetime columns with clear periodicity. Avoid when dates are non-cyclic or already encoded; consider ActKBinsDiscretizer or ActLogTransformer instead."
+    _description_long: str = textwrap.dedent('''\
+        Cyclical encoding turns calendar components (month, weekday, hour, etc.)
+        into sine and cosine values. This preserves the circular nature of time,
+        so end and start points on a cycle stay close in feature space while
+        providing numeric values suitable for ML models.
+    ''')
+
+    _COMPONENTS: dict[str, tuple[str, int]] = {
+        'month': ('month', 12),
+        'dayofweek': ('dayofweek', 7),
+        'day': ('day', 31),
+        'hour': ('hour', 24),
+        'minute': ('minute', 60),
+        'second': ('second', 60),
+    }
+    _ALIASES: dict[str, str] = {
+        'weekday': 'dayofweek',
+        'dow': 'dayofweek',
+        'dayofmonth': 'day',
+        'dom': 'day',
+    }
+
+    def __init__(self) -> None:
+        self.configuration = {
+            'components': {
+                'description': 'Date components to encode (month, dayofweek, day, hour, minute, second).',
+                'default': ('month', 'dayofweek', 'day', 'hour')
+            },
+            'drop_original': {
+                'description': 'Drop original date columns after encoding.',
+                'default': True
+            }
+        }
+        self.columns: list[str] = []
+        self.encoding_plan: dict[str, list[tuple[str, str, int, str, str]]] = {}
+        self.drop_original: bool = True
+
+    @staticmethod
+    def _is_datetime(series: pd.Series) -> bool:
+        return pd.api.types.is_datetime64_any_dtype(series)
+
+    @staticmethod
+    def _coerce_bool(value: object, default: bool) -> bool:
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, str):
+            lowered = value.strip().lower()
+            if lowered in ('1', 'true', 'yes', 'y'):
+                return True
+            if lowered in ('0', 'false', 'no', 'n'):
+                return False
+        return default
+
+    @staticmethod
+    def _unique_name(name: str, reserved: set[str]) -> str:
+        if name not in reserved:
+            return name
+        idx = 1
+        candidate = f"{name}_{idx}"
+        while candidate in reserved:
+            idx += 1
+            candidate = f"{name}_{idx}"
+        return candidate
+
+    def _resolve_components(self) -> list[str]:
+        raw = self.get_config('components')
+        if raw is None:
+            raw_components = list(self._COMPONENTS.keys())
+        elif isinstance(raw, str):
+            raw_components = [raw]
+        else:
+            try:
+                raw_components = list(raw)
+            except TypeError:
+                raw_components = [str(raw)]
+
+        components: list[str] = []
+        seen: set[str] = set()
+        for component in raw_components:
+            if component is None:
+                continue
+            component = str(component).strip().lower()
+            if not component:
+                continue
+            component = self._ALIASES.get(component, component)
+            if component not in self._COMPONENTS:
+                continue
+            if component in seen:
+                continue
+            seen.add(component)
+            components.append(component)
+
+        if not components:
+            components = list(self._COMPONENTS.keys())
+
+        return components
+
+    def _build_plan(self, dataset: Dataset) -> dict[str, list[tuple[str, str, int, str, str]]]:
+        columns = dataset.get_columns_names_by_type(DataType.DATE)
+        if not columns or dataset.X.empty:
+            return {}
+
+        components = self._resolve_components()
+        if not components:
+            return {}
+
+        reserved = set(dataset.X.columns)
+        plan: dict[str, list[tuple[str, str, int, str, str]]] = {}
+
+        for column in columns:
+            if column not in dataset.X.columns:
+                continue
+            series = dataset.X[column]
+            if not self._is_datetime(series):
+                continue
+
+            column_plan: list[tuple[str, str, int, str, str]] = []
+            for component in components:
+                accessor, period = self._COMPONENTS[component]
+                values = getattr(series.dt, accessor)
+                if values.nunique(dropna=True) <= 1:
+                    continue
+                base = f"{column}_{component}"
+                sin_name = self._unique_name(f"{base}_sin", reserved)
+                reserved.add(sin_name)
+                cos_name = self._unique_name(f"{base}_cos", reserved)
+                reserved.add(cos_name)
+                column_plan.append((component, accessor, period, sin_name, cos_name))
+
+            if column_plan:
+                plan[column] = column_plan
+
+        return plan
+
+    def fit(self, dataset: Dataset) -> Actionable:
+        self.encoding_plan = self._build_plan(dataset)
+        self.columns = list(self.encoding_plan.keys())
+        self.drop_original = self._coerce_bool(self.get_config('drop_original'), True)
+        self.explanations = []
+
+        for column, parts in self.encoding_plan.items():
+            components = ", ".join([part[0] for part in parts])
+            self.explanations.append(
+                f'Encoded date column **`{column}`** into cyclical features ({components}).'
+            )
+
+        return self
+
+    def transform(self, X: pd.DataFrame) -> pd.DataFrame:
+        """Apply cyclical encoding to date columns.
+
+        :param pd.DataFrame X: DataFrame to transform
+        :return: Transformed dataset
+        """
+        if not self.encoding_plan:
+            return X
+
+        for column, parts in self.encoding_plan.items():
+            if column not in X.columns:
+                continue
+            series = X[column]
+            if not self._is_datetime(series):
+                series = pd.to_datetime(series, errors='coerce')
+
+            for _, accessor, period, sin_name, cos_name in parts:
+                values = getattr(series.dt, accessor).astype(float)
+                angles = (2.0 * np.pi * values) / float(period)
+                X[sin_name] = np.sin(angles)
+                X[cos_name] = np.cos(angles)
+
+        if self.drop_original and self.columns:
+            to_drop = [column for column in self.columns if column in X.columns]
+            if to_drop:
+                X = X.drop(columns=to_drop)
+
+        return X
+
+    def suitable(self, dataset: Dataset) -> bool:
+        if dataset.X.empty:
+            return False
+        return bool(self._build_plan(dataset))
+
+    def priorize(self, candidate: Candidate = None) -> float:
+        if candidate is None:
+            return 0.0
+        dataset = candidate.dataset
+        if dataset.X.empty:
+            return 0.0
+        return 0.5 if self._build_plan(dataset) else 0.0