PyIAML 1.0.0__py3-none-any.whl

iaml/statistics/mad_statistic.py

@@ -0,0 +1,66 @@
+"""[STATISTIC] Median Absolute Deviation."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class MADStatistic(Statistic):
+    """[STATISTIC] Median Absolute Deviation."""
+
+    name: str = "Median Absolute Deviation"
+    _description: str = textwrap.dedent("""\
+        Median absolute deviation measures the typical absolute deviation from the median.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Median absolute deviation measures the typical absolute deviation from the median,
+        optionally per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'mad'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute median absolute deviation for each numerical column."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        data = []
+        columns = []
+
+        def mad(series: pd.Series) -> float:
+            median = series.median()
+            return (series - median).abs().median()
+
+        if dataset.type_of_target == 'continuous':
+            for col in dataset.X.columns:
+                if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                    columns.append(col)
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        data.append(mad(dataset.X[col]))
+                    else:
+                        data.append(None)
+            return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        if label == 'all':
+                            data.append(mad(dataset.X[col]))
+                        else:
+                            data.append(mad(dataset.X.loc[dataset.y == label][col]))
+                    else:
+                        data.append(None)
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/mean.py

@@ -0,0 +1,61 @@
+"""[STATISTIC] Mean."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class MeanStatistic(Statistic):
+    """[STATISTIC] Mean."""
+
+    name: str = "Mean"
+    _description: str = textwrap.dedent("""\
+        Mean measures the average of numerical columns.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Mean measures the average of numerical columns, optionally per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'mean'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute mean for each numerical column."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        data = []
+        columns = []
+
+        if dataset.type_of_target == 'continuous':
+            for col in dataset.X.columns:
+                if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                    columns.append(col)
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        data.append(dataset.X[col].mean())
+                    else:
+                        data.append(None)
+            return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        if label == 'all':
+                            data.append(dataset.X[col].mean())
+                        else:
+                            data.append(dataset.X.loc[dataset.y == label][col].mean())
+                    else:
+                        data.append(None)
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/median_statistic.py

@@ -0,0 +1,61 @@
+"""[STATISTIC] Median."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class MedianStatistic(Statistic):
+    """[STATISTIC] Median."""
+
+    name: str = "Median"
+    _description: str = textwrap.dedent("""\
+        Median measures the central value of numerical columns.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Median measures the central value of numerical columns, per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'median'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute median for each numerical column."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        data = []
+        columns = []
+
+        if dataset.type_of_target == 'continuous':
+            for col in dataset.X.columns:
+                if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                    columns.append(col)
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        data.append(dataset.X[col].median())
+                    else:
+                        data.append(None)
+            return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        if label == 'all':
+                            data.append(dataset.X[col].median())
+                        else:
+                            data.append(dataset.X.loc[dataset.y == label][col].median())
+                    else:
+                        data.append(None)
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/minmax.py

@@ -0,0 +1,60 @@
+"""[STATISTIC] Min / Max."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class BoundStatistic(Statistic):
+    """[STATISTIC] Bound."""
+
+    name: str = "Bound"
+    _description: str = textwrap.dedent("""\
+        Bound measures extreme values (min/max) of numerical columns.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Bound measures extreme values (min/max) of numerical columns, per class for classification.
+        """)
+    refs: list[dict] = []
+
+    bound_to_compute = ['min', 'max']
+
+    def __init__(self, bound: str = 'min') -> None:
+        self.bound = bound
+
+    def __str__(self) -> str:
+        return self.bound
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute min/max for numerical columns."""
+        if dataset.type_of_target in ['survival', 'continuous']:
+            return pd.DataFrame()
+
+        class_labels = list(pd.unique(dataset.y))
+        df = pd.DataFrame()
+        for bound in BoundStatistic.bound_to_compute:
+            self.bound = bound
+            data = []
+            columns = []
+            for col in dataset.X.columns:
+                if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                    for label in ['all'] + class_labels:
+                        columns.append(f"{col}_{label}")
+                        if dataset.columns_types[col][1] == DataType.NUMERIC:
+                            if label == 'all':
+                                value = getattr(dataset.X[col], bound)()
+                            else:
+                                value = getattr(dataset.X.loc[dataset.y == label][col], bound)()
+                            data.append(value)
+                        else:
+                            data.append(None)
+            df = pd.concat([df, pd.DataFrame([data], index=[str(self)], columns=columns)])
+        return df
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target not in ['survival', 'continuous']

iaml/statistics/missing_rate_statistic.py

@@ -0,0 +1,62 @@
+"""[STATISTIC] Missing Rate."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class MissingRateStatistic(Statistic):
+    """[STATISTIC] Missing Rate."""
+
+    name: str = "Missing Rate"
+    _description: str = textwrap.dedent("""\
+        Missing rate measures the percentage of missing values per column.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Missing rate measures the percentage of missing values per column,
+        optionally per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'missing_rate'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute missing rate for each column."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        data = []
+        columns = []
+
+        if dataset.type_of_target == 'continuous':
+            for col in dataset.X.columns:
+                if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                    columns.append(col)
+                    values = dataset.X[col]
+                    n_rows = int(values.shape[0])
+                    rate = values.isna().mean() if n_rows else 0.0
+                    data.append(rate)
+            return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if label == 'all':
+                        values = dataset.X[col]
+                    else:
+                        values = dataset.X.loc[dataset.y == label][col]
+                    n_rows = int(values.shape[0])
+                    rate = values.isna().mean() if n_rows else 0.0
+                    data.append(rate)
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/mode.py

@@ -0,0 +1,47 @@
+"""[STATISTIC] Mode."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class ModeStatistic(Statistic):
+    """[STATISTIC] Mode."""
+
+    name: str = "Mode"
+    _description: str = textwrap.dedent("""\
+        Mode measures the most common values of each column.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Mode measures the most common values of each column, per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'mode'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute mode for each column."""
+        if dataset.type_of_target in ['survival', 'continuous']:
+            return pd.DataFrame()
+
+        class_labels = list(pd.unique(dataset.y))
+        data = []
+        columns = []
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if label == 'all':
+                        data.append(dataset.X[col].mode().to_list())
+                    else:
+                        data.append(dataset.X.loc[dataset.y == label][col].mode().to_list())
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target not in ['survival', 'continuous']

iaml/statistics/most_frequent_ratio.py

@@ -0,0 +1,81 @@
+"""[STATISTIC] Most Frequent Ratio."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class MostFrequentRatioStatistic(Statistic):
+    """[STATISTIC] Most Frequent Ratio."""
+
+    name: str = "Most Frequent Ratio"
+    _description: str = textwrap.dedent("""\
+        Most frequent ratio measures the share of the dominant category in
+        categorical columns.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Most frequent ratio measures the ratio of the most common category in
+        categorical columns, optionally per class.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'most_frequent_ratio'
+
+    def _select_columns(self, dataset: Dataset) -> list[str]:
+        columns = dataset.get_columns_names_by_type(DataType.CATEGORICAL)
+        category_columns = list(dataset.X.select_dtypes(include=['category']).columns)
+        seen = set()
+        ordered = []
+        for column in columns + category_columns:
+            if column in dataset.X.columns and column not in seen:
+                ordered.append(column)
+                seen.add(column)
+        return ordered
+
+    def _most_frequent_ratio(self, series: pd.Series) -> float:
+        counts = series.value_counts(dropna=True)
+        if counts.empty:
+            return 0.0
+        total = int(counts.sum())
+        if total <= 0:
+            return 0.0
+        top = int(counts.max())
+        return top / total
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute most frequent ratio for categorical columns."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        columns = self._select_columns(dataset)
+        if not columns:
+            return pd.DataFrame()
+
+        data = []
+        df_columns = []
+
+        if dataset.type_of_target == 'continuous':
+            for col in columns:
+                df_columns.append(col)
+                data.append(self._most_frequent_ratio(dataset.X[col]))
+            return pd.DataFrame([data], index=[str(self)], columns=df_columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col in columns:
+            for label in ['all'] + class_labels:
+                df_columns.append(f"{col}_{label}")
+                if label == 'all':
+                    values = dataset.X[col]
+                else:
+                    values = dataset.X.loc[dataset.y == label][col]
+                data.append(self._most_frequent_ratio(values))
+        return pd.DataFrame([data], index=[str(self)], columns=df_columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/outlier_count_iqr_statistic.py

@@ -0,0 +1,76 @@
+"""[STATISTIC] Outlier Count (IQR)."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class OutlierCountIQRStatistic(Statistic):
+    """[STATISTIC] Outlier Count (IQR)."""
+
+    name: str = "Outlier Count (IQR)"
+    _description: str = textwrap.dedent("""\
+        Outlier count measures how many values fall outside 1.5*IQR.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Outlier count measures how many values fall outside 1.5*IQR,
+        optionally per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'outlier_count_iqr'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute outlier counts (1.5*IQR rule) for each numerical column."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        def count_outliers(values: pd.Series) -> int:
+            values = values.dropna()
+            if values.empty:
+                return 0
+            q1 = values.quantile(0.25)
+            q3 = values.quantile(0.75)
+            iqr = q3 - q1
+            if pd.isna(iqr):
+                return 0
+            lower = q1 - 1.5 * iqr
+            upper = q3 + 1.5 * iqr
+            return int(((values < lower) | (values > upper)).sum())
+
+        data = []
+        columns = []
+
+        if dataset.type_of_target == 'continuous':
+            for col in dataset.X.columns:
+                if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                    columns.append(col)
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        data.append(count_outliers(dataset.X[col]))
+                    else:
+                        data.append(None)
+            return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        if label == 'all':
+                            values = dataset.X[col]
+                        else:
+                            values = dataset.X.loc[dataset.y == label][col]
+                        data.append(count_outliers(values))
+                    else:
+                        data.append(None)
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/quantile.py

@@ -0,0 +1,59 @@
+"""[STATISTIC] Quantile."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class QuantileStatistic(Statistic):
+    """[STATISTIC] Quantile."""
+
+    name: str = "Quantile"
+    _description: str = textwrap.dedent("""\
+        Quantile measures percentile values of numerical columns.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Quantile measures percentile values of numerical columns, per class for classification.
+        """)
+    refs: list[dict] = []
+
+    quantile_to_compute = [0.1, 0.25, 0.50, 0.75, 0.90]
+
+    def __init__(self, percentile: float = 0.5) -> None:
+        self.percentile = percentile
+
+    def __str__(self) -> str:
+        return f'quantile_{self.percentile}'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute quantiles for numerical columns."""
+        if dataset.type_of_target in ['survival', 'continuous']:
+            return pd.DataFrame()
+
+        class_labels = list(pd.unique(dataset.y))
+        df = pd.DataFrame()
+        for quantile in QuantileStatistic.quantile_to_compute:
+            self.percentile = quantile
+            data = []
+            columns = []
+            for col in dataset.X.columns:
+                if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                    for label in ['all'] + class_labels:
+                        columns.append(f"{col}_{label}")
+                        if dataset.columns_types[col][1] == DataType.NUMERIC:
+                            if label == 'all':
+                                data.append(dataset.X[col].quantile(self.percentile))
+                            else:
+                                data.append(dataset.X.loc[dataset.y == label][col].quantile(self.percentile))
+                        else:
+                            data.append(None)
+            df = pd.concat([df, pd.DataFrame([data], index=[str(self)], columns=columns)])
+        return df
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target not in ['survival', 'continuous']

iaml/statistics/range.py

@@ -0,0 +1,53 @@
+"""[STATISTIC] Range."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class RangeStatistic(Statistic):
+    """[STATISTIC] Range."""
+
+    name: str = "Range"
+    _description: str = textwrap.dedent("""\
+        Range measures the dispersion between min and max for numerical columns.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Range measures the dispersion between min and max for numerical columns,
+        per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'range'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute range for numerical columns."""
+        if dataset.type_of_target in ['survival', 'continuous']:
+            return pd.DataFrame()
+
+        class_labels = list(pd.unique(dataset.y))
+        data = []
+        columns = []
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        if label == 'all':
+                            value = dataset.X[col].max() - dataset.X[col].min()
+                        else:
+                            values = dataset.X.loc[dataset.y == label][col]
+                            value = values.max() - values.min()
+                        data.append(value)
+                    else:
+                        data.append(None)
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target not in ['survival', 'continuous']