PyIAML 1.0.0__py3-none-any.whl

iaml/statistics/rare_category_rate.py

@@ -0,0 +1,92 @@
+"""[STATISTIC] Rare Category Rate."""
+from __future__ import annotations
+
+import math
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class RareCategoryRateStatistic(Statistic):
+    """[STATISTIC] Rare Category Rate."""
+
+    name: str = "Rare Category Rate"
+    _description: str = textwrap.dedent("""\
+        Rare category rate measures the share of rare categories in categorical columns.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Rare category rate measures the ratio of categories whose frequency is below a
+        threshold for categorical columns, optionally per class.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'rare_category_rate'
+
+    def _select_columns(self, dataset: Dataset) -> list[str]:
+        columns = dataset.get_columns_names_by_type(DataType.CATEGORICAL)
+        category_columns = list(dataset.X.select_dtypes(include=['category']).columns)
+        seen = set()
+        ordered = []
+        for column in columns + category_columns:
+            if column in dataset.X.columns and column not in seen:
+                ordered.append(column)
+                seen.add(column)
+        return ordered
+
+    def _rare_ratio(self, series: pd.Series, threshold: float) -> float:
+        counts = series.value_counts(dropna=True)
+        total = int(counts.sum())
+        if total <= 0 or counts.empty:
+            return 0.0
+        if threshold <= 0:
+            return 0.0
+        ratios = counts / total
+        rare_count = int((ratios < threshold).sum())
+        return rare_count / len(counts)
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute the rare category rate for categorical columns."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        threshold = kwargs.get('threshold')
+        if threshold is None:
+            threshold = kwargs.get('min_frequency', 0.01)
+        try:
+            threshold = float(threshold)
+        except (TypeError, ValueError):
+            threshold = 0.01
+        if not math.isfinite(threshold):
+            threshold = 0.01
+
+        columns = self._select_columns(dataset)
+        if not columns:
+            return pd.DataFrame()
+
+        data = []
+        df_columns = []
+
+        if dataset.type_of_target == 'continuous':
+            for col in columns:
+                df_columns.append(col)
+                data.append(self._rare_ratio(dataset.X[col], threshold))
+            return pd.DataFrame([data], index=[str(self)], columns=df_columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col in columns:
+            for label in ['all'] + class_labels:
+                df_columns.append(f"{col}_{label}")
+                if label == 'all':
+                    values = dataset.X[col]
+                else:
+                    values = dataset.X.loc[dataset.y == label][col]
+                data.append(self._rare_ratio(values, threshold))
+        return pd.DataFrame([data], index=[str(self)], columns=df_columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/skewness.py

@@ -0,0 +1,53 @@
+"""[STATISTIC] Skewness."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class SkewnessStatistic(Statistic):
+    """[STATISTIC] Skewness."""
+
+    name: str = "Skewness"
+    _description: str = textwrap.dedent("""\
+        Skewness measures the asymmetry of numerical distributions.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Skewness measures the asymmetry of numerical distributions, per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'skewness'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute skewness for numerical columns."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        if dataset.type_of_target == 'continuous':
+            return pd.DataFrame()
+
+        class_labels = list(pd.unique(dataset.y))
+        data = []
+        columns = []
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        if label == 'all':
+                            data.append(dataset.X[col].skew())
+                        else:
+                            data.append(dataset.X.loc[dataset.y == label][col].skew())
+                    else:
+                        data.append(None)
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/stdev.py

@@ -0,0 +1,50 @@
+"""[STATISTIC] Standard deviation."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class StdevStatistic(Statistic):
+    """[STATISTIC] Standard deviation."""
+
+    name: str = "Stdev"
+    _description: str = textwrap.dedent("""\
+        Stdev measures the standard deviation of numerical columns.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Stdev measures the standard deviation of numerical columns, per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'stdev'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute standard deviation for numerical columns."""
+        if dataset.type_of_target in ['survival', 'continuous']:
+            return pd.DataFrame()
+
+        class_labels = list(pd.unique(dataset.y))
+        data = []
+        columns = []
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        if label == 'all':
+                            data.append(dataset.X[col].std())
+                        else:
+                            data.append(dataset.X.loc[dataset.y == label][col].std())
+                    else:
+                        data.append(None)
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target not in ['survival', 'continuous']

iaml/statistics/summary_table_statistic.py

@@ -0,0 +1,60 @@
+"""[STATISTIC] Summary Table."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class SummaryTableStatistic(Statistic):
+    """[STATISTIC] Summary Table."""
+
+    name: str = "Summary Table"
+    _description: str = textwrap.dedent("""\
+        Summary table reports global dataset properties.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Summary table reports global dataset properties: number of rows, number of
+        usable columns, total missing values, and memory usage.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'summary_table'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute global summary table for the dataset."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        columns = [
+            col for col in dataset.X.columns
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT)
+        ]
+
+        if columns:
+            data_frame = dataset.X[columns]
+            n_missing_total = int(data_frame.isna().sum().sum())
+            memory = int(data_frame.memory_usage(deep=True).sum())
+        else:
+            n_missing_total = 0
+            memory = 0
+
+        data = [[
+            dataset.X.shape[0],
+            len(columns),
+            n_missing_total,
+            memory,
+        ]]
+        return pd.DataFrame(
+            data,
+            index=[str(self)],
+            columns=["n_rows", "n_cols", "n_missing_total", "memory"],
+        )
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/time_by_group_statistic.py

@@ -0,0 +1,83 @@
+"""[STATISTIC] Time By Group."""
+from __future__ import annotations
+
+import textwrap
+import numpy as np
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class TimeByGroupStatistic(Statistic):
+    """[STATISTIC] Time By Group."""
+
+    name: str = "Time By Group"
+    _description: str = textwrap.dedent("""\
+        Time by group reports event rate and median time by category.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Time by group reports event rate and median survival time per category
+        for categorical features in survival targets.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'time_by_group'
+
+    def _select_columns(self, dataset: Dataset) -> list[str]:
+        columns = dataset.get_columns_names_by_type(DataType.CATEGORICAL)
+        category_columns = list(dataset.X.select_dtypes(include=['category']).columns)
+        seen = set()
+        ordered = []
+        for column in columns + category_columns:
+            if column in dataset.X.columns and column not in seen:
+                ordered.append(column)
+                seen.add(column)
+        return ordered
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute event rate and median time by categorical group."""
+        if dataset.type_of_target != 'survival':
+            return pd.DataFrame()
+
+        columns = self._select_columns(dataset)
+        if not columns:
+            return pd.DataFrame()
+
+        samples = Dataset.normalize_survival_target(dataset.y)
+        if not samples or len(samples) != len(dataset.X):
+            return pd.DataFrame()
+
+        events = np.asarray([event for event, _ in samples], dtype=float)
+        times = np.asarray([time for _, time in samples], dtype=float)
+
+        frame = dataset.X[columns].copy()
+        frame['_event'] = events
+        frame['_time'] = times
+
+        data = []
+        df_columns = []
+        for col in columns:
+            values = frame[[col, '_event', '_time']].dropna(subset=[col])
+            if values.empty:
+                continue
+            grouped = values.groupby(col, sort=True)
+            event_rates = grouped['_event'].mean()
+            median_times = grouped['_time'].median()
+            for group_value in event_rates.index:
+                value_label = str(group_value)
+                df_columns.append(f"{col}_{value_label}_event_rate")
+                data.append(float(event_rates.loc[group_value]))
+                df_columns.append(f"{col}_{value_label}_median_time")
+                data.append(float(median_times.loc[group_value]))
+
+        if not df_columns:
+            return pd.DataFrame()
+
+        return pd.DataFrame([data], index=[str(self)], columns=df_columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target == 'survival'

iaml/statistics/time_summary_statistic.py

@@ -0,0 +1,56 @@
+"""[STATISTIC] Time Summary."""
+from __future__ import annotations
+
+import textwrap
+import numpy as np
+import pandas as pd
+
+from ..dataset import Dataset
+from ..statistic import Statistic
+
+
+class TimeSummaryStatistic(Statistic):
+    """[STATISTIC] Time Summary."""
+
+    name: str = "Time Summary"
+    _description: str = textwrap.dedent("""\
+        Time summary reports key statistics of survival times.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Time summary reports min/median/max and selected quantiles of survival times.
+        """)
+    refs: list[dict] = []
+
+    quantiles = [0.1, 0.25, 0.75, 0.9]
+
+    def __str__(self) -> str:
+        return 'time_summary'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute survival time summary statistics."""
+        if dataset.type_of_target != 'survival':
+            return pd.DataFrame()
+
+        samples = Dataset.normalize_survival_target(dataset.y)
+        times = np.asarray([time for _, time in samples], dtype=float)
+
+        columns = ['time_min', 'time_median', 'time_max']
+        columns += [f"time_quantile_{quantile}" for quantile in self.quantiles]
+
+        if times.size:
+            time_min = float(np.min(times))
+            time_median = float(np.median(times))
+            time_max = float(np.max(times))
+            quantile_values = np.quantile(times, self.quantiles).astype(float).tolist()
+        else:
+            time_min = None
+            time_median = None
+            time_max = None
+            quantile_values = [None] * len(self.quantiles)
+
+        data = [[time_min, time_median, time_max] + quantile_values]
+        return pd.DataFrame(data, index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target == 'survival'

iaml/statistics/top_k_value_counts.py

@@ -0,0 +1,68 @@
+"""[STATISTIC] Top-K Value Counts."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class TopKValueCountsStatistic(Statistic):
+    """[STATISTIC] Top-K Value Counts."""
+
+    name: str = "Top-K Value Counts"
+    _description: str = textwrap.dedent("""\
+        Top-k value counts list the most frequent categories and their shares.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Top-k value counts list the most frequent categories and their shares for
+        categorical columns, optionally per class.
+        """)
+    refs: list[dict] = []
+
+    def __init__(self, k: int = 5) -> None:
+        self.k = k
+
+    def __str__(self) -> str:
+        return f'top_{self.k}_value_counts'
+
+    def _top_k_counts(self, series: pd.Series) -> list[tuple[object, int, float]]:
+        counts = series.value_counts(dropna=True)
+        if counts.empty:
+            return []
+        top_counts = counts.head(self.k)
+        total = counts.sum()
+        return [(idx, int(val), float(val) / total) for idx, val in top_counts.items()]
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute top-k value counts for categorical columns."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        data = []
+        columns = []
+
+        if dataset.type_of_target == 'continuous':
+            for col in dataset.X.columns:
+                if dataset.columns_types[col][1] == DataType.CATEGORICAL:
+                    columns.append(col)
+                    data.append(self._top_k_counts(dataset.X[col]))
+            return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] == DataType.CATEGORICAL:
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if label == 'all':
+                        values = dataset.X[col]
+                    else:
+                        values = dataset.X.loc[dataset.y == label][col]
+                    data.append(self._top_k_counts(values))
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/unique_count_statistic.py

@@ -0,0 +1,57 @@
+"""[STATISTIC] Unique Count."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class UniqueCountStatistic(Statistic):
+    """[STATISTIC] Unique Count."""
+
+    name: str = "Unique Count"
+    _description: str = textwrap.dedent("""\
+        Unique count measures the number of distinct non-null values per column.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Unique count measures the number of distinct non-null values per column,
+        optionally per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'nunique'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute unique counts for each column."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        data = []
+        columns = []
+
+        if dataset.type_of_target == 'continuous':
+            for col in dataset.X.columns:
+                if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                    columns.append(col)
+                    data.append(dataset.X[col].nunique(dropna=True))
+            return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if label == 'all':
+                        values = dataset.X[col]
+                    else:
+                        values = dataset.X.loc[dataset.y == label][col]
+                    data.append(values.nunique(dropna=True))
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/value_counts.py

@@ -0,0 +1,63 @@
+"""[STATISTIC] Value counts."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class ValueCountsStatistic(Statistic):
+    """[STATISTIC] Value counts."""
+
+    name: str = "Value Counts"
+    _description: str = textwrap.dedent("""\
+        Value counts measures value frequencies for categorical columns.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Value counts measures value frequencies for categorical columns, optionally per class.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'value_counts'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute value counts for categorical columns."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        data = []
+        columns = []
+
+        if dataset.type_of_target == 'continuous':
+            for col in dataset.X.columns:
+                if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                    columns.append(col)
+                    if dataset.columns_types[col][1] == DataType.CATEGORICAL:
+                        counts = dataset.X[col].value_counts()
+                        data.append(list((idx, val) for idx, val in counts.items()))
+                    else:
+                        data.append(None)
+            return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if dataset.columns_types[col][1] == DataType.CATEGORICAL:
+                        if label == 'all':
+                            counts = dataset.X[col].value_counts()
+                        else:
+                            counts = dataset.X.loc[dataset.y == label][col].value_counts()
+                        data.append(list((idx, val) for idx, val in counts.items()))
+                    else:
+                        data.append(None)
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/variance.py

@@ -0,0 +1,51 @@
+"""[STATISTIC] Variance."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class VarianceStatistic(Statistic):
+    """[STATISTIC] Variance."""
+
+    name: str = "Variance"
+    _description: str = textwrap.dedent("""\
+        Variance measures unbiased variance for numerical columns.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Variance measures unbiased variance for numerical columns (normalized by N-1),
+        per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'variance'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute variance for numerical columns."""
+        if dataset.type_of_target in ['survival', 'continuous']:
+            return pd.DataFrame()
+
+        class_labels = list(pd.unique(dataset.y))
+        data = []
+        columns = []
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        if label == 'all':
+                            data.append(dataset.X[col].var())
+                        else:
+                            data.append(dataset.X.loc[dataset.y == label][col].var())
+                    else:
+                        data.append(None)
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target not in ['survival', 'continuous']