PyIAML 1.0.0__py3-none-any.whl

iaml/statistic.py

@@ -0,0 +1,66 @@
+"""[STATISTIC] Parent of descriptive statistics."""
+from __future__ import annotations
+
+from typing import Any
+import pandas as pd
+
+from .dataset import Dataset
+from .reference import Reference
+
+
+class Statistic:
+    """[STATISTIC] Parent of descriptive statistics."""
+
+    name: str = ""
+    """Name of the statistic."""
+
+    _description: str = ""
+    """Short description of the statistic."""
+
+    _description_long: str = ""
+    """Long description of the statistic."""
+
+    refs: list[dict[str, Any]] = []
+    """List of references for this statistic."""
+
+    @classmethod
+    def all_subclasses(cls) -> list['Statistic']:
+        """Return all statistic subclasses."""
+        subclasses = cls.__subclasses__()
+        for subclass in subclasses:
+            subclasses += subclass.all_subclasses()
+        return subclasses
+
+    @classmethod
+    def get_refs(cls) -> list[Reference]:
+        """Get bibliography references."""
+        if hasattr(cls, 'refs'):
+            return [Reference(ref, cls.__name__) for ref in cls.refs]
+        return []
+
+    @property
+    def description(self) -> str:
+        """Return the short description."""
+        return self._description.replace('\n', '')
+
+    @property
+    def description_long(self) -> str:
+        """Return the long description."""
+        return self._description_long.replace('\n', '')
+
+    def explain(self) -> str:
+        """Describe statistic."""
+        return self.description_long
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute the statistic on the dataset."""
+        raise NotImplementedError('Subclass must implement abstract method')
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Return True if the statistic applies to the dataset."""
+        return False
+
+    @property
+    def name(self) -> str:
+        """Return the statistic formatted name."""
+        return ' '.join(x.title() for x in str(self).split('_'))

iaml/statistics/__init__.py

@@ -0,0 +1,77 @@
+"""All statistics to describe original dataset."""
+from .count import CountStatistic
+from .mean import MeanStatistic
+from .minmax import BoundStatistic
+from .mode import ModeStatistic
+from .quantile import QuantileStatistic
+from .range import RangeStatistic
+from .stdev import StdevStatistic
+from .value_counts import ValueCountsStatistic
+from .variance import VarianceStatistic
+from .violin import ViolinStatistic
+
+from .summary_table_statistic import SummaryTableStatistic
+from .data_type_summary_statistic import DataTypeSummaryStatistic
+from .duplicate_row_statistic import DuplicateRowStatistic
+from .unique_count_statistic import UniqueCountStatistic
+from .missing_rate_statistic import MissingRateStatistic
+from .cardinality_ratio_statistic import CardinalityRatioStatistic
+from .median_statistic import MedianStatistic
+from .iqr_statistic import IQRStatistic
+from .mad_statistic import MADStatistic
+from .coef_variation_statistic import CoefVariationStatistic
+from .outlier_count_iqr_statistic import OutlierCountIQRStatistic
+from .skewness import SkewnessStatistic
+from .kurtosis import KurtosisStatistic
+from .top_k_value_counts import TopKValueCountsStatistic
+from .entropy_statistic import EntropyStatistic
+from .rare_category_rate import RareCategoryRateStatistic
+from .most_frequent_ratio import MostFrequentRatioStatistic
+from .category_cooccurrence_statistic import CategoryCooccurrenceStatistic
+from .grouped_mean_statistic import GroupedMeanStatistic
+from .effect_size_statistic import EffectSizeStatistic
+from .anova_statistic import ANOVAStatistic
+from .chi_square_statistic import ChiSquareStatistic
+from .correlation_with_target import CorrelationWithTargetStatistic
+from .event_rate_statistic import EventRateStatistic
+from .time_summary_statistic import TimeSummaryStatistic
+from .time_by_group_statistic import TimeByGroupStatistic
+
+__all__ = [
+    "CountStatistic",
+    "MeanStatistic",
+    "BoundStatistic",
+    "ModeStatistic",
+    "QuantileStatistic",
+    "RangeStatistic",
+    "StdevStatistic",
+    "ValueCountsStatistic",
+    "VarianceStatistic",
+    "ViolinStatistic",
+    "SummaryTableStatistic",
+    "DataTypeSummaryStatistic",
+    "DuplicateRowStatistic",
+    "UniqueCountStatistic",
+    "MissingRateStatistic",
+    "CardinalityRatioStatistic",
+    "MedianStatistic",
+    "IQRStatistic",
+    "MADStatistic",
+    "CoefVariationStatistic",
+    "OutlierCountIQRStatistic",
+    "SkewnessStatistic",
+    "KurtosisStatistic",
+    "TopKValueCountsStatistic",
+    "EntropyStatistic",
+    "RareCategoryRateStatistic",
+    "MostFrequentRatioStatistic",
+    "CategoryCooccurrenceStatistic",
+    "GroupedMeanStatistic",
+    "EffectSizeStatistic",
+    "ANOVAStatistic",
+    "ChiSquareStatistic",
+    "CorrelationWithTargetStatistic",
+    "EventRateStatistic",
+    "TimeSummaryStatistic",
+    "TimeByGroupStatistic",
+]

iaml/statistics/anova_statistic.py

@@ -0,0 +1,80 @@
+"""[STATISTIC] ANOVA."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+from scipy import stats
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class ANOVAStatistic(Statistic):
+    """[STATISTIC] ANOVA."""
+
+    name: str = "ANOVA"
+    _description: str = textwrap.dedent("""\
+        ANOVA tests whether numerical feature means differ across classes.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        ANOVA computes one-way p-values for numerical columns across classes.
+        Per-class columns report ANOVA p-values comparing each class against the
+        rest of the data, while the `_all` suffix reports the overall ANOVA p-value.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'anova'
+
+    def _anova_pvalue(self, groups: list[pd.Series]) -> float | None:
+        cleaned = []
+        for values in groups:
+            if values is None:
+                continue
+            series = values.dropna()
+            if series.size > 1:
+                cleaned.append(series.to_numpy())
+        if len(cleaned) < 2:
+            return None
+        result = stats.f_oneway(*cleaned)
+        pvalue = float(result.pvalue)
+        if pd.isna(pvalue):
+            return None
+        return pvalue
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute ANOVA p-values for numerical columns."""
+        if dataset.type_of_target in ['survival', 'continuous']:
+            return pd.DataFrame()
+
+        numeric_columns = dataset.get_columns_names_by_type(DataType.NUMERIC)
+        if not numeric_columns:
+            return pd.DataFrame()
+
+        class_labels = list(pd.unique(dataset.y))
+        class_masks = {label: (dataset.y == label) for label in class_labels}
+
+        data = []
+        columns = []
+
+        for col in numeric_columns:
+            col_values = dataset.X[col]
+            overall_groups = [col_values[mask] for mask in class_masks.values()]
+            overall_pvalue = self._anova_pvalue(overall_groups)
+
+            for label in ['all'] + class_labels:
+                columns.append(f"{col}_{label}")
+                if label == 'all':
+                    data.append(overall_pvalue)
+                else:
+                    mask = class_masks[label]
+                    class_values = col_values[mask]
+                    rest_values = col_values[~mask]
+                    data.append(self._anova_pvalue([class_values, rest_values]))
+
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target not in ['survival', 'continuous']

iaml/statistics/cardinality_ratio_statistic.py

@@ -0,0 +1,63 @@
+"""[STATISTIC] Cardinality Ratio."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class CardinalityRatioStatistic(Statistic):
+    """[STATISTIC] Cardinality Ratio."""
+
+    name: str = "Cardinality Ratio"
+    _description: str = textwrap.dedent("""\
+        Cardinality ratio measures the ratio of unique values to total rows.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Cardinality ratio measures the ratio of unique values to total rows,
+        optionally per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'cardinality_ratio'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute cardinality ratios for each column."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        data = []
+        columns = []
+
+        if dataset.type_of_target == 'continuous':
+            n_rows = int(dataset.X.shape[0])
+            for col in dataset.X.columns:
+                if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                    columns.append(col)
+                    n_unique = dataset.X[col].nunique(dropna=True)
+                    ratio = n_unique / n_rows if n_rows else 0.0
+                    data.append(ratio)
+            return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if label == 'all':
+                        values = dataset.X[col]
+                    else:
+                        values = dataset.X.loc[dataset.y == label][col]
+                    n_rows = int(values.shape[0])
+                    n_unique = values.nunique(dropna=True)
+                    ratio = n_unique / n_rows if n_rows else 0.0
+                    data.append(ratio)
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/category_cooccurrence_statistic.py

@@ -0,0 +1,79 @@
+"""[STATISTIC] Category Cooccurrence."""
+from __future__ import annotations
+
+import itertools
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class CategoryCooccurrenceStatistic(Statistic):
+    """[STATISTIC] Category Cooccurrence."""
+
+    name: str = "Category Cooccurrence"
+    _description: str = textwrap.dedent("""\
+        Category cooccurrence counts value pairs between categorical column pairs.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Category cooccurrence counts value pairs between categorical column pairs,
+        optionally per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'category_cooccurrence'
+
+    def _select_columns(self, dataset: Dataset) -> list[str]:
+        columns = dataset.get_columns_names_by_type(DataType.CATEGORICAL)
+        category_columns = list(dataset.X.select_dtypes(include=['category']).columns)
+        seen = set()
+        ordered = []
+        for column in columns + category_columns:
+            if column in dataset.X.columns and column not in seen:
+                ordered.append(column)
+                seen.add(column)
+        return ordered
+
+    def _cooccurrence(self, frame: pd.DataFrame, col_a: str, col_b: str) -> list[tuple[tuple, int]]:
+        values = frame[[col_a, col_b]].dropna()
+        if values.empty:
+            return []
+        counts = values.groupby([col_a, col_b], sort=True).size()
+        return [((idx_a, idx_b), int(count)) for (idx_a, idx_b), count in counts.items()]
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute category cooccurrence for categorical column pairs."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        columns = self._select_columns(dataset)
+        if len(columns) < 2:
+            return pd.DataFrame()
+
+        pairs = list(itertools.combinations(columns, 2))
+        data = []
+        df_columns = []
+
+        if dataset.type_of_target == 'continuous':
+            for col_a, col_b in pairs:
+                df_columns.append(f"{col_a}__{col_b}")
+                data.append(self._cooccurrence(dataset.X, col_a, col_b))
+            return pd.DataFrame([data], index=[str(self)], columns=df_columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col_a, col_b in pairs:
+            for label in ['all'] + class_labels:
+                df_columns.append(f"{col_a}__{col_b}_{label}")
+                if label == 'all':
+                    frame = dataset.X
+                else:
+                    frame = dataset.X.loc[dataset.y == label]
+                data.append(self._cooccurrence(frame, col_a, col_b))
+        return pd.DataFrame([data], index=[str(self)], columns=df_columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/chi_square_statistic.py

@@ -0,0 +1,81 @@
+"""[STATISTIC] Chi-square."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+from scipy.stats import chi2_contingency
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class ChiSquareStatistic(Statistic):
+    """[STATISTIC] Chi-square."""
+
+    name: str = "Chi-square"
+    _description: str = textwrap.dedent("""\
+        Chi-square measures association between categorical columns and classes.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Chi-square computes the chi-square test statistic for each categorical column
+        against the target classes. The `_all` suffix reports the multi-class statistic,
+        while per-class values compare a class against the rest.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'chi_square'
+
+    def _select_columns(self, dataset: Dataset) -> list[str]:
+        columns = dataset.get_columns_names_by_type(DataType.CATEGORICAL)
+        category_columns = list(dataset.X.select_dtypes(include=['category']).columns)
+        seen = set()
+        ordered = []
+        for column in columns + category_columns:
+            if column in dataset.X.columns and column not in seen:
+                ordered.append(column)
+                seen.add(column)
+        return ordered
+
+    def _chi2_statistic(self, feature: pd.Series, target: pd.Series) -> float | None:
+        if not isinstance(target, pd.Series):
+            target = pd.Series(target, index=feature.index)
+        mask = feature.notna() & target.notna()
+        if not mask.any():
+            return None
+        table = pd.crosstab(feature[mask], target[mask])
+        if table.empty or table.shape[0] < 2 or table.shape[1] < 2:
+            return None
+        stat, _, _, _ = chi2_contingency(table, correction=False)
+        if pd.isna(stat):
+            return None
+        return float(stat)
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute chi-square statistics for categorical columns."""
+        if dataset.type_of_target in ['survival', 'continuous']:
+            return pd.DataFrame()
+
+        columns = self._select_columns(dataset)
+        if not columns:
+            return pd.DataFrame()
+
+        class_labels = list(pd.unique(dataset.y))
+        data = []
+        df_columns = []
+
+        for col in columns:
+            for label in ['all'] + class_labels:
+                df_columns.append(f"{col}_{label}")
+                if label == 'all':
+                    data.append(self._chi2_statistic(dataset.X[col], dataset.y))
+                else:
+                    binary_target = pd.Series(dataset.y == label, index=dataset.X.index)
+                    data.append(self._chi2_statistic(dataset.X[col], binary_target))
+
+        return pd.DataFrame([data], index=[str(self)], columns=df_columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target not in ['survival', 'continuous']

iaml/statistics/coef_variation_statistic.py

@@ -0,0 +1,72 @@
+"""[STATISTIC] Coefficient of variation."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class CoefVariationStatistic(Statistic):
+    """[STATISTIC] Coefficient of variation."""
+
+    name: str = "Coef Variation"
+    _description: str = textwrap.dedent("""\
+        Coefficient of variation measures relative dispersion (std/mean).
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Coefficient of variation measures relative dispersion (std/mean),
+        optionally per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'coef_variation'
+
+    def _coef_variation(self, values: pd.Series) -> float | None:
+        mean = values.mean()
+        if pd.isna(mean) or mean == 0:
+            return None
+        stdev = values.std()
+        if pd.isna(stdev):
+            return None
+        return stdev / mean
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute coefficient of variation for numerical columns."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        data = []
+        columns = []
+
+        if dataset.type_of_target == 'continuous':
+            for col in dataset.X.columns:
+                if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                    columns.append(col)
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        data.append(self._coef_variation(dataset.X[col]))
+                    else:
+                        data.append(None)
+            return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        if label == 'all':
+                            values = dataset.X[col]
+                        else:
+                            values = dataset.X.loc[dataset.y == label][col]
+                        data.append(self._coef_variation(values))
+                    else:
+                        data.append(None)
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/correlation_with_target.py

@@ -0,0 +1,105 @@
+"""[STATISTIC] Correlation with target."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class CorrelationWithTargetStatistic(Statistic):
+    """[STATISTIC] Correlation with target."""
+
+    name: str = "Correlation with target"
+    _description: str = textwrap.dedent("""\
+        Correlation (continuous targets) or eta (classification) for numeric columns.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        For continuous targets, this reports Pearson correlations between each numeric
+        feature and the target. For classification targets, this reports the eta
+        correlation ratio between each numeric feature and the target classes. The
+        `_all` suffix reports the multi-class statistic, while per-class values compare
+        a class against the rest.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'correlation_with_target'
+
+    def _pearson_corr(self, feature: pd.Series, target: pd.Series) -> float | None:
+        if not isinstance(target, pd.Series):
+            target = pd.Series(target, index=feature.index)
+        values = pd.to_numeric(feature, errors='coerce')
+        target_values = pd.to_numeric(target, errors='coerce')
+        mask = values.notna() & target_values.notna()
+        if mask.sum() < 2:
+            return None
+        if values[mask].nunique() < 2 or target_values[mask].nunique() < 2:
+            return None
+        corr = values[mask].corr(target_values[mask])
+        if pd.isna(corr):
+            return None
+        return float(corr)
+
+    def _eta(self, feature: pd.Series, target: pd.Series) -> float | None:
+        if not isinstance(target, pd.Series):
+            target = pd.Series(target, index=feature.index)
+        mask = feature.notna() & target.notna()
+        if not mask.any():
+            return None
+        values = feature[mask]
+        groups = target[mask]
+        if groups.nunique() < 2:
+            return None
+        overall_mean = values.mean()
+        ss_total = ((values - overall_mean) ** 2).sum()
+        if ss_total == 0:
+            return None
+        ss_between = 0.0
+        for _, group_values in values.groupby(groups):
+            count = group_values.size
+            if count == 0:
+                continue
+            mean = group_values.mean()
+            ss_between += count * (mean - overall_mean) ** 2
+        eta_sq = ss_between / ss_total
+        if pd.isna(eta_sq):
+            return None
+        return float(eta_sq) ** 0.5
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute correlations or eta with the target for numeric columns."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        numeric_columns = dataset.get_columns_names_by_type(DataType.NUMERIC)
+        if not numeric_columns:
+            return pd.DataFrame()
+
+        data = []
+        columns = []
+
+        if dataset.type_of_target == 'continuous':
+            for col in numeric_columns:
+                columns.append(col)
+                data.append(self._pearson_corr(dataset.X[col], dataset.y))
+            return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col in numeric_columns:
+            overall_eta = self._eta(dataset.X[col], dataset.y)
+            for label in ['all'] + class_labels:
+                columns.append(f"{col}_{label}")
+                if label == 'all':
+                    data.append(overall_eta)
+                else:
+                    binary_target = pd.Series(dataset.y == label, index=dataset.X.index)
+                    data.append(self._eta(dataset.X[col], binary_target))
+
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'