PyIAML 1.0.0__py3-none-any.whl

iaml/statistics/count.py

@@ -0,0 +1,72 @@
+"""[STATISTIC] Count."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class CountStatistic(Statistic):
+    """[STATISTIC] Count."""
+
+    name: str = "Count"
+    _description: str = textwrap.dedent("""\
+        Count measures the number of non-null values for each column.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Count measures the number of non-null values (or null values when requested)
+        for each column.
+        """)
+    refs: list[dict] = []
+
+    count_to_compute = [True, False]
+
+    def __init__(self, null_count: bool = False) -> None:
+        self.null_count = null_count
+
+    def __str__(self) -> str:
+        return 'null_count' if self.null_count else 'count'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute counts for each column."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        df = pd.DataFrame()
+        if dataset.type_of_target == 'continuous':
+            for cnt in CountStatistic.count_to_compute:
+                self.null_count = cnt
+                data = []
+                columns = []
+                for col in dataset.X.columns:
+                    if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                        columns.append(col)
+                        value = dataset.X[col].isna().sum() if self.null_count else dataset.X[col].count()
+                        data.append(value)
+                df = pd.concat([df, pd.DataFrame([data], index=[str(self)], columns=columns)])
+            return df
+
+        class_labels = list(pd.unique(dataset.y))
+        for cnt in CountStatistic.count_to_compute:
+            self.null_count = cnt
+            data = []
+            columns = []
+            for col in dataset.X.columns:
+                if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                    for label in ['all'] + class_labels:
+                        columns.append(f"{col}_{label}")
+                        if label == 'all':
+                            value = dataset.X[col].isna().sum() if self.null_count else dataset.X[col].count()
+                        else:
+                            values = dataset.X.loc[dataset.y == label][col]
+                            value = values.isna().sum() if self.null_count else values.count()
+                        data.append(value)
+            df = pd.concat([df, pd.DataFrame([data], index=[str(self)], columns=columns)])
+        return df
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/data_type_summary_statistic.py

@@ -0,0 +1,74 @@
+"""[STATISTIC] Data Type Summary."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class DataTypeSummaryStatistic(Statistic):
+    """[STATISTIC] Data Type Summary."""
+
+    name: str = "Data Type Summary"
+    _description: str = textwrap.dedent("""\
+        Data type summary counts column types and ratios.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Data type summary reports the number of numeric, categorical, text, and date
+        columns, along with ratios relative to the total number of columns.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'data_type_summary'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute data type counts and ratios for the dataset."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        total_columns = len(dataset.X.columns)
+        n_numeric = len(dataset.get_columns_names_by_type(DataType.NUMERIC))
+        n_categorical = len(dataset.get_columns_names_by_type(DataType.CATEGORICAL))
+        n_text = len(dataset.get_columns_names_by_type([DataType.TEXT, DataType.SHORT_TEXT]))
+        n_date = len(dataset.get_columns_names_by_type(DataType.DATE))
+
+        if total_columns:
+            ratio_numeric = n_numeric / total_columns
+            ratio_categorical = n_categorical / total_columns
+            ratio_text = n_text / total_columns
+            ratio_date = n_date / total_columns
+        else:
+            ratio_numeric = 0.0
+            ratio_categorical = 0.0
+            ratio_text = 0.0
+            ratio_date = 0.0
+
+        data = [[
+            n_numeric,
+            n_categorical,
+            n_text,
+            n_date,
+            ratio_numeric,
+            ratio_categorical,
+            ratio_text,
+            ratio_date,
+        ]]
+        columns = [
+            'n_numeric',
+            'n_categorical',
+            'n_text',
+            'n_date',
+            'ratio_numeric',
+            'ratio_categorical',
+            'ratio_text',
+            'ratio_date',
+        ]
+        return pd.DataFrame(data, index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/duplicate_row_statistic.py

@@ -0,0 +1,56 @@
+"""[STATISTIC] Duplicate Rows."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class DuplicateRowStatistic(Statistic):
+    """[STATISTIC] Duplicate Rows."""
+
+    name: str = "Duplicate Rows"
+    _description: str = textwrap.dedent("""\
+        Duplicate rows reports the number and ratio of duplicated rows.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Duplicate rows counts duplicated rows based on usable columns and
+        provides the ratio relative to the total number of rows.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'duplicate_rows'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute number and ratio of duplicated rows."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        columns = [
+            col for col in dataset.X.columns
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT)
+        ]
+
+        n_rows = int(dataset.X.shape[0])
+        if columns:
+            data_frame = dataset.X[columns]
+            n_duplicate = int(data_frame.duplicated().sum())
+        else:
+            n_duplicate = 0
+
+        ratio = n_duplicate / n_rows if n_rows else 0.0
+
+        data = [[n_duplicate, ratio]]
+        return pd.DataFrame(
+            data,
+            index=[str(self)],
+            columns=["n_duplicate_rows", "ratio_duplicate_rows"],
+        )
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/effect_size_statistic.py

@@ -0,0 +1,129 @@
+"""[STATISTIC] Effect size (Cohen d)."""
+from __future__ import annotations
+
+import textwrap
+import numpy as np
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class EffectSizeStatistic(Statistic):
+    """[STATISTIC] Effect size (Cohen d)."""
+
+    name: str = "Effect Size"
+    _description: str = textwrap.dedent("""\
+        Effect size measures standardized differences (Cohen d) between classes.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Effect size computes Cohen's d for each numerical column per class
+        compared against the rest of the data. The `_all` suffix reports
+        the mean absolute effect size across classes for each feature.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'effect_size'
+
+    def _safe_variance(self, n: float, total: float, total_sq: float) -> float | None:
+        if n is None or n <= 1:
+            return None
+        denom = n - 1.0
+        var = (total_sq - (total * total) / n) / denom
+        if pd.isna(var) or var < 0:
+            return None
+        return float(var)
+
+    def _cohen_d(self, mean_a: float, mean_b: float, var_a: float | None,
+                 var_b: float | None, n_a: float, n_b: float) -> float | None:
+        if var_a is None or var_b is None:
+            return None
+        if n_a <= 1 or n_b <= 1:
+            return None
+        pooled_denom = (n_a + n_b - 2.0)
+        if pooled_denom <= 0:
+            return None
+        pooled_var = ((n_a - 1.0) * var_a + (n_b - 1.0) * var_b) / pooled_denom
+        if pooled_var <= 0 or pd.isna(pooled_var):
+            return None
+        pooled_std = np.sqrt(pooled_var)
+        if pooled_std == 0 or pd.isna(pooled_std):
+            return None
+        return (mean_a - mean_b) / pooled_std
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute Cohen's d for numerical columns."""
+        if dataset.type_of_target in ['survival', 'continuous']:
+            return pd.DataFrame()
+
+        numeric_columns = dataset.get_columns_names_by_type(DataType.NUMERIC)
+        if not numeric_columns:
+            return pd.DataFrame()
+
+        class_labels = list(pd.unique(dataset.y))
+
+        values = dataset.X[numeric_columns]
+        total_n = values.count()
+        total_sum = values.sum()
+        total_sumsq = (values ** 2).sum()
+
+        grouped = values.groupby(dataset.y)
+        class_n = grouped.count()
+        class_sum = grouped.sum()
+        class_sumsq = (values ** 2).groupby(dataset.y).sum()
+
+        data = []
+        columns = []
+
+        for col in numeric_columns:
+            col_total_n = float(total_n.get(col, 0))
+            col_total_sum = float(total_sum.get(col, 0))
+            col_total_sumsq = float(total_sumsq.get(col, 0))
+            abs_effects = []
+
+            for label in ['all'] + class_labels:
+                columns.append(f"{col}_{label}")
+                if label == 'all':
+                    data.append(None)
+                    continue
+
+                n_a = float(class_n.at[label, col]) if label in class_n.index else 0.0
+                if n_a <= 0:
+                    data.append(None)
+                    continue
+
+                sum_a = float(class_sum.at[label, col]) if label in class_sum.index else 0.0
+                sumsq_a = float(class_sumsq.at[label, col]) if label in class_sumsq.index else 0.0
+
+                n_b = col_total_n - n_a
+                if n_b <= 0:
+                    data.append(None)
+                    continue
+
+                sum_b = col_total_sum - sum_a
+                sumsq_b = col_total_sumsq - sumsq_a
+
+                mean_a = sum_a / n_a if n_a > 0 else None
+                mean_b = sum_b / n_b if n_b > 0 else None
+                if mean_a is None or mean_b is None or pd.isna(mean_a) or pd.isna(mean_b):
+                    data.append(None)
+                    continue
+
+                var_a = self._safe_variance(n_a, sum_a, sumsq_a)
+                var_b = self._safe_variance(n_b, sum_b, sumsq_b)
+                d_value = self._cohen_d(mean_a, mean_b, var_a, var_b, n_a, n_b)
+                data.append(d_value)
+                if d_value is not None and not pd.isna(d_value):
+                    abs_effects.append(abs(d_value))
+
+            if abs_effects:
+                data_index = columns.index(f"{col}_all")
+                data[data_index] = float(np.mean(abs_effects))
+
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target not in ['survival', 'continuous']

iaml/statistics/entropy_statistic.py

@@ -0,0 +1,69 @@
+"""[STATISTIC] Entropy."""
+from __future__ import annotations
+
+import textwrap
+import numpy as np
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class EntropyStatistic(Statistic):
+    """[STATISTIC] Entropy."""
+
+    name: str = "Entropy"
+    _description: str = textwrap.dedent("""\
+        Entropy measures the distribution uncertainty for categorical columns.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Entropy measures the distribution uncertainty for categorical columns,
+        optionally per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'entropy'
+
+    def _entropy(self, series: pd.Series) -> float:
+        values = series.dropna()
+        if values.empty:
+            return 0.0
+        counts = values.value_counts()
+        total = counts.sum()
+        if total == 0:
+            return 0.0
+        probs = counts / total
+        return float(-(probs * np.log2(probs)).sum())
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute entropy for categorical columns."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        data = []
+        columns = []
+
+        if dataset.type_of_target == 'continuous':
+            for col in dataset.X.columns:
+                if dataset.columns_types[col][1] == DataType.CATEGORICAL:
+                    columns.append(col)
+                    data.append(self._entropy(dataset.X[col]))
+            return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] == DataType.CATEGORICAL:
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if label == 'all':
+                        values = dataset.X[col]
+                    else:
+                        values = dataset.X.loc[dataset.y == label][col]
+                    data.append(self._entropy(values))
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/event_rate_statistic.py

@@ -0,0 +1,52 @@
+"""[STATISTIC] Event Rate."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..statistic import Statistic
+
+
+class EventRateStatistic(Statistic):
+    """[STATISTIC] Event Rate."""
+
+    name: str = "Event Rate"
+    _description: str = textwrap.dedent("""\
+        Event rate measures the proportion of events versus censoring in survival data.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Event rate measures the proportion of events versus censoring in survival targets.
+        Rates are reported as fractions in [0, 1].
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'event_rate'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute event and censoring rates for survival targets."""
+        if dataset.type_of_target != 'survival':
+            return pd.DataFrame()
+
+        samples = Dataset.normalize_survival_target(dataset.y)
+        n_samples = len(samples)
+
+        if n_samples:
+            event_count = int(sum(event for event, _ in samples))
+            censor_count = n_samples - event_count
+            event_rate = event_count / n_samples
+            censor_rate = censor_count / n_samples
+        else:
+            event_count = 0
+            censor_count = 0
+            event_rate = 0.0
+            censor_rate = 0.0
+
+        data = [[event_count, censor_count, event_rate, censor_rate]]
+        columns = ['event_count', 'censor_count', 'event_rate', 'censor_rate']
+        return pd.DataFrame(data, index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target == 'survival'

iaml/statistics/grouped_mean_statistic.py

@@ -0,0 +1,60 @@
+"""[STATISTIC] Grouped mean."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class GroupedMeanStatistic(Statistic):
+    """[STATISTIC] Grouped mean."""
+
+    name: str = "Grouped Mean"
+    _description: str = textwrap.dedent("""\
+        Grouped mean measures per-class mean offsets from the overall mean.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Grouped mean measures numerical column means per class as offsets from the overall mean.
+        The overall mean is reported under the `_all` column suffix for each feature.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'grouped_mean'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute grouped mean offsets for numerical columns."""
+        if dataset.type_of_target in ['survival', 'continuous']:
+            return pd.DataFrame()
+
+        numeric_columns = dataset.get_columns_names_by_type(DataType.NUMERIC)
+        if not numeric_columns:
+            return pd.DataFrame()
+
+        overall_means = dataset.X[numeric_columns].mean()
+        grouped_means = dataset.X[numeric_columns].groupby(dataset.y).mean()
+        class_labels = list(pd.unique(dataset.y))
+
+        data = []
+        columns = []
+        for col in numeric_columns:
+            overall = overall_means.get(col)
+            for label in ['all'] + class_labels:
+                columns.append(f"{col}_{label}")
+                if label == 'all':
+                    data.append(overall)
+                else:
+                    class_mean = grouped_means.at[label, col] if label in grouped_means.index else pd.NA
+                    if pd.isna(class_mean) or pd.isna(overall):
+                        data.append(None)
+                    else:
+                        data.append(class_mean - overall)
+
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target not in ['survival', 'continuous']

iaml/statistics/iqr_statistic.py

@@ -0,0 +1,66 @@
+"""[STATISTIC] IQR."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class IQRStatistic(Statistic):
+    """[STATISTIC] IQR."""
+
+    name: str = "IQR"
+    _description: str = textwrap.dedent("""\
+        IQR measures dispersion as Q3 minus Q1 for numerical columns.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        IQR measures dispersion as Q3 minus Q1 for numerical columns, per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'iqr'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute IQR for each numerical column."""
+        if dataset.type_of_target == 'survival':
+            return pd.DataFrame()
+
+        data = []
+        columns = []
+
+        if dataset.type_of_target == 'continuous':
+            for col in dataset.X.columns:
+                if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                    columns.append(col)
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        q3 = dataset.X[col].quantile(0.75)
+                        q1 = dataset.X[col].quantile(0.25)
+                        data.append(q3 - q1)
+                    else:
+                        data.append(None)
+            return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+        class_labels = list(pd.unique(dataset.y))
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        if label == 'all':
+                            series = dataset.X[col]
+                        else:
+                            series = dataset.X.loc[dataset.y == label][col]
+                        q3 = series.quantile(0.75)
+                        q1 = series.quantile(0.25)
+                        data.append(q3 - q1)
+                    else:
+                        data.append(None)
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target != 'survival'

iaml/statistics/kurtosis.py

@@ -0,0 +1,50 @@
+"""[STATISTIC] Kurtosis."""
+from __future__ import annotations
+
+import textwrap
+import pandas as pd
+
+from ..dataset import Dataset
+from ..data_type import DataType
+from ..statistic import Statistic
+
+
+class KurtosisStatistic(Statistic):
+    """[STATISTIC] Kurtosis."""
+
+    name: str = "Kurtosis"
+    _description: str = textwrap.dedent("""\
+        Kurtosis measures the tailedness of numerical distributions.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        Kurtosis measures the tailedness of numerical distributions, per class for classification.
+        """)
+    refs: list[dict] = []
+
+    def __str__(self) -> str:
+        return 'kurtosis'
+
+    def compute(self, dataset: Dataset, **kwargs) -> pd.DataFrame:
+        """Compute kurtosis for numerical columns."""
+        if dataset.type_of_target in ['survival', 'continuous']:
+            return pd.DataFrame()
+
+        class_labels = list(pd.unique(dataset.y))
+        data = []
+        columns = []
+        for col in dataset.X.columns:
+            if dataset.columns_types[col][1] not in (DataType.TEXT, DataType.SHORT_TEXT):
+                for label in ['all'] + class_labels:
+                    columns.append(f"{col}_{label}")
+                    if dataset.columns_types[col][1] == DataType.NUMERIC:
+                        if label == 'all':
+                            data.append(dataset.X[col].kurtosis())
+                        else:
+                            data.append(dataset.X.loc[dataset.y == label][col].kurtosis())
+                    else:
+                        data.append(None)
+        return pd.DataFrame([data], index=[str(self)], columns=columns)
+
+    def suitable(self, dataset: Dataset) -> bool:  # pylint: disable=unused-argument
+        """Does this statistic apply to the dataset?"""
+        return dataset.type_of_target not in ['survival', 'continuous']