PyIAML 1.0.0__py3-none-any.whl

iaml/plots/correlation_heatmap_plot.py

@@ -0,0 +1,201 @@
+"""[PLOT] Correlation heatmap for descriptive statistics."""
+from __future__ import annotations
+
+import io
+import textwrap
+from typing import Any
+
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from ..data_type import DataType
+from ..dataset import Dataset
+from ..plot import StatisticPlot, capture
+
+
+def _is_missing(value: Any) -> bool:
+    if value is None:
+        return True
+    if isinstance(value, float) and pd.isna(value):
+        return True
+    return False
+
+
+def _plot_placeholder(message: str) -> None:
+    plt.figure()
+    plt.text(0.5, 0.5, message, ha='center', va='center')
+    plt.axis('off')
+
+
+def _to_numeric_frame(frame: pd.DataFrame) -> pd.DataFrame:
+    numeric = frame.apply(pd.to_numeric, errors='coerce')
+    return numeric
+
+
+def _square_corr_from_dataframe(frame: pd.DataFrame) -> pd.DataFrame | None:
+    if frame.empty:
+        return None
+    if frame.shape[0] != frame.shape[1]:
+        return None
+    if set(frame.index) != set(frame.columns):
+        return None
+    ordered = frame.reindex(index=frame.index, columns=frame.index)
+    numeric = _to_numeric_frame(ordered)
+    if not np.isfinite(numeric.to_numpy()).any():
+        return None
+    return numeric
+
+
+def _corr_from_value(value: Any) -> pd.DataFrame | None:
+    if _is_missing(value):
+        return None
+    if isinstance(value, pd.DataFrame):
+        return _square_corr_from_dataframe(value)
+    if isinstance(value, dict):
+        matrix = None
+        if 'matrix' in value:
+            matrix = value['matrix']
+        elif 'values' in value:
+            matrix = value['values']
+        if matrix is None:
+            return None
+        df = pd.DataFrame(matrix)
+        labels = value.get('labels')
+        if labels is not None:
+            df.index = labels
+            df.columns = labels
+        if 'index' in value:
+            df.index = value['index']
+        if 'columns' in value:
+            df.columns = value['columns']
+        return _square_corr_from_dataframe(df)
+    if isinstance(value, (list, tuple, np.ndarray)):
+        array = np.asarray(value)
+        if array.ndim == 2:
+            return _square_corr_from_dataframe(pd.DataFrame(array))
+    return None
+
+
+def _corr_from_row(row: pd.Series) -> pd.DataFrame | None:
+    if not isinstance(row, pd.Series):
+        return None
+    for item in row:
+        corr = _corr_from_value(item)
+        if corr is not None:
+            return corr
+    return None
+
+
+def _extract_corr_dataframe(dataframe: pd.DataFrame) -> pd.DataFrame | None:
+    corr_df = _square_corr_from_dataframe(dataframe)
+    if corr_df is not None:
+        return corr_df
+    for key in ('correlation', 'correlation_matrix', 'corr', 'corr_matrix'):
+        if key in dataframe.index:
+            return _corr_from_row(dataframe.loc[key])
+    return None
+
+
+def _figure_size(n_features: int) -> float:
+    if n_features <= 1:
+        return 4.0
+    return float(min(12.0, max(4.5, 0.5 * n_features + 2.0)))
+
+
+class CorrelationHeatmapPlot(StatisticPlot):
+    """[PLOT] Correlation Heatmap Plot."""
+
+    name: str = "Correlation Heatmap"
+    _description: str = textwrap.dedent("""\
+        Correlation heatmaps summarize relationships between numeric features.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        This plot displays a correlation matrix for numeric columns, helping
+        identify strong positive or negative relationships.
+        """)
+    refs: list[dict] = []
+
+    title: str = "Correlation heatmap"
+    description: str = textwrap.dedent("""\
+        The correlation heatmap visualizes pairwise correlations among numeric columns.
+        """)
+    group_by_feature: bool = False
+
+    def __str__(self) -> str:
+        return 'correlation'
+
+    @capture
+    def compute(
+        self,
+        dataframe: pd.DataFrame,
+        dataset: Dataset | None = None,
+        base_name: str | None = None,
+        **kwargs,
+    ) -> 'CorrelationHeatmapPlot':
+        """Compute correlation heatmap statistics."""
+        self._binary_image = io.BytesIO()
+
+        if dataframe.empty:
+            _plot_placeholder("No statistics available")
+            plt.savefig(self._binary_image, format='png')
+            return self
+
+        corr_df = _extract_corr_dataframe(dataframe)
+
+        if corr_df is None and dataset is not None:
+            numeric_columns = dataset.get_columns_names_by_type(DataType.NUMERIC)
+            if numeric_columns:
+                corr_df = dataset.X[numeric_columns].corr()
+
+        if corr_df is None or corr_df.empty:
+            _plot_placeholder("Correlation statistics not available")
+            plt.savefig(self._binary_image, format='png')
+            return self
+
+        corr_df = _to_numeric_frame(corr_df)
+        if not np.isfinite(corr_df.to_numpy()).any():
+            _plot_placeholder("Correlation statistics not available")
+            plt.savefig(self._binary_image, format='png')
+            return self
+
+        if dataset is not None:
+            numeric_columns = dataset.get_columns_names_by_type(DataType.NUMERIC)
+            if numeric_columns:
+                available = [col for col in corr_df.index if col in numeric_columns]
+                if available:
+                    corr_df = corr_df.loc[available, available]
+
+        if corr_df.empty:
+            _plot_placeholder("Correlation statistics not available")
+            plt.savefig(self._binary_image, format='png')
+            return self
+
+        n_features = corr_df.shape[0]
+        fig_size = _figure_size(n_features)
+        fig, ax = plt.subplots(figsize=(fig_size, fig_size))
+
+        values = corr_df.to_numpy(dtype=float)
+        masked = np.ma.masked_invalid(values)
+        image = ax.imshow(masked, cmap='coolwarm', vmin=-1, vmax=1)
+        plt.colorbar(image, ax=ax, fraction=0.046, pad=0.04)
+
+        labels = [str(label) for label in corr_df.columns]
+        ticks = np.arange(n_features)
+        ax.set_xticks(ticks)
+        ax.set_yticks(ticks)
+        ax.set_xticklabels(labels, rotation=45, ha='right')
+        ax.set_yticklabels(labels)
+
+        label_size = 10 if n_features <= 12 else 8 if n_features <= 20 else 6
+        ax.tick_params(axis='both', which='major', labelsize=label_size)
+
+        title = "Correlation heatmap"
+        if base_name:
+            title = f"Correlation heatmap: {base_name}"
+        ax.set_title(title)
+        ax.set_xlabel('Features')
+        ax.set_ylabel('Features')
+        plt.tight_layout()
+        plt.savefig(self._binary_image, format='png')
+        return self

iaml/plots/cumulative_hazard_plot.py

@@ -0,0 +1,72 @@
+"""[PLOT] Cumulative Hazard Model Comparison Plot using sksurv"""
+import io
+from typing import TYPE_CHECKING
+import textwrap
+from sksurv.nonparametric import nelson_aalen_estimator
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from ..metric_plot import MetricPlot, capture
+if TYPE_CHECKING:
+    from ..iaml_pipeline import IAMLPipeline
+
+
+class CumulativeHazardModelComparisonPlot(MetricPlot):
+    """[PLOT] Cumulative Hazard Model Comparison Plot using sksurv"""
+
+    title: str = "Cumulative Hazard"
+    description: str = textwrap.dedent("""
+        The Cumulative Hazard Model Comparison Plot is a diagnostic tool used to evaluate the performance of 
+        survival models by comparing predicted cumulative hazard functions against the observed cumulative hazards.
+
+        The x-axis represents time, and the y-axis represents the cumulative hazard. Ideally, 
+        the model-predicted cumulative hazard curves should closely align with the observed curves, 
+        indicating good model performance. Discrepancies between the two curves highlight 
+        areas where the model's predictions diverge from reality, signaling potential issues with 
+        the model's predictive ability.
+
+        Additionally, a Cox proportional hazards model can be trained to serve as a baseline for comparison.
+        """)
+
+    @capture
+    def compute(
+        self,
+        estimator: 'IAMLPipeline',
+        X: pd.DataFrame,
+        y: pd.Series,
+        X_train: pd.DataFrame = None,
+        y_train: pd.Series = None,
+        **kwargs) -> MetricPlot:
+        self._binary_image = io.BytesIO()
+
+        # Fit the cumulative hazard model on observed data
+        event, time = zip(*y)
+
+        # Observed cumulative hazard using nelson_aalen_estimator
+        time, cumulative_hazard = nelson_aalen_estimator(event, time)
+        plt.step(time, cumulative_hazard, where="post", label="Observed", color='blue')
+
+        # Current model prediction
+        hazard_predictions = estimator.predict_cumulative_hazard_function(X)
+
+        mean_hazard_prob = np.mean([fn.y for fn in hazard_predictions], axis=0)
+        mean_hazard_time = hazard_predictions[0].x
+
+        plt.step(mean_hazard_time, mean_hazard_prob,
+                where="post", label="Model prediction", color="green")
+
+        # Customize and save the plot
+        plt.title("Cumulative Hazard Curve vs Model Predicted")
+        plt.xlabel("Time")
+        plt.ylabel("Cumulative Hazard")
+        plt.ylim([0, 1])
+        plt.legend()
+
+        plt.savefig(self._binary_image, format='png')
+        plt.close()
+
+        return self
+
+    @classmethod
+    def suitable(cls, type_of_target: str) -> bool:
+        return type_of_target == 'survival'

iaml/plots/density_plot.py

@@ -0,0 +1,210 @@
+"""[PLOT] Density plot for descriptive statistics."""
+from __future__ import annotations
+
+import io
+import textwrap
+from typing import Any
+
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from scipy import stats
+
+from ..data_type import DataType
+from ..dataset import Dataset
+from ..plot import StatisticPlot, capture
+
+
+def _is_missing(value: Any) -> bool:
+    if value is None:
+        return True
+    if isinstance(value, float) and pd.isna(value):
+        return True
+    return False
+
+
+def _column_label(column: str, base_name: str | None) -> str:
+    column_str = str(column)
+    base_name_str = str(base_name) if base_name is not None else None
+    if base_name_str:
+        if column_str == base_name_str:
+            return 'all'
+        prefix = f"{base_name_str}_"
+        if column_str.startswith(prefix):
+            return column_str[len(prefix):]
+    if column_str.endswith('_all'):
+        return 'all'
+    return column_str
+
+
+def _plot_placeholder(message: str) -> None:
+    plt.figure()
+    plt.text(0.5, 0.5, message, ha='center', va='center')
+    plt.axis('off')
+
+
+def _sample_values(values: np.ndarray, max_points: int = 2000) -> np.ndarray:
+    if values.size <= max_points:
+        return values
+    indices = np.linspace(0, values.size - 1, max_points, dtype=int)
+    return values[indices]
+
+
+def _kde_from_values(values: np.ndarray) -> tuple[np.ndarray, np.ndarray] | None:
+    if values.size < 2:
+        return None
+    try:
+        numeric = values.astype(float)
+    except (TypeError, ValueError):
+        return None
+    numeric = numeric[np.isfinite(numeric)]
+    if numeric.size < 2:
+        return None
+
+    sample = _sample_values(numeric)
+    vmin = float(np.min(sample))
+    vmax = float(np.max(sample))
+    if vmin == vmax:
+        eps = 1e-3 if abs(vmin) < 1 else abs(vmin) * 1e-3
+        support = np.array([vmin - eps, vmin, vmin + eps], dtype=float)
+        density = np.array([0.0, 1.0, 0.0], dtype=float)
+        return support, density
+
+    support = np.linspace(vmin, vmax, 100)
+    try:
+        kernel = stats.gaussian_kde(sample)
+        density = kernel(support)
+        return support, density
+    except Exception:
+        bins = int(np.sqrt(sample.size))
+        bins = max(5, min(20, bins))
+        counts, bin_edges = np.histogram(sample, bins=bins, density=True)
+        centers = (bin_edges[:-1] + bin_edges[1:]) / 2.0
+        if centers.size == 0:
+            return None
+        return centers, counts
+
+
+def _extract_density_data(value: Any) -> tuple[np.ndarray, np.ndarray] | None:
+    if _is_missing(value):
+        return None
+    if isinstance(value, dict):
+        if 'density' in value and 'support' in value:
+            try:
+                density = np.asarray(value['density'], dtype=float).ravel()
+                support = np.asarray(value['support'], dtype=float).ravel()
+            except (TypeError, ValueError):
+                return None
+            if density.size == 0 or support.size == 0:
+                return None
+            if density.size != support.size:
+                return None
+            mask = np.isfinite(density) & np.isfinite(support)
+            if not np.any(mask):
+                return None
+            density = density[mask]
+            support = support[mask]
+            order = np.argsort(support)
+            return support[order], density[order]
+        if 'values' in value:
+            return _kde_from_values(np.asarray(value['values']))
+    if isinstance(value, (list, tuple, np.ndarray, pd.Series)):
+        if isinstance(value, (list, tuple)) and len(value) == 2:
+            first = np.asarray(value[0])
+            second = np.asarray(value[1])
+            if first.ndim == 1 and second.ndim == 1 and first.size == second.size:
+                return first.astype(float), second.astype(float)
+        return _kde_from_values(np.asarray(value))
+    return None
+
+
+class DensityPlot(StatisticPlot):
+    """[PLOT] Density Plot."""
+
+    name: str = "Density Plot"
+    _description: str = textwrap.dedent("""\
+        Density plots show smoothed distributions for numeric columns.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        This plot renders kernel density estimates for numeric columns using
+        precomputed density statistics when available.
+        """)
+    refs: list[dict] = []
+
+    title: str = "Density plot"
+    description: str = textwrap.dedent("""\
+        The density plot displays kernel density estimates for numeric columns.
+        """)
+    group_by_feature: bool = True
+
+    def __str__(self) -> str:
+        return 'density'
+
+    @capture
+    def compute(
+        self,
+        dataframe: pd.DataFrame,
+        base_name: str | None = None,
+        dataset: Dataset | None = None,
+        **kwargs,
+    ) -> 'DensityPlot':
+        """Compute density plot statistics."""
+        self._binary_image = io.BytesIO()
+
+        if dataframe.empty:
+            _plot_placeholder("No statistics available")
+            plt.savefig(self._binary_image, format='png')
+            return self
+
+        density_key = str(self)
+        if density_key not in dataframe.index:
+            _plot_placeholder("Density statistics not available")
+            plt.savefig(self._binary_image, format='png')
+            return self
+
+        if dataset is not None:
+            numeric_columns = set(dataset.get_columns_names_by_type(DataType.NUMERIC))
+            columns_to_show = [col for col in dataframe.columns if col in numeric_columns]
+        else:
+            columns_to_show = list(dataframe.columns)
+
+        density_row = dataframe.loc[density_key]
+        entries: list[tuple[str, tuple[np.ndarray, np.ndarray]]] = []
+        for col in columns_to_show:
+            density_data = _extract_density_data(density_row.get(col))
+            if density_data is not None:
+                entries.append((col, density_data))
+
+        if not entries:
+            _plot_placeholder("No numeric density statistics available")
+            plt.savefig(self._binary_image, format='png')
+            return self
+
+        n_plots = len(entries)
+        n_cols = 1 if n_plots == 1 else 2
+        n_rows = int(np.ceil(n_plots / n_cols))
+        fig, axes = plt.subplots(n_rows, n_cols, figsize=(6 * n_cols, 3.5 * n_rows))
+        axes_list = np.atleast_1d(axes).ravel()
+
+        for ax, (col, (support, density)) in zip(axes_list, entries):
+            if support.size == 0 or density.size == 0 or support.size != density.size:
+                ax.text(0.5, 0.5, "Invalid density data", ha='center', va='center')
+                ax.axis('off')
+                continue
+            ax.plot(support, density, color='tab:blue')
+            ax.fill_between(support, density, alpha=0.3, color='tab:blue')
+            ax.set_title(_column_label(col, base_name))
+            ax.set_xlabel('Value')
+            ax.set_ylabel('Density')
+
+        for ax in axes_list[len(entries):]:
+            ax.axis('off')
+
+        if base_name:
+            fig.suptitle(f"Density plot: {base_name}")
+            plt.tight_layout(rect=(0, 0, 1, 0.95))
+        else:
+            plt.tight_layout()
+
+        plt.savefig(self._binary_image, format='png')
+        return self

iaml/plots/histogram_plot.py

@@ -0,0 +1,179 @@
+"""[PLOT] Histogram plot for descriptive statistics."""
+from __future__ import annotations
+
+import io
+import textwrap
+from typing import Any
+
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from ..data_type import DataType
+from ..dataset import Dataset
+from ..plot import StatisticPlot, capture
+
+
+def _is_missing(value: object) -> bool:
+    if value is None:
+        return True
+    if isinstance(value, float) and pd.isna(value):
+        return True
+    return False
+
+
+def _column_label(column: str, base_name: str | None) -> str:
+    column_str = str(column)
+    base_name_str = str(base_name) if base_name is not None else None
+    if base_name_str:
+        if column_str == base_name_str:
+            return 'all'
+        prefix = f"{base_name_str}_"
+        if column_str.startswith(prefix):
+            return column_str[len(prefix):]
+    if column_str.endswith('_all'):
+        return 'all'
+    return column_str
+
+
+def _plot_placeholder(message: str) -> None:
+    plt.figure()
+    plt.text(0.5, 0.5, message, ha='center', va='center')
+    plt.axis('off')
+
+
+def _histogram_from_values(values: np.ndarray) -> tuple[np.ndarray, np.ndarray] | None:
+    if values.size == 0:
+        return None
+    try:
+        numeric = values.astype(float)
+    except (ValueError, TypeError):
+        return None
+    numeric = numeric[np.isfinite(numeric)]
+    if numeric.size == 0:
+        return None
+    bins = int(np.sqrt(numeric.size))
+    bins = max(5, min(20, bins))
+    counts, bin_edges = np.histogram(numeric, bins=bins)
+    return counts, bin_edges
+
+
+def _extract_hist_data(value: Any) -> tuple[np.ndarray, np.ndarray] | None:
+    if _is_missing(value):
+        return None
+    if isinstance(value, dict):
+        if 'counts' in value and ('bins' in value or 'bin_edges' in value):
+            counts = np.asarray(value['counts'])
+            bins = np.asarray(value.get('bins', value.get('bin_edges')))
+            return counts, bins
+        if 'hist' in value and ('bins' in value or 'bin_edges' in value):
+            counts = np.asarray(value['hist'])
+            bins = np.asarray(value.get('bins', value.get('bin_edges')))
+            return counts, bins
+        if 'values' in value:
+            return _histogram_from_values(np.asarray(value['values']))
+    if isinstance(value, (list, tuple, np.ndarray, pd.Series)):
+        if isinstance(value, (list, tuple)) and len(value) == 2:
+            first = np.asarray(value[0])
+            second = np.asarray(value[1])
+            if first.ndim == 1 and second.ndim == 1:
+                if first.size == second.size + 1:
+                    return second, first
+                if second.size == first.size + 1:
+                    return first, second
+                if first.size == second.size and first.size > 0:
+                    bins = np.arange(first.size + 1)
+                    return first, bins
+        return _histogram_from_values(np.asarray(value))
+    return None
+
+
+class HistogramPlot(StatisticPlot):
+    """[PLOT] Histogram Plot."""
+
+    name: str = "Histogram Plot"
+    _description: str = textwrap.dedent("""\
+        Histograms show distributions for numeric columns.
+        """)
+    _description_long: str = textwrap.dedent("""\
+        This plot renders a histogram per numeric column to visualize the distribution
+        of values. It relies on precomputed histogram statistics when available.
+        """)
+    refs: list[dict] = []
+
+    title: str = "Histogram plot"
+    description: str = textwrap.dedent("""\
+        The histogram plot displays distributions for numeric columns.
+        """)
+    group_by_feature: bool = True
+
+    def __str__(self) -> str:
+        return 'histogram'
+
+    @capture
+    def compute(
+        self,
+        dataframe: pd.DataFrame,
+        base_name: str | None = None,
+        dataset: Dataset | None = None,
+        **kwargs) -> 'HistogramPlot':
+        """Compute histogram plot statistics."""
+        self._binary_image = io.BytesIO()
+
+        if dataframe.empty:
+            _plot_placeholder("No statistics available")
+            plt.savefig(self._binary_image, format='png')
+            return self
+
+        hist_key = str(self)
+        if hist_key not in dataframe.index:
+            _plot_placeholder("Histogram statistics not available")
+            plt.savefig(self._binary_image, format='png')
+            return self
+
+        if dataset is not None:
+            numeric_columns = set(dataset.get_columns_names_by_type(DataType.NUMERIC))
+            columns_to_show = [col for col in dataframe.columns if col in numeric_columns]
+        else:
+            columns_to_show = list(dataframe.columns)
+
+        histogram_row = dataframe.loc[hist_key]
+        entries: list[tuple[str, tuple[np.ndarray, np.ndarray]]] = []
+        for col in columns_to_show:
+            hist_data = _extract_hist_data(histogram_row.get(col))
+            if hist_data is not None:
+                entries.append((col, hist_data))
+
+        if not entries:
+            _plot_placeholder("No numeric histogram statistics available")
+            plt.savefig(self._binary_image, format='png')
+            return self
+
+        n_plots = len(entries)
+        n_cols = 1 if n_plots == 1 else 2
+        n_rows = int(np.ceil(n_plots / n_cols))
+        fig, axes = plt.subplots(n_rows, n_cols, figsize=(6 * n_cols, 3.5 * n_rows))
+        axes_list = np.atleast_1d(axes).ravel()
+
+        for ax, (col, (counts, bins)) in zip(axes_list, entries):
+            if bins.size != counts.size + 1:
+                ax.text(0.5, 0.5, "Invalid histogram data", ha='center', va='center')
+                ax.axis('off')
+                continue
+            widths = np.diff(bins)
+            ax.bar(bins[:-1], counts, width=widths, align='edge', edgecolor='black')
+            ax.set_title(_column_label(col, base_name))
+            ax.set_xlabel('Value')
+            ax.set_ylabel('Count')
+
+        for ax in axes_list[len(entries):]:
+            ax.axis('off')
+
+        if base_name:
+            fig.suptitle(f"Histogram: {base_name}")
+            plt.tight_layout(rect=(0, 0, 1, 0.95))
+        else:
+            plt.tight_layout()
+
+        plt.savefig(self._binary_image, format='png')
+        return self