scitex 2.14.0__py3-none-any.whl → 2.15.1__py3-none-any.whl

scitex/ai/classification/reporters/_mixins/_metrics.py

@@ -0,0 +1,275 @@
+#!/usr/bin/env python3
+# Timestamp: "2026-01-24 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex-python/src/scitex/ai/classification/reporters/_mixins/_metrics.py
+
+"""
+Metrics calculation mixin for classification reporter.
+"""
+
+from __future__ import annotations
+
+from pprint import pprint
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+import pandas as pd
+
+from scitex.logging import getLogger
+
+from ._constants import FILENAME_PATTERNS, FOLD_DIR_PREFIX_PATTERN
+
+logger = getLogger(__name__)
+
+
+class MetricsMixin:
+    """Mixin providing metrics calculation methods."""
+
+    def calculate_metrics(
+        self,
+        y_true: np.ndarray,
+        y_pred: np.ndarray,
+        y_proba: Optional[np.ndarray] = None,
+        labels: Optional[List[str]] = None,
+        fold: Optional[int] = None,
+        verbose: bool = True,
+        store_y_true: bool = True,
+        store_y_pred: bool = True,
+        store_y_proba: bool = True,
+        model: Optional[Any] = None,
+        feature_names: Optional[List[str]] = None,
+    ) -> Dict[str, Any]:
+        """Calculate and save classification metrics using unified API."""
+        from ..reporter_utils import (
+            calc_bacc,
+            calc_clf_report,
+            calc_conf_mat,
+            calc_mcc,
+        )
+
+        if verbose:
+            if fold:
+                print()
+                logger.info(f"Calculating metrics for fold #{fold:02d}...")
+            else:
+                logger.info("Calculating metrics...")
+
+        if len(y_true) != len(y_pred):
+            raise ValueError("y_true and y_pred must have same length")
+
+        if y_proba is not None and len(y_true) != len(y_proba):
+            raise ValueError("y_true and y_proba must have same length")
+
+        if fold is None:
+            fold = 0
+
+        if labels is None:
+            unique_labels = sorted(np.unique(np.concatenate([y_true, y_pred])))
+            labels = [f"Class_{i}" for i in unique_labels]
+
+        metrics = {}
+        metrics["balanced-accuracy"] = calc_bacc(y_true, y_pred, fold=fold)
+        metrics["mcc"] = calc_mcc(y_true, y_pred, fold=fold)
+        metrics["confusion_matrix"] = calc_conf_mat(
+            y_true=y_true, y_pred=y_pred, labels=labels, fold=fold
+        )
+        metrics["classification_report"] = calc_clf_report(
+            y_true, y_pred, labels, fold=fold
+        )
+
+        if y_proba is not None:
+            try:
+                from scitex.ai.metrics import calc_pre_rec_auc, calc_roc_auc
+
+                metrics["roc-auc"] = calc_roc_auc(
+                    y_true, y_proba, labels=labels, fold=fold, return_curve=False
+                )
+                metrics["pr-auc"] = calc_pre_rec_auc(
+                    y_true, y_proba, labels=labels, fold=fold, return_curve=False
+                )
+            except Exception as e:
+                logger.warning(f"Could not calculate AUC metrics: {e}")
+
+        metrics = self._round_numeric(metrics)
+        metrics["labels"] = labels
+
+        if verbose:
+            logger.info("Metrics calculated:")
+            pprint(metrics)
+
+        self.fold_metrics[fold] = metrics.copy()
+
+        if y_proba is not None:
+            self.all_predictions.append(
+                {
+                    "fold": fold,
+                    "y_true": y_true.copy(),
+                    "y_proba": y_proba.copy(),
+                }
+            )
+
+        self._save_fold_metrics(metrics, fold, labels)
+        self._create_plots(y_true, y_pred, y_proba, labels, fold, metrics)
+
+        if model is not None and feature_names is not None:
+            self._extract_and_save_feature_importance(
+                model, feature_names, fold, metrics, verbose
+            )
+
+        if store_y_true or store_y_pred or store_y_proba:
+            self._store_raw_predictions(
+                y_true, y_pred, y_proba, fold, store_y_true, store_y_pred, store_y_proba
+            )
+
+        return metrics
+
+    def _extract_and_save_feature_importance(
+        self, model, feature_names, fold, metrics, verbose
+    ):
+        """Extract and save feature importance from model."""
+        try:
+            from scitex.ai.feature_selection import extract_feature_importance
+
+            importance_dict = extract_feature_importance(
+                model, feature_names, method="auto"
+            )
+            if importance_dict:
+                metrics["feature-importance"] = importance_dict
+                self.fold_metrics[fold]["feature-importance"] = importance_dict
+
+                fold_dir = FOLD_DIR_PREFIX_PATTERN.format(fold=fold)
+                filename = FILENAME_PATTERNS["feature_importance_json"].format(
+                    fold=fold
+                )
+                self.storage.save(importance_dict, f"{fold_dir}/{filename}")
+
+                if verbose:
+                    logger.info("  Feature importance extracted and saved")
+        except Exception as e:
+            logger.warning(f"Could not extract feature importance: {e}")
+
+    def _store_raw_predictions(
+        self, y_true, y_pred, y_proba, fold, store_y_true, store_y_pred, store_y_proba
+    ):
+        """Store raw prediction data as CSV files."""
+        fold_dir = FOLD_DIR_PREFIX_PATTERN.format(fold=fold)
+        sample_indices = np.arange(len(y_true))
+
+        n_samples = len(y_true)
+        estimated_size_mb = 0
+        if store_y_true:
+            estimated_size_mb += n_samples * 0.0001
+        if store_y_pred:
+            estimated_size_mb += n_samples * 0.0001
+        if store_y_proba and y_proba is not None:
+            n_classes = 1 if y_proba.ndim == 1 else y_proba.shape[1]
+            estimated_size_mb += n_samples * n_classes * 0.0001
+
+        if estimated_size_mb > 10:
+            logger.warning(
+                f"Storing raw predictions for fold {fold} will create "
+                f"~{estimated_size_mb:.1f}MB of CSV files."
+            )
+
+        if store_y_true:
+            filename = FILENAME_PATTERNS["y_true"].format(fold=fold)
+            df_y_true = pd.DataFrame(
+                {"sample_index": sample_indices, "fold": fold, "y_true": y_true}
+            )
+            self.storage.save(df_y_true, f"{fold_dir}/{filename}")
+
+        if store_y_pred:
+            filename = FILENAME_PATTERNS["y_pred"].format(fold=fold)
+            df_y_pred = pd.DataFrame(
+                {"sample_index": sample_indices, "fold": fold, "y_pred": y_pred}
+            )
+            self.storage.save(df_y_pred, f"{fold_dir}/{filename}")
+
+        if store_y_proba and y_proba is not None:
+            filename = FILENAME_PATTERNS["y_proba"].format(fold=fold)
+            if y_proba.ndim == 1:
+                df_y_proba = pd.DataFrame(
+                    {"sample_index": sample_indices, "fold": fold, "y_proba": y_proba}
+                )
+            else:
+                data = {"sample_index": sample_indices, "fold": fold}
+                for i in range(y_proba.shape[1]):
+                    data[f"proba_class_{i}"] = y_proba[:, i]
+                df_y_proba = pd.DataFrame(data)
+            self.storage.save(df_y_proba, f"{fold_dir}/{filename}")
+
+    def _extract_metric_value(self, metric_data: Any) -> Optional[float]:
+        """Extract numeric value from metric data."""
+        if metric_data is None:
+            return None
+        if isinstance(metric_data, dict) and "value" in metric_data:
+            return float(metric_data["value"])
+        if isinstance(metric_data, (int, float, np.number)):
+            return float(metric_data)
+        return None
+
+    def get_summary(self) -> Dict[str, Any]:
+        """Get summary of all calculated metrics across folds."""
+        if not self.fold_metrics:
+            return {"error": "No metrics calculated yet"}
+
+        summary = {
+            "output_dir": str(self.output_dir),
+            "total_folds": len(self.fold_metrics),
+            "metrics_summary": {},
+        }
+
+        confusion_matrices = []
+        for fold_metrics in self.fold_metrics.values():
+            if "confusion_matrix" in fold_metrics:
+                cm_data = fold_metrics["confusion_matrix"]
+                if isinstance(cm_data, dict) and "value" in cm_data:
+                    cm_data = cm_data["value"]
+                if cm_data is not None:
+                    confusion_matrices.append(cm_data)
+
+        if confusion_matrices:
+            overall_cm = np.sum(confusion_matrices, axis=0)
+            summary["overall_confusion_matrix"] = overall_cm.tolist()
+            overall_cm_normalized = overall_cm / overall_cm.sum()
+            summary["overall_confusion_matrix_normalized"] = self._round_numeric(
+                overall_cm_normalized.tolist()
+            )
+
+        scalar_metrics = ["balanced-accuracy", "mcc", "roc-auc", "pr-auc"]
+        for metric_name in scalar_metrics:
+            values = []
+            for fold_metrics in self.fold_metrics.values():
+                if metric_name in fold_metrics:
+                    metric_val = fold_metrics[metric_name]
+                    if isinstance(metric_val, dict) and "value" in metric_val:
+                        values.append(metric_val["value"])
+                    else:
+                        values.append(metric_val)
+
+            if values:
+                values = np.array(values)
+                summary["metrics_summary"][metric_name] = {
+                    "mean": self._round_numeric(np.mean(values)),
+                    "std": self._round_numeric(np.std(values)),
+                    "min": self._round_numeric(np.min(values)),
+                    "max": self._round_numeric(np.max(values)),
+                    "values": self._round_numeric(values.tolist()),
+                }
+
+        feature_importances_list = []
+        for fold_metrics in self.fold_metrics.values():
+            if "feature-importance" in fold_metrics:
+                feature_importances_list.append(fold_metrics["feature-importance"])
+
+        if feature_importances_list:
+            from scitex.ai.feature_selection import aggregate_feature_importances
+
+            aggregated_importances = aggregate_feature_importances(
+                feature_importances_list
+            )
+            summary["feature-importance"] = aggregated_importances
+
+        return summary
+
+
+# EOF

scitex/ai/classification/reporters/_mixins/_plotting.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+# Timestamp: "2026-01-24 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex-python/src/scitex/ai/classification/reporters/_mixins/_plotting.py
+
+"""
+Plotting mixin for classification reporter.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+
+from scitex.logging import getLogger
+
+from ._constants import FILENAME_PATTERNS, FOLD_DIR_PREFIX_PATTERN
+
+logger = getLogger(__name__)
+
+
+class PlottingMixin:
+    """Mixin providing plotting methods."""
+
+    def _create_plots(
+        self,
+        y_true: np.ndarray,
+        y_pred: np.ndarray,
+        y_proba: Optional[np.ndarray],
+        labels: List[str],
+        fold: int,
+        metrics: Dict[str, Any],
+    ) -> None:
+        """Create and save plots with metric-based filenames."""
+        fold_dir = self._create_subdir_if_needed(
+            FOLD_DIR_PREFIX_PATTERN.format(fold=fold)
+        )
+        fold_dir.mkdir(parents=True, exist_ok=True)
+
+        # Confusion matrix plot
+        if "confusion_matrix" in metrics:
+            self._create_confusion_matrix_plot(metrics, labels, fold, fold_dir)
+
+        # ROC and PR curves
+        if y_proba is not None:
+            self._create_roc_curve_plot(
+                y_true, y_proba, labels, fold, fold_dir, metrics
+            )
+            self._create_pr_curve_plot(y_true, y_proba, labels, fold, fold_dir, metrics)
+
+        # Metrics dashboard
+        summary_filename = FILENAME_PATTERNS["metrics_summary"].format(fold=fold)
+        self.plotter.create_metrics_visualization(
+            metrics=metrics,
+            y_true=y_true,
+            y_pred=y_pred,
+            y_proba=y_proba,
+            labels=labels,
+            save_path=fold_dir / summary_filename,
+            title="Classification Metrics Dashboard",
+            fold=fold,
+            verbose=False,
+        )
+
+    def _create_confusion_matrix_plot(self, metrics, labels, fold, fold_dir):
+        """Create confusion matrix plot."""
+        cm_data = metrics["confusion_matrix"]
+        if isinstance(cm_data, dict) and "value" in cm_data:
+            cm_data = cm_data["value"]
+
+        balanced_acc = metrics.get("balanced-accuracy", {})
+        if isinstance(balanced_acc, dict) and "value" in balanced_acc:
+            balanced_acc = balanced_acc["value"]
+        elif isinstance(balanced_acc, (float, np.floating)):
+            balanced_acc = float(balanced_acc)
+        else:
+            balanced_acc = None
+
+        if balanced_acc is not None:
+            title = (
+                f"Confusion Matrix (Fold {fold:02d}) - Balanced Acc: {balanced_acc:.3f}"
+            )
+            filename = FILENAME_PATTERNS["confusion_matrix_jpg"].format(
+                fold=fold, bacc=balanced_acc
+            )
+        else:
+            title = f"Confusion Matrix (Fold {fold:02d})"
+            filename = FILENAME_PATTERNS["confusion_matrix_jpg_no_bacc"].format(
+                fold=fold
+            )
+
+        self.plotter.create_confusion_matrix_plot(
+            cm_data, labels=labels, save_path=fold_dir / filename, title=title
+        )
+
+    def _create_roc_curve_plot(self, y_true, y_proba, labels, fold, fold_dir, metrics):
+        """Create ROC curve plot."""
+        roc_auc = metrics.get("roc-auc", {})
+        if isinstance(roc_auc, dict) and "value" in roc_auc:
+            roc_auc_val = roc_auc["value"]
+            roc_filename = FILENAME_PATTERNS["roc_curve_jpg"].format(
+                fold=fold, auc=roc_auc_val
+            )
+        else:
+            roc_filename = FILENAME_PATTERNS["roc_curve_jpg_no_auc"].format(fold=fold)
+
+        self.plotter.create_roc_curve(
+            y_true,
+            y_proba,
+            labels=labels,
+            save_path=fold_dir / roc_filename,
+            title=f"ROC Curve (Fold {fold:02d})",
+        )
+
+    def _create_pr_curve_plot(self, y_true, y_proba, labels, fold, fold_dir, metrics):
+        """Create precision-recall curve plot."""
+        pr_auc = metrics.get("pr-auc", {})
+        if isinstance(pr_auc, dict) and "value" in pr_auc:
+            pr_auc_val = pr_auc["value"]
+            pr_filename = FILENAME_PATTERNS["pr_curve_jpg"].format(
+                fold=fold, ap=pr_auc_val
+            )
+        else:
+            pr_filename = FILENAME_PATTERNS["pr_curve_jpg_no_ap"].format(fold=fold)
+
+        self.plotter.create_precision_recall_curve(
+            y_true,
+            y_proba,
+            labels=labels,
+            save_path=fold_dir / pr_filename,
+            title=f"Precision-Recall Curve (Fold {fold:02d})",
+        )
+
+    def create_cv_aggregation_visualizations(
+        self,
+        output_dir: Optional[Path] = None,
+        show_individual_folds: bool = True,
+        fold_alpha: float = 0.15,
+    ) -> None:
+        """Create CV aggregation visualizations with faded individual fold lines."""
+        if not self.all_predictions:
+            logger.warning("No predictions stored for CV aggregation visualizations")
+            return
+
+        if output_dir is None:
+            output_dir = self._create_subdir_if_needed("cv_summary")
+            output_dir.mkdir(parents=True, exist_ok=True)
+
+        n_folds = len(self.all_predictions)
+
+        # ROC curve
+        roc_save_path = output_dir / f"roc_cv_aggregation_n{n_folds}.jpg"
+        self.plotter.create_cv_aggregation_plot(
+            fold_predictions=self.all_predictions,
+            curve_type="roc",
+            save_path=roc_save_path,
+            show_individual_folds=show_individual_folds,
+            fold_alpha=fold_alpha,
+            title=f"ROC Curves - Cross Validation (n={n_folds} folds)",
+            verbose=True,
+        )
+        logger.info("Created CV aggregation ROC plot with faded fold lines")
+
+        # PR curve
+        pr_save_path = output_dir / f"pr_cv_aggregation_n{n_folds}.jpg"
+        self.plotter.create_cv_aggregation_plot(
+            fold_predictions=self.all_predictions,
+            curve_type="pr",
+            save_path=pr_save_path,
+            show_individual_folds=show_individual_folds,
+            fold_alpha=fold_alpha,
+            title=f"Precision-Recall Curves - Cross Validation (n={n_folds} folds)",
+            verbose=True,
+        )
+        logger.info("Created CV aggregation PR plot with faded fold lines")
+
+
+# EOF

scitex/ai/classification/reporters/_mixins/_reports.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# Timestamp: "2026-01-24 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex-python/src/scitex/ai/classification/reporters/_mixins/_reports.py
+
+"""
+Reports generation mixin for classification reporter.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Dict
+
+from scitex.logging import getLogger
+
+from ._constants import FOLD_DIR_PREFIX_PATTERN
+
+logger = getLogger(__name__)
+
+
+class ReportsMixin:
+    """Mixin providing report generation methods."""
+
+    def generate_reports(self) -> Dict[str, Path]:
+        """Generate comprehensive reports in multiple formats."""
+        from ..reporter_utils.reporting import generate_org_report
+
+        results = {
+            "config": {
+                "n_folds": len(self.fold_metrics),
+                "output_dir": str(self.output_dir),
+            },
+            "session_config": self.session_config,
+            "summary": {},
+            "folds": [],
+            "plots": {},
+        }
+
+        summary = self.get_summary()
+
+        if "metrics_summary" in summary:
+            results["summary"] = summary["metrics_summary"]
+
+        if "feature-importance" in summary:
+            results["summary"]["feature-importance"] = summary["feature-importance"]
+
+        for fold, fold_data in self.fold_metrics.items():
+            fold_result = {"fold_id": fold}
+            fold_result.update(fold_data)
+
+            try:
+                calling_file_dir = Path(__file__).parent.parent / "reporter_utils"
+                storage_out_path = (
+                    calling_file_dir
+                    / "storage_out"
+                    / self.output_dir
+                    / FOLD_DIR_PREFIX_PATTERN.format(fold=fold)
+                    / "features.json"
+                )
+
+                regular_path = (
+                    self.output_dir
+                    / FOLD_DIR_PREFIX_PATTERN.format(fold=fold)
+                    / "features.json"
+                )
+
+                features_json = None
+                if storage_out_path.exists():
+                    features_json = storage_out_path
+                elif regular_path.exists():
+                    features_json = regular_path
+
+                if features_json:
+                    with open(features_json) as f:
+                        features_data = json.load(f)
+                        for key in [
+                            "n_train",
+                            "n_test",
+                            "n_train_seizure",
+                            "n_train_interictal",
+                            "n_test_seizure",
+                            "n_test_interictal",
+                        ]:
+                            if key in features_data:
+                                fold_result[key] = int(features_data[key])
+            except Exception:
+                pass
+
+            results["folds"].append(fold_result)
+
+        cv_summary_dir = self.output_dir / "cv_summary"
+        if cv_summary_dir.exists():
+            for plot_file in cv_summary_dir.glob("*.jpg"):
+                plot_key = f"cv_summary_{plot_file.stem}"
+                results["plots"][plot_key] = str(plot_file.relative_to(self.output_dir))
+
+        for fold_dir in sorted(self.output_dir.glob("fold_*")):
+            fold_num = fold_dir.name.replace("fold_", "")
+            for plot_file in fold_dir.glob("*.jpg"):
+                plot_key = f"fold_{fold_num}_{plot_file.stem}"
+                results["plots"][plot_key] = str(plot_file.relative_to(self.output_dir))
+
+        reports_dir = self._create_subdir_if_needed("reports")
+        generated_files = {}
+
+        org_path = reports_dir / "classification_report.org"
+        generate_org_report(results, org_path, include_plots=True, convert_formats=True)
+        generated_files["org"] = org_path
+        logger.info(f"Generated org-mode report: {org_path}")
+
+        try:
+            import os
+            import shutil
+            import subprocess
+
+            if shutil.which("pdflatex"):
+                original_dir = Path.cwd()
+                try:
+                    os.chdir(reports_dir)
+
+                    for _ in range(2):
+                        result = subprocess.run(
+                            [
+                                "pdflatex",
+                                "-interaction=nonstopmode",
+                                "classification_report.tex",
+                            ],
+                            capture_output=True,
+                            text=True,
+                            timeout=30,
+                        )
+
+                    pdf_path = reports_dir / "classification_report.pdf"
+                    if pdf_path.exists():
+                        generated_files["pdf"] = pdf_path
+                        logger.info(f"Generated PDF report: {pdf_path}")
+
+                        for ext in [".aux", ".log", ".out", ".toc"]:
+                            aux_file = reports_dir / f"classification_report{ext}"
+                            if aux_file.exists():
+                                aux_file.unlink()
+                finally:
+                    os.chdir(original_dir)
+            else:
+                logger.warning("pdflatex not found. Skipping PDF generation.")
+        except Exception as e:
+            logger.warning(f"Could not generate PDF report: {e}")
+
+        return generated_files
+
+
+# EOF