scitex 2.5.0__py3-none-any.whl → 2.7.3__py3-none-any.whl

scitex/ai/classification/reporters/reporter_utils/__init__.py

@@ -16,65 +16,57 @@ from scitex.ai.metrics import (
     calc_conf_mat,
     calc_clf_report,
     calc_roc_auc,
-    calc_pre_rec_auc
+    calc_pre_rec_auc,
 )
 
 from .storage import (
     MetricStorage,
     save_metric,
-    create_directory_structure_lazy as organize_outputs
+    create_directory_structure_lazy as organize_outputs,
 )
 
-from .validation import (
-    MetricValidator,
-    validate_completeness,
-    check_required_metrics
-)
+from .validation import MetricValidator, validate_completeness, check_required_metrics
 
 from .reporting import (
     generate_markdown_report,
     generate_latex_report,
     create_summary_statistics,
-    export_for_paper
+    export_for_paper,
 )
 
 from .aggregation import (
     aggregate_fold_metrics,
     calculate_mean_std,
     create_summary_table,
-    aggregate_confusion_matrices
+    aggregate_confusion_matrices,
 )
 
 __all__ = [
     # Metrics
-    'calc_bacc',
-    'calc_mcc',
-    'calc_conf_mat',
-    'calc_clf_report',
-    'calc_roc_auc',
-    'calc_pre_rec_auc',
-
+    "calc_bacc",
+    "calc_mcc",
+    "calc_conf_mat",
+    "calc_clf_report",
+    "calc_roc_auc",
+    "calc_pre_rec_auc",
     # Storage
-    'MetricStorage',
-    'save_metric',
-    'save_figure',
-    'save_dataframe',
-    'organize_outputs',
-
+    "MetricStorage",
+    "save_metric",
+    "save_figure",
+    "save_dataframe",
+    "organize_outputs",
     # Validation
-    'MetricValidator',
-    'validate_completeness',
-    'check_required_metrics',
-
+    "MetricValidator",
+    "validate_completeness",
+    "check_required_metrics",
     # Reporting
-    'generate_markdown_report',
-    'generate_latex_report',
-    'create_summary_statistics',
-    'export_for_paper',
-
+    "generate_markdown_report",
+    "generate_latex_report",
+    "create_summary_statistics",
+    "export_for_paper",
     # Aggregation
-    'aggregate_fold_metrics',
-    'calculate_mean_std',
-    'create_summary_table',
-    'aggregate_confusion_matrices'
-]
+    "aggregate_fold_metrics",
+    "calculate_mean_std",
+    "create_summary_table",
+    "aggregate_confusion_matrices",
+]

scitex/ai/classification/reporters/reporter_utils/aggregation.py

@@ -14,24 +14,23 @@ from collections import defaultdict
 
 
 def aggregate_fold_metrics(
-    fold_results: List[Dict[str, Any]],
-    metrics_to_aggregate: Optional[List[str]] = None
+    fold_results: List[Dict[str, Any]], metrics_to_aggregate: Optional[List[str]] = None
 ) -> Dict[str, np.ndarray]:
     """
     Aggregate metrics across folds into arrays.
-    
+
     Parameters
     ----------
     fold_results : List[Dict[str, Any]]
         List of metric dictionaries for each fold
     metrics_to_aggregate : List[str], optional
         Specific metrics to aggregate. If None, aggregate all numeric metrics.
-        
+
     Returns
     -------
     Dict[str, np.ndarray]
         Arrays of metric values across folds
-        
+
     Examples
     --------
     >>> fold_results = [
@@ -43,7 +42,7 @@ def aggregate_fold_metrics(
     >>> print(f"BA values: {aggregated['balanced_accuracy']}")
     """
     aggregated = defaultdict(list)
-    
+
     # Determine metrics to aggregate
     if metrics_to_aggregate is None:
         # Find all numeric metrics
@@ -53,55 +52,54 @@ def aggregate_fold_metrics(
                 # Check if value is numeric or contains numeric 'value' field
                 if isinstance(value, (int, float)):
                     all_metrics.add(key)
-                elif isinstance(value, dict) and 'value' in value:
-                    if isinstance(value['value'], (int, float)):
+                elif isinstance(value, dict) and "value" in value:
+                    if isinstance(value["value"], (int, float)):
                         all_metrics.add(key)
         metrics_to_aggregate = list(all_metrics)
-    
+
     # Aggregate each metric
     for fold in fold_results:
         for metric in metrics_to_aggregate:
             if metric in fold:
                 value = fold[metric]
-                
+
                 # Extract numeric value
                 if isinstance(value, dict):
-                    if 'value' in value:
-                        value = value['value']
+                    if "value" in value:
+                        value = value["value"]
                     else:
                         continue
-                
+
                 if isinstance(value, (int, float)):
                     aggregated[metric].append(float(value))
-    
+
     # Convert lists to numpy arrays
     result = {}
     for metric, values in aggregated.items():
         if values:  # Only include metrics with data
             result[metric] = np.array(values)
-    
+
     return result
 
 
 def calculate_mean_std(
-    values: Union[List[float], np.ndarray],
-    ddof: int = 1
+    values: Union[List[float], np.ndarray], ddof: int = 1
 ) -> Tuple[float, float]:
     """
     Calculate mean and standard deviation.
-    
+
     Parameters
     ----------
     values : Union[List[float], np.ndarray]
         Values to calculate statistics for
     ddof : int
         Degrees of freedom for std calculation
-        
+
     Returns
     -------
     Tuple[float, float]
         (mean, std)
-        
+
     Examples
     --------
     >>> mean, std = calculate_mean_std([0.85, 0.87, 0.83])
@@ -117,11 +115,11 @@ def create_summary_table(
     fold_results: List[Dict[str, Any]],
     metrics: Optional[List[str]] = None,
     include_stats: bool = True,
-    format_digits: int = 3
+    format_digits: int = 3,
 ) -> pd.DataFrame:
     """
     Create a summary table with fold results and statistics.
-    
+
     Parameters
     ----------
     fold_results : List[Dict[str, Any]]
@@ -132,12 +130,12 @@ def create_summary_table(
         Whether to include mean/std rows
     format_digits : int
         Number of decimal places
-        
+
     Returns
     -------
     pd.DataFrame
         Summary table with folds as rows and metrics as columns
-        
+
     Examples
     --------
     >>> fold_results = [
@@ -150,92 +148,88 @@ def create_summary_table(
     """
     # Prepare data for DataFrame
     data = []
-    
+
     # Determine metrics to include
     if metrics is None:
         metrics = set()
         for fold in fold_results:
             for key, value in fold.items():
-                if key not in ['fold', 'fold_id', 'fold']:
+                if key not in ["fold", "fold_id", "fold"]:
                     # Check if it's a numeric metric
                     if isinstance(value, (int, float)):
                         metrics.add(key)
-                    elif isinstance(value, dict) and 'value' in value:
-                        if isinstance(value['value'], (int, float)):
+                    elif isinstance(value, dict) and "value" in value:
+                        if isinstance(value["value"], (int, float)):
                             metrics.add(key)
         metrics = sorted(list(metrics))
-    
+
     # Add fold results
     for i, fold in enumerate(fold_results):
-        row = {'Fold': fold.get('fold_id', fold.get('fold', i))}
-        
+        row = {"Fold": fold.get("fold_id", fold.get("fold", i))}
+
         for metric in metrics:
             if metric in fold:
                 value = fold[metric]
-                
+
                 # Extract numeric value
-                if isinstance(value, dict) and 'value' in value:
-                    value = value['value']
-                
+                if isinstance(value, dict) and "value" in value:
+                    value = value["value"]
+
                 if isinstance(value, (int, float)):
-                    row[metric.replace('_', ' ').title()] = value
+                    row[metric.replace("_", " ").title()] = value
                 else:
-                    row[metric.replace('_', ' ').title()] = None
+                    row[metric.replace("_", " ").title()] = None
             else:
-                row[metric.replace('_', ' ').title()] = None
-        
+                row[metric.replace("_", " ").title()] = None
+
         data.append(row)
-    
+
     # Create DataFrame
     df = pd.DataFrame(data)
-    
+
     # Add statistics rows if requested
     if include_stats and len(data) > 0:
         # Calculate statistics for each metric
-        mean_row = {'Fold': 'Mean'}
-        std_row = {'Fold': 'Std'}
-        
+        mean_row = {"Fold": "Mean"}
+        std_row = {"Fold": "Std"}
+
         for col in df.columns:
-            if col != 'Fold':
+            if col != "Fold":
                 values = df[col].dropna().values
                 if len(values) > 0:
                     mean, std = calculate_mean_std(values)
                     mean_row[col] = mean
                     std_row[col] = std
-        
+
         # Append statistics rows
-        df = pd.concat([
-            df,
-            pd.DataFrame([mean_row]),
-            pd.DataFrame([std_row])
-        ], ignore_index=True)
-    
+        df = pd.concat(
+            [df, pd.DataFrame([mean_row]), pd.DataFrame([std_row])], ignore_index=True
+        )
+
     # Format numeric columns
     for col in df.columns:
-        if col != 'Fold':
+        if col != "Fold":
             df[col] = df[col].apply(
-                lambda x: f"{x:.{format_digits}f}" if pd.notna(x) else 'N/A'
+                lambda x: f"{x:.{format_digits}f}" if pd.notna(x) else "N/A"
             )
-    
+
     return df
 
 
-def aggregate_confusion_matrices(
-    confusion_matrices: List[np.ndarray]
-) -> np.ndarray:
+def aggregate_confusion_matrices(confusion_matrices: List[np.ndarray]) -> np.ndarray:
     """
     Aggregate confusion matrices across folds.
-    
+
     Parameters
     ----------
     confusion_matrices : List[np.ndarray]
         List of confusion matrices from each fold
-        
+
     Returns
     -------
     np.ndarray
         Summed confusion matrix
-        
+
     Examples
     --------
     >>> cms = [np.array([[8, 2], [1, 9]]) for _ in range(3)]
@@ -244,37 +238,38 @@ def aggregate_confusion_matrices(
     """
     if not confusion_matrices:
         raise ValueError("No confusion matrices provided")
-    
+
     # Check all matrices have same shape
     shape = confusion_matrices[0].shape
     for cm in confusion_matrices[1:]:
         if cm.shape != shape:
-            raise ValueError(f"Inconsistent confusion matrix shapes: {shape} vs {cm.shape}")
-    
+            raise ValueError(
+                f"Inconsistent confusion matrix shapes: {shape} vs {cm.shape}"
+            )
+
     # Sum all matrices
     total = np.sum(confusion_matrices, axis=0)
     return total
 
 
 def aggregate_classification_reports(
-    reports: List[pd.DataFrame],
-    weighted_average: bool = True
+    reports: List[pd.DataFrame], weighted_average: bool = True
 ) -> pd.DataFrame:
     """
     Aggregate classification reports across folds.
-    
+
     Parameters
     ----------
     reports : List[pd.DataFrame]
         List of classification report DataFrames
     weighted_average : bool
         Whether to use weighted average based on support
-        
+
     Returns
     -------
     pd.DataFrame
         Aggregated classification report
-        
+
     Examples
     --------
     >>> reports = [report_fold1_df, report_fold2_df, report_fold3_df]
@@ -283,75 +278,73 @@ def aggregate_classification_reports(
     """
     if not reports:
         raise ValueError("No classification reports provided")
-    
+
     # Get all class labels (rows)
     all_labels = set()
     for report in reports:
         all_labels.update(report.index.tolist())
     all_labels = sorted(list(all_labels))
-    
+
     # Initialize aggregated data
     aggregated = {}
-    
+
     for label in all_labels:
-        label_data = {'precision': [], 'recall': [], 'f1-score': [], 'support': []}
-        
+        label_data = {"precision": [], "recall": [], "f1-score": [], "support": []}
+
         for report in reports:
             if label in report.index:
                 row = report.loc[label]
-                for metric in ['precision', 'recall', 'f1-score']:
+                for metric in ["precision", "recall", "f1-score"]:
                     if metric in row:
                         label_data[metric].append(row[metric])
-                if 'support' in row:
-                    label_data['support'].append(row['support'])
-        
+                if "support" in row:
+                    label_data["support"].append(row["support"])
+
         # Calculate aggregated metrics
-        if label_data['support'] and weighted_average:
+        if label_data["support"] and weighted_average:
             # Weighted average based on support
-            total_support = sum(label_data['support'])
-            weights = np.array(label_data['support']) / total_support
-            
+            total_support = sum(label_data["support"])
+            weights = np.array(label_data["support"]) / total_support
+
             aggregated[label] = {}
-            for metric in ['precision', 'recall', 'f1-score']:
+            for metric in ["precision", "recall", "f1-score"]:
                 if label_data[metric]:
                     aggregated[label][metric] = np.average(
-                        label_data[metric], 
-                        weights=weights[:len(label_data[metric])]
+                        label_data[metric], weights=weights[: len(label_data[metric])]
                     )
-            aggregated[label]['support'] = sum(label_data['support'])
+            aggregated[label]["support"] = sum(label_data["support"])
         else:
             # Simple average
             aggregated[label] = {}
-            for metric in ['precision', 'recall', 'f1-score']:
+            for metric in ["precision", "recall", "f1-score"]:
                 if label_data[metric]:
                     aggregated[label][metric] = np.mean(label_data[metric])
-            if label_data['support']:
-                aggregated[label]['support'] = sum(label_data['support'])
-    
+            if label_data["support"]:
+                aggregated[label]["support"] = sum(label_data["support"])
+
     # Convert to DataFrame
-    agg_df = pd.DataFrame.from_dict(aggregated, orient='index')
+    agg_df = pd.DataFrame.from_dict(aggregated, orient="index")
     return agg_df
 
 
 def calculate_metric_confidence_interval(
-    values: Union[List[float], np.ndarray],
-    confidence: float = 0.95
+    values: Union[List[float], np.ndarray], confidence: float = 0.95
 ) -> Tuple[float, float, float]:
     """
     Calculate confidence interval for a metric.
-    
+
     Parameters
     ----------
     values : Union[List[float], np.ndarray]
         Metric values across folds
     confidence : float
         Confidence level (e.g., 0.95 for 95%)
-        
+
     Returns
     -------
     Tuple[float, float, float]
         (mean, lower_bound, upper_bound)
-        
+
     Examples
     --------
     >>> values = [0.85, 0.87, 0.83, 0.86, 0.84]
@@ -359,99 +352,88 @@ def calculate_metric_confidence_interval(
     >>> print(f"Mean: {mean:.3f} [{lower:.3f}, {upper:.3f}]")
     """
     from scipy import stats
-    
+
     values = np.asarray(values)
     n = len(values)
-    
+
     if n < 2:
         # Not enough data for confidence interval
         mean = float(values[0]) if n == 1 else 0.0
         return mean, mean, mean
-    
+
     mean = np.mean(values)
     sem = stats.sem(values)  # Standard error of the mean
-    
+
     # Calculate confidence interval
-    interval = stats.t.interval(
-        confidence,
-        n - 1,
-        loc=mean,
-        scale=sem
-    )
-    
+    interval = stats.t.interval(confidence, n - 1, loc=mean, scale=sem)
+
     return float(mean), float(interval[0]), float(interval[1])
 
 
-def merge_fold_results(
-    results_dir: Union[str, Path],
-    n_folds: int
-) -> Dict[str, Any]:
+def merge_fold_results(results_dir: Union[str, Path], n_folds: int) -> Dict[str, Any]:
     """
     Merge results from multiple fold directories.
-    
+
     Parameters
     ----------
     results_dir : Union[str, Path]
         Base directory containing fold subdirectories
     n_folds : int
         Number of folds to merge
-        
+
     Returns
     -------
     Dict[str, Any]
         Merged results dictionary
-        
+
     Examples
     --------
     >>> merged = merge_fold_results("./results", n_folds=5)
     >>> print(f"Found {len(merged['folds'])} folds")
     """
     results_dir = Path(results_dir)
-    merged = {
-        'folds': [],
-        'metrics': defaultdict(list)
-    }
-    
+    merged = {"folds": [], "metrics": defaultdict(list)}
+
     # Load each fold
     for fold in range(n_folds):
         fold_dir = results_dir / f"fold_{fold:02d}"
         if not fold_dir.exists():
             fold_dir = results_dir / f"fold_{fold}"  # Try without padding
-        
+
         if fold_dir.exists():
-            fold_data = {'fold_id': fold}
-            
+            fold_data = {"fold_id": fold}
+
             # Load metrics
-            metrics_dir = fold_dir / 'metrics'
+            metrics_dir = fold_dir / "metrics"
             if metrics_dir.exists():
-                for metric_file in metrics_dir.glob('*.json'):
-                    with open(metric_file, 'r') as f:
+                for metric_file in metrics_dir.glob("*.json"):
+                    with open(metric_file, "r") as f:
                         metric_data = json.load(f)
                         metric_name = metric_file.stem
                         fold_data[metric_name] = metric_data
-                        
+
                         # Add to aggregated metrics
-                        if 'value' in metric_data:
-                            merged['metrics'][metric_name].append(metric_data['value'])
-            
+                        if "value" in metric_data:
+                            merged["metrics"][metric_name].append(metric_data["value"])
+
             # Load confusion matrix if exists
-            cm_path = fold_dir / 'confusion_matrix.npy'
+            cm_path = fold_dir / "confusion_matrix.npy"
             if cm_path.exists():
-                fold_data['confusion_matrix'] = np.load(cm_path)
-            
-            merged['folds'].append(fold_data)
-    
+                fold_data["confusion_matrix"] = np.load(cm_path)
+
+            merged["folds"].append(fold_data)
+
     # Calculate summary statistics
-    merged['summary'] = {}
-    for metric_name, values in merged['metrics'].items():
+    merged["summary"] = {}
+    for metric_name, values in merged["metrics"].items():
         if values:
             mean, std = calculate_mean_std(values)
-            merged['summary'][metric_name] = {
-                'mean': mean,
-                'std': std,
-                'min': float(np.min(values)),
-                'max': float(np.max(values)),
-                'values': values
+            merged["summary"][metric_name] = {
+                "mean": mean,
+                "std": std,
+                "min": float(np.min(values)),
+                "max": float(np.max(values)),
+                "values": values,
             }
-    
+
     return merged