nkululeko 0.88.3__tar.gz → 0.88.5__tar.gz

{nkululeko-0.88.3 → nkululeko-0.88.5}/CHANGELOG.md

@@ -1,6 +1,16 @@
 Changelog
 =========
 
+Version 0.88.5
+--------------
+* add a unique name to the uncertainty plot
+* fix error in speaker embedding (still need speechbrain < 1.0)
+* add get_target_name function in util
+
+Version 0.88.4
+--------------
+* added more ensemble methods, e.g. based on uncertainty
+
 Version 0.88.3
 --------------
 * fixed bug in false uncertainty estimation

{nkululeko-0.88.3/nkululeko.egg-info → nkululeko-0.88.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.88.3
+Version: 0.88.5
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -68,7 +68,8 @@ A project to detect speaker characteristics by machine learning experiments with
 
 The idea is to have a framework (based on e.g. sklearn and torch) that can be used to rapidly and automatically analyse audio data and explore machine learning models based on that data.
 
-* NEW with nkululeko: [Finetune transformer-models](http://blog.syntheticspeech.de/2024/05/29/nkululeko-how-to-finetune-a-transformer-model/)
+* NEW with nkululek: [Ensemble learning](http://blog.syntheticspeech.de/2024/06/25/nkululeko-ensemble-classifiers-with-late-fusion/)
+* NEW: [Finetune transformer-models](http://blog.syntheticspeech.de/2024/05/29/nkululeko-how-to-finetune-a-transformer-model/)
 * The latest features can be seen in [the ini-file](./ini_file.md) options that are used to control Nkululeko
 * Below is a [Hello World example](#helloworld) that should set you up fastly, also on [Google Colab](https://colab.research.google.com/drive/1GYNBd5cdZQ1QC3Jm58qoeMaJg3UuPhjw?usp=sharing#scrollTo=4G_SjuF9xeQf), and [with Kaggle](https://www.kaggle.com/felixburk/nkululeko-hello-world-example)
 * [Here's a blog post on how to set up nkululeko on your computer.](http://blog.syntheticspeech.de/2021/08/30/how-to-set-up-your-first-nkululeko-project/)
@@ -203,7 +204,8 @@ All of them take *--config <my_config.ini>* as an argument.
 * **nkululeko.nkululeko**: do machine learning experiments combining features and learners
 * **nkululeko.ensemble**: [combine several nkululeko experiments](http://blog.syntheticspeech.de/2024/06/25/nkululeko-ensemble-classifiers-with-late-fusion/) and report on late fusion results
   * *configurations*: which experiments to combine
-  * *--method* (optional): majority_voting, mean, max, sum
+  * *--method* (optional): majority_voting, mean, max, sum, max_class, uncertainty_threshold, uncertainty_weighted, confidence_weighted
+  * *--threshold*: uncertainty threshold (1.0 means no threshold)
   * *--outfile* (optional): name of CSV file for output
   * *--no_labels* (optional): indicate that no ground truth is given 
 * **nkululeko.multidb**: do [multiple experiments](http://blog.syntheticspeech.de/2024/01/02/nkululeko-compare-several-databases/), comparing several databases cross and in itself
@@ -273,6 +275,8 @@ There's my [blog](http://blog.syntheticspeech.de/?s=nkululeko) with tutorials:
 * [Tweak the target variable for database comparison](http://blog.syntheticspeech.de/2024/03/13/nkululeko-how-to-tweak-the-target-variable-for-database-comparison/)
 * [How to run multiple experiments in one go](http://blog.syntheticspeech.de/2022/03/28/how-to-run-multiple-experiments-in-one-go-with-nkululeko/)
 * [How to finetune a transformer-model](http://blog.syntheticspeech.de/2024/05/29/nkululeko-how-to-finetune-a-transformer-model/)
+* [Ensemble (combine) classifiers with late-fusion](http://blog.syntheticspeech.de/2024/06/25/nkululeko-ensemble-classifiers-with-late-fusion/)
+
 
 ### <a name="helloworld">Hello World example</a>
 * NEW: [Here's a Google colab that runs this example out-of-the-box](https://colab.research.google.com/drive/1GYNBd5cdZQ1QC3Jm58qoeMaJg3UuPhjw?usp=sharing#scrollTo=4G_SjuF9xeQf), and here is the same [with Kaggle](https://www.kaggle.com/felixburk/nkululeko-hello-world-example)
@@ -356,6 +360,16 @@ F. Burkhardt, Johannes Wagner, Hagen Wierstorf, Florian Eyben and Björn Schulle
 Changelog
 =========
 
+Version 0.88.5
+--------------
+* add a unique name to the uncertainty plot
+* fix error in speaker embedding (still need speechbrain < 1.0)
+* add get_target_name function in util
+
+Version 0.88.4
+--------------
+* added more ensemble methods, e.g. based on uncertainty
+
 Version 0.88.3
 --------------
 * fixed bug in false uncertainty estimation

{nkululeko-0.88.3 → nkululeko-0.88.5}/README.md

@@ -24,7 +24,8 @@ A project to detect speaker characteristics by machine learning experiments with
 
 The idea is to have a framework (based on e.g. sklearn and torch) that can be used to rapidly and automatically analyse audio data and explore machine learning models based on that data.
 
-* NEW with nkululeko: [Finetune transformer-models](http://blog.syntheticspeech.de/2024/05/29/nkululeko-how-to-finetune-a-transformer-model/)
+* NEW with nkululek: [Ensemble learning](http://blog.syntheticspeech.de/2024/06/25/nkululeko-ensemble-classifiers-with-late-fusion/)
+* NEW: [Finetune transformer-models](http://blog.syntheticspeech.de/2024/05/29/nkululeko-how-to-finetune-a-transformer-model/)
 * The latest features can be seen in [the ini-file](./ini_file.md) options that are used to control Nkululeko
 * Below is a [Hello World example](#helloworld) that should set you up fastly, also on [Google Colab](https://colab.research.google.com/drive/1GYNBd5cdZQ1QC3Jm58qoeMaJg3UuPhjw?usp=sharing#scrollTo=4G_SjuF9xeQf), and [with Kaggle](https://www.kaggle.com/felixburk/nkululeko-hello-world-example)
 * [Here's a blog post on how to set up nkululeko on your computer.](http://blog.syntheticspeech.de/2021/08/30/how-to-set-up-your-first-nkululeko-project/)
@@ -159,7 +160,8 @@ All of them take *--config <my_config.ini>* as an argument.
 * **nkululeko.nkululeko**: do machine learning experiments combining features and learners
 * **nkululeko.ensemble**: [combine several nkululeko experiments](http://blog.syntheticspeech.de/2024/06/25/nkululeko-ensemble-classifiers-with-late-fusion/) and report on late fusion results
   * *configurations*: which experiments to combine
-  * *--method* (optional): majority_voting, mean, max, sum
+  * *--method* (optional): majority_voting, mean, max, sum, max_class, uncertainty_threshold, uncertainty_weighted, confidence_weighted
+  * *--threshold*: uncertainty threshold (1.0 means no threshold)
   * *--outfile* (optional): name of CSV file for output
   * *--no_labels* (optional): indicate that no ground truth is given 
 * **nkululeko.multidb**: do [multiple experiments](http://blog.syntheticspeech.de/2024/01/02/nkululeko-compare-several-databases/), comparing several databases cross and in itself
@@ -229,6 +231,8 @@ There's my [blog](http://blog.syntheticspeech.de/?s=nkululeko) with tutorials:
 * [Tweak the target variable for database comparison](http://blog.syntheticspeech.de/2024/03/13/nkululeko-how-to-tweak-the-target-variable-for-database-comparison/)
 * [How to run multiple experiments in one go](http://blog.syntheticspeech.de/2022/03/28/how-to-run-multiple-experiments-in-one-go-with-nkululeko/)
 * [How to finetune a transformer-model](http://blog.syntheticspeech.de/2024/05/29/nkululeko-how-to-finetune-a-transformer-model/)
+* [Ensemble (combine) classifiers with late-fusion](http://blog.syntheticspeech.de/2024/06/25/nkululeko-ensemble-classifiers-with-late-fusion/)
+
 
 ### <a name="helloworld">Hello World example</a>
 * NEW: [Here's a Google colab that runs this example out-of-the-box](https://colab.research.google.com/drive/1GYNBd5cdZQ1QC3Jm58qoeMaJg3UuPhjw?usp=sharing#scrollTo=4G_SjuF9xeQf), and here is the same [with Kaggle](https://www.kaggle.com/felixburk/nkululeko-hello-world-example)

{nkululeko-0.88.3 → nkululeko-0.88.5}/nkululeko/constants.py

@@ -1,2 +1,2 @@
-VERSION="0.88.3"
+VERSION="0.88.5"
 SAMPLING_RATE = 16000

nkululeko-0.88.5/nkululeko/ensemble.py

@@ -0,0 +1,343 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from typing import List
+import configparser
+import time
+from argparse import ArgumentParser
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import balanced_accuracy_score
+
+from nkululeko.constants import VERSION
+from nkululeko.experiment import Experiment
+from nkululeko.utils.util import Util
+
+# import torch
+
+# Constants
+DEFAULT_METHOD = "mean"
+DEFAULT_OUTFILE = "ensemble_result.csv"
+
+
+def majority_voting(ensemble_preds_ls):
+    all_predictions = pd.concat([df["predicted"] for df in ensemble_preds_ls], axis=1)
+    return all_predictions.mode(axis=1).iloc[:, 0]
+
+
+def mean_ensemble(ensemble_preds, labels):
+    for label in labels:
+        ensemble_preds[label] = ensemble_preds[label].mean(axis=1)
+    return ensemble_preds[labels].idxmax(axis=1)
+
+
+def max_ensemble(ensemble_preds, labels):
+    for label in labels:
+        ensemble_preds[label] = ensemble_preds[label].max(axis=1)
+    return ensemble_preds[labels].idxmax(axis=1)
+
+
+def sum_ensemble(ensemble_preds, labels):
+    for label in labels:
+        ensemble_preds[label] = ensemble_preds[label].sum(axis=1)
+    return ensemble_preds[labels].idxmax(axis=1)
+
+
+def uncertainty_ensemble(ensemble_preds):
+    """Same as uncertainty_threshold with a threshold of 0.1"""
+    final_predictions = []
+    best_uncertainty = []
+    for _, row in ensemble_preds.iterrows():
+        uncertainties = row[["uncertainty"]].values
+        min_uncertainty_idx = np.argmin(uncertainties)
+        final_predictions.append(row["predicted"].iloc[min_uncertainty_idx])
+        best_uncertainty.append(uncertainties[min_uncertainty_idx])
+
+    return final_predictions, best_uncertainty
+
+
+def max_class_ensemble(ensemble_preds_ls, labels):
+    """Compare the highest probabilites of all models across classes (instead of same class as in max_ensemble) and return the highest probability and the class"""
+    final_preds = []
+    final_probs = []
+
+    for _, row in pd.concat(ensemble_preds_ls, axis=1).iterrows():
+        max_probs = []
+        max_classes = []
+
+        for model_df in ensemble_preds_ls:
+            model_probs = row[labels].astype(float)
+            max_prob = model_probs.max()
+            max_class = model_probs.idxmax()
+
+            max_probs.append(max_prob)
+            max_classes.append(max_class)
+
+        best_model_index = np.argmax(max_probs)
+
+        final_preds.append(max_classes[best_model_index])
+        final_probs.append(max_probs[best_model_index])
+
+    return pd.Series(final_preds), pd.Series(final_probs)
+
+
+def uncertainty_threshold_ensemble(ensemble_preds_ls, labels, threshold):
+    final_predictions = []
+    final_uncertainties = []
+
+    for idx in ensemble_preds_ls[0].index:
+        uncertainties = [df.loc[idx, "uncertainty"] for df in ensemble_preds_ls]
+        min_uncertainty_idx = np.argmin(uncertainties)
+        min_uncertainty = uncertainties[min_uncertainty_idx]
+
+        if min_uncertainty <= threshold:
+            # Use the prediction with low uncertainty
+            final_predictions.append(
+                ensemble_preds_ls[min_uncertainty_idx].loc[idx, "predicted"]
+            )
+            final_uncertainties.append(min_uncertainty)
+        else:  # for uncertainty above threshold
+            # Calculate mean of probabilities same class different model
+            mean_probs = np.mean(
+                [df.loc[idx, labels].values for df in ensemble_preds_ls], axis=0
+            )
+            final_predictions.append(labels[np.argmax(mean_probs)])
+            final_uncertainties.append(np.mean(uncertainties))
+
+    return final_predictions
+
+
+def uncertainty_weighted_ensemble(ensemble_preds_ls, labels):
+    """Weighted ensemble based on uncertainty, normalized for each class"""
+    final_predictions = []
+    final_uncertainties = []
+
+    for idx in ensemble_preds_ls[0].index:
+        uncertainties = [df.loc[idx, "uncertainty"] for df in ensemble_preds_ls]
+        # Convert uncertainties to accuracies/confidence
+        accuracies = [1 - uncertainty for uncertainty in uncertainties]
+
+        # Calculate weights (inverse of uncertainties)
+        weights = [
+            1 / uncertainty if uncertainty != 0 else 1e10
+            for uncertainty in uncertainties
+        ]
+
+        # Normalize weights for each class
+        total_weight = sum(weights)
+        normalized_weights = [w / total_weight for w in weights]
+
+        # Calculate weighted probabilities for each class
+        weighted_probs = {label: 0 for label in labels}
+        for df, weight in zip(ensemble_preds_ls, normalized_weights):
+            for label in labels:
+                weighted_probs[label] += df.loc[idx, label] * weight
+
+        # Select the class with the highest weighted probability
+        predicted_class = max(weighted_probs, key=weighted_probs.get)
+        final_predictions.append(predicted_class)
+
+        # Use the lowest accuracy as the final uncertainty
+        final_uncertainties.append(1 - min(accuracies))
+
+    return final_predictions, final_uncertainties
+
+
+def confidence_weighted_ensemble(ensemble_preds_ls, labels):
+    """Weighted ensemble based on confidence, normalized for all samples per model"""
+    final_predictions = []
+    final_confidences = []
+
+    for idx in ensemble_preds_ls[0].index:
+        class_probabilities = {label: 0 for label in labels}
+        total_confidence = 0
+
+        for df in ensemble_preds_ls:
+            row = df.loc[idx]
+            confidence = 1 - row["uncertainty"]  # confidence score
+            total_confidence += confidence
+
+            for label in labels:
+                class_probabilities[label] += row[label] * confidence
+
+        # Normalize probabilities
+        for label in labels:
+            class_probabilities[label] /= total_confidence
+
+        predicted_class = max(class_probabilities, key=class_probabilities.get)
+        final_predictions.append(predicted_class)
+        final_confidences.append(max(class_probabilities.values()))
+
+    return final_predictions, final_confidences
+
+
+def ensemble_predictions(
+    config_files: List[str], method: str, threshold: float, no_labels: bool
+) -> pd.DataFrame:
+    """
+    Ensemble predictions from multiple experiments.
+
+    Args:
+        config_files (list): List of configuration file paths.
+        method (str): Ensemble method to use. Options are 'majority_voting', 'mean', 'max', or 'sum'.
+        no_labels (bool): Flag indicating whether the predictions have labels or not.
+
+    Returns:
+        pandas.DataFrame: The ensemble predictions.
+
+    Raises:
+        ValueError: If an unknown ensemble method is provided.
+        AssertionError: If the number of config files is less than 2 for majority voting.
+
+    """
+    ensemble_preds_ls = []
+    for config_file in config_files:
+        if no_labels:
+            # for ensembling results from Nkululeko.demo
+            preds = pd.read_csv(config_file)
+            labels = preds.columns[1:-2]
+        else:
+            # for ensembling results from Nkululeko.nkululeko
+            config = configparser.ConfigParser()
+            config.read(config_file)
+            expr = Experiment(config)
+            module = "ensemble"
+            expr.set_module(module)
+            util = Util(module, has_config=True)
+            util.debug(
+                f"running {expr.name} from config {config_file}, nkululeko version"
+                f" {VERSION}"
+            )
+
+            # get labels
+            labels = expr.util.get_labels()
+            # load the experiment
+            # get CSV files of predictions
+            pred_name = expr.util.get_pred_name()
+            util.debug(f"Loading predictions from {pred_name}")
+            preds = pd.read_csv(pred_name)
+
+        ensemble_preds_ls.append(preds)
+
+    # pd concate
+    ensemble_preds = pd.concat(ensemble_preds_ls, axis=1)
+
+    if method == "majority_voting":
+        assert (
+            len(ensemble_preds_ls) > 2
+        ), "Majority voting only works for more than two models"
+        ensemble_preds["predicted"] = majority_voting(ensemble_preds_ls)
+    elif method == "mean":
+        ensemble_preds["predicted"] = mean_ensemble(ensemble_preds, labels)
+    elif method == "max":
+        ensemble_preds["predicted"] = max_ensemble(ensemble_preds, labels)
+    elif method == "sum":
+        ensemble_preds["predicted"] = sum_ensemble(ensemble_preds, labels)
+    elif method == "max_class":
+        ensemble_preds["predicted"], ensemble_preds["max_probability"] = (
+            max_class_ensemble(ensemble_preds_ls, labels)
+        )
+    elif method == "uncertainty_threshold":
+        ensemble_preds["predicted"] = uncertainty_threshold_ensemble(
+            ensemble_preds_ls, labels, threshold
+        )
+    elif method == "uncertainty_weighted":
+        ensemble_preds["predicted"], ensemble_preds["uncertainty"] = (
+            uncertainty_weighted_ensemble(ensemble_preds_ls, labels)
+        )
+    elif method == "confidence_weighted":
+        ensemble_preds["predicted"], ensemble_preds["confidence"] = (
+            confidence_weighted_ensemble(ensemble_preds_ls, labels)
+        )
+    else:
+        raise ValueError(f"Unknown ensemble method: {method}")
+
+    # get the highest value from all labels to infer the label
+    # replace the old first predicted column
+    if method in ["mean", "max", "sum"]:
+        ensemble_preds["predicted"] = ensemble_preds[labels].idxmax(axis=1)
+
+    if no_labels:
+        return ensemble_preds
+
+    # Drop start, end columns
+    ensemble_preds = ensemble_preds.drop(columns=["start", "end"])
+
+    # Drop other column except until truth
+    ensemble_preds = ensemble_preds.iloc[:, : len(labels) + 3]
+
+    # calculate UAR from predicted and truth columns
+
+    truth = ensemble_preds["truth"]
+    predicted = ensemble_preds["predicted"]
+    uar = balanced_accuracy_score(truth, predicted)
+    acc = (truth == predicted).mean()
+    Util("ensemble").debug(f"{method}: UAR: {uar:.3f}, ACC: {acc:.3f}")
+
+    return ensemble_preds
+
+
+def main(src_dir: Path) -> None:
+    parser = ArgumentParser()
+    parser.add_argument(
+        "configs",
+        nargs="+",
+        help="Paths to the configuration files of the experiments to ensemble. \
+             Can be INI files for Nkululeko.nkululeo or CSV files from Nkululeko.demo.",
+    )
+    parser.add_argument(
+        "--method",
+        default=DEFAULT_METHOD,
+        choices=[
+            "majority_voting",
+            "mean",
+            "max",
+            "sum",
+            "max_class",
+            # "uncertainty_lowest",
+            # "entropy",
+            "uncertainty_threshold",
+            "uncertainty_weighted",
+            "confidence_weighted",
+        ],
+        help=f"Ensemble method to use (default: {DEFAULT_METHOD})",
+    )
+    # add threshold if method is uncertainty_threshold
+    parser.add_argument(
+        "--threshold",
+        default=1.0,
+        type=float,
+        help="Threshold for uncertainty_threshold method (default: 1.0, i.e. no threshold)",
+    )
+    parser.add_argument(
+        "--outfile",
+        default=DEFAULT_OUTFILE,
+        help=f"Output file path for the ensemble predictions (default: {DEFAULT_OUTFILE})",
+    )
+    parser.add_argument(
+        "--no_labels",
+        action="store_true",
+        help="True if true labels are not available. For Nkululeko.demo results.",
+    )
+
+    args = parser.parse_args()
+
+    start = time.time()
+
+    ensemble_preds = ensemble_predictions(
+        args.configs, args.method, args.threshold, args.no_labels
+    )
+
+    # save to csv
+    ensemble_preds.to_csv(args.outfile, index=False)
+    Util("ensemble").debug(f"Ensemble predictions saved to: {args.outfile}")
+    Util("ensemble").debug(f"Ensemble done, used {time.time()-start:.2f} seconds")
+
+    Util("ensemble").debug("DONE")
+
+
+if __name__ == "__main__":
+    cwd = Path(__file__).parent
+    main(cwd)

{nkululeko-0.88.3 → nkululeko-0.88.5}/nkululeko/feat_extract/feats_agender.py

@@ -1,13 +1,14 @@
 # feats_agender.py
+
 from nkululeko.feat_extract.featureset import Featureset
 import os
-import pandas as pd
+# import pandas as pd
 import audeer
 import nkululeko.glob_conf as glob_conf
 import audonnx
 import numpy as np
 import audinterface
-
+import torch
 
 class AgenderSet(Featureset):
     """
@@ -32,7 +33,8 @@ class AgenderSet(Featureset):
             archive_path = audeer.download_url(
                 model_url, cache_root, verbose=True)
             audeer.extract_archive(archive_path, model_root)
-        device = self.util.config_val("MODEL", "device", "cpu")
+        cuda = "cuda" if torch.cuda.is_available() else "cpu"
+        device = self.util.config_val("MODEL", "device", cuda)
         self.model = audonnx.load(model_root, device=device)
         self.util.debug(f"initialized agender model")
         self.model_loaded = True

{nkululeko-0.88.3 → nkululeko-0.88.5}/nkululeko/feat_extract/feats_ast.py

@@ -100,10 +100,8 @@ class Ast(Featureset):
             embeddings = torch.mean(last_hidden_state, dim=1)
             embeddings = embeddings.cpu().numpy()
 
-            # convert the same from (768,) to (1, 768)
-            # embeddings = embeddings.reshape(1, -1)
-            print(f"hs shape: {embeddings.shape}")
-            
+            # print(f"hs shape: {embeddings.shape}")
+            # hs shape: (1, 768)
         
         except Exception as e:
             self.util.error(f"Error extracting embeddings for file {file}: {str(e)}, fill with")

{nkululeko-0.88.3 → nkululeko-0.88.5}/nkululeko/feat_extract/feats_spkrec.py

@@ -24,7 +24,7 @@ class Spkrec(Featureset):
     def __init__(self, name, data_df, feat_type):
         """Constructor. is_train is needed to distinguish from test/dev sets,
         because they use the codebook from the training"""
-        super().__init__(name, data_df)
+        super().__init__(name, data_df, feat_type)
         # check if device is not set, use cuda if available
         cuda = "cuda" if torch.cuda.is_available() else "cpu"
         self.device = self.util.config_val("MODEL", "device", cuda)

{nkululeko-0.88.3 → nkululeko-0.88.5}/nkululeko/modelrunner.py

@@ -238,21 +238,21 @@ class Modelrunner:
             if balancing == "ros":
                 from imblearn.over_sampling import RandomOverSampler
 
-                sampler = RandomOverSampler()
+                sampler = RandomOverSampler(random_state=42)
                 X_res, y_res = sampler.fit_resample(
                     self.feats_train, self.df_train[self.target]
                 )
             elif balancing == "smote":
                 from imblearn.over_sampling import SMOTE
 
-                sampler = SMOTE()
+                sampler = SMOTE(random_state=42)
                 X_res, y_res = sampler.fit_resample(
                     self.feats_train, self.df_train[self.target]
                 )
             elif balancing == "adasyn":
                 from imblearn.over_sampling import ADASYN
 
-                sampler = ADASYN()
+                sampler = ADASYN(random_state=42)
                 X_res, y_res = sampler.fit_resample(
                     self.feats_train, self.df_train[self.target]
                 )

{nkululeko-0.88.3 → nkululeko-0.88.5}/nkululeko/models/model_svm.py

@@ -1,5 +1,6 @@
 # model_svm.py
 
+import random
 from sklearn import svm
 from nkululeko.models.model import Model
 
@@ -24,6 +25,7 @@ class SVM_model(Model):
             gamma="scale",
             probability=True,
             class_weight=class_weight,
+            random_state=42,    # for consistent result
         )  # set up the classifier
 
     def set_c(self, c):

{nkululeko-0.88.3 → nkululeko-0.88.5}/nkululeko/reporting/reporter.py

@@ -2,28 +2,32 @@ import ast
 import glob
 import json
 import math
-import os
 
-from confidence_intervals import evaluate_with_conf_int
 import matplotlib.pyplot as plt
 import numpy as np
+
+# from torch import is_tensor
+from audmetric import (
+    accuracy,
+    concordance_cc,
+    mean_absolute_error,
+    mean_squared_error,
+    unweighted_average_recall,
+)
+
+# import os
+from confidence_intervals import evaluate_with_conf_int
 from scipy.special import softmax
-from scipy.stats import entropy
-from scipy.stats import pearsonr
-from sklearn.metrics import ConfusionMatrixDisplay
-from sklearn.metrics import auc
-from sklearn.metrics import classification_report
-from sklearn.metrics import confusion_matrix
-from sklearn.metrics import r2_score
-from sklearn.metrics import roc_auc_score
-from sklearn.metrics import roc_curve
-from torch import is_tensor
-
-from audmetric import accuracy
-from audmetric import concordance_cc
-from audmetric import mean_absolute_error
-from audmetric import mean_squared_error
-from audmetric import unweighted_average_recall
+from scipy.stats import entropy, pearsonr
+from sklearn.metrics import (
+    ConfusionMatrixDisplay,
+    auc,
+    classification_report,
+    confusion_matrix,
+    r2_score,
+    roc_auc_score,
+    roc_curve,
+)
 
 import nkululeko.glob_conf as glob_conf
 from nkululeko.plots import Plots
@@ -166,7 +170,8 @@ class Reporter:
             )
             probas["uncertainty"] = uncertainty
             probas["correct"] = probas.predicted == probas.truth
-            sp = os.path.join(self.util.get_path("store"), "pred_df.csv")
+            sp = self.util.get_pred_name()
+
             self.probas = probas
             probas.to_csv(sp)
             self.util.debug(f"Saved probabilities to {sp}")
@@ -174,7 +179,13 @@ class Reporter:
             ax, caption = plots.plotcatcont(
                 probas, "correct", "uncertainty", "uncertainty", "correct"
             )
-            plots.save_plot(ax, caption, "Uncertainty", "uncertainty", "samples")
+            plots.save_plot(
+                ax,
+                caption,
+                "Uncertainty",
+                "uncertainty_samples",
+                self.util.get_exp_name(),
+            )
 
     def set_id(self, run, epoch):
         """Make the report identifiable with run and epoch index."""

{nkululeko-0.88.3 → nkululeko-0.88.5}/nkululeko/runmanager.py

@@ -50,7 +50,7 @@ class Runmanager:
         self.last_epochs = []  # keep the epoch of best result per run
         # for all runs
         for run in range(int(self.util.config_val("EXP", "runs", 1))):
-            self.util.debug(f"run {run}")
+            self.util.debug(f"run {run} using model {glob_conf.config['MODEL']['type']}")
             # set the run index as global variable for reporting
             self.util.set_config_val("EXP", "run", run)
             self.modelrunner = Modelrunner(