nkululeko 0.88.3__py3-none-any.whl → 0.88.5__py3-none-any.whl

nkululeko/constants.py

@@ -1,2 +1,2 @@
-VERSION="0.88.3"
+VERSION="0.88.5"
 SAMPLING_RATE = 16000

nkululeko/ensemble.py

@@ -1,20 +1,181 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-
+from typing import List
 import configparser
 import time
 from argparse import ArgumentParser
 from pathlib import Path
 
+import numpy as np
 import pandas as pd
+from sklearn.metrics import balanced_accuracy_score
 
 from nkululeko.constants import VERSION
 from nkululeko.experiment import Experiment
 from nkululeko.utils.util import Util
 
+# import torch
+
+# Constants
+DEFAULT_METHOD = "mean"
+DEFAULT_OUTFILE = "ensemble_result.csv"
+
+
+def majority_voting(ensemble_preds_ls):
+    all_predictions = pd.concat([df["predicted"] for df in ensemble_preds_ls], axis=1)
+    return all_predictions.mode(axis=1).iloc[:, 0]
+
+
+def mean_ensemble(ensemble_preds, labels):
+    for label in labels:
+        ensemble_preds[label] = ensemble_preds[label].mean(axis=1)
+    return ensemble_preds[labels].idxmax(axis=1)
+
+
+def max_ensemble(ensemble_preds, labels):
+    for label in labels:
+        ensemble_preds[label] = ensemble_preds[label].max(axis=1)
+    return ensemble_preds[labels].idxmax(axis=1)
+
+
+def sum_ensemble(ensemble_preds, labels):
+    for label in labels:
+        ensemble_preds[label] = ensemble_preds[label].sum(axis=1)
+    return ensemble_preds[labels].idxmax(axis=1)
+
+
+def uncertainty_ensemble(ensemble_preds):
+    """Same as uncertainty_threshold with a threshold of 0.1"""
+    final_predictions = []
+    best_uncertainty = []
+    for _, row in ensemble_preds.iterrows():
+        uncertainties = row[["uncertainty"]].values
+        min_uncertainty_idx = np.argmin(uncertainties)
+        final_predictions.append(row["predicted"].iloc[min_uncertainty_idx])
+        best_uncertainty.append(uncertainties[min_uncertainty_idx])
+
+    return final_predictions, best_uncertainty
+
+
+def max_class_ensemble(ensemble_preds_ls, labels):
+    """Compare the highest probabilites of all models across classes (instead of same class as in max_ensemble) and return the highest probability and the class"""
+    final_preds = []
+    final_probs = []
+
+    for _, row in pd.concat(ensemble_preds_ls, axis=1).iterrows():
+        max_probs = []
+        max_classes = []
+
+        for model_df in ensemble_preds_ls:
+            model_probs = row[labels].astype(float)
+            max_prob = model_probs.max()
+            max_class = model_probs.idxmax()
+
+            max_probs.append(max_prob)
+            max_classes.append(max_class)
+
+        best_model_index = np.argmax(max_probs)
+
+        final_preds.append(max_classes[best_model_index])
+        final_probs.append(max_probs[best_model_index])
+
+    return pd.Series(final_preds), pd.Series(final_probs)
+
+
+def uncertainty_threshold_ensemble(ensemble_preds_ls, labels, threshold):
+    final_predictions = []
+    final_uncertainties = []
+
+    for idx in ensemble_preds_ls[0].index:
+        uncertainties = [df.loc[idx, "uncertainty"] for df in ensemble_preds_ls]
+        min_uncertainty_idx = np.argmin(uncertainties)
+        min_uncertainty = uncertainties[min_uncertainty_idx]
+
+        if min_uncertainty <= threshold:
+            # Use the prediction with low uncertainty
+            final_predictions.append(
+                ensemble_preds_ls[min_uncertainty_idx].loc[idx, "predicted"]
+            )
+            final_uncertainties.append(min_uncertainty)
+        else:  # for uncertainty above threshold
+            # Calculate mean of probabilities same class different model
+            mean_probs = np.mean(
+                [df.loc[idx, labels].values for df in ensemble_preds_ls], axis=0
+            )
+            final_predictions.append(labels[np.argmax(mean_probs)])
+            final_uncertainties.append(np.mean(uncertainties))
+
+    return final_predictions
+
 
-def ensemble_predictions(config_files, method, no_labels):
+def uncertainty_weighted_ensemble(ensemble_preds_ls, labels):
+    """Weighted ensemble based on uncertainty, normalized for each class"""
+    final_predictions = []
+    final_uncertainties = []
+
+    for idx in ensemble_preds_ls[0].index:
+        uncertainties = [df.loc[idx, "uncertainty"] for df in ensemble_preds_ls]
+        # Convert uncertainties to accuracies/confidence
+        accuracies = [1 - uncertainty for uncertainty in uncertainties]
+
+        # Calculate weights (inverse of uncertainties)
+        weights = [
+            1 / uncertainty if uncertainty != 0 else 1e10
+            for uncertainty in uncertainties
+        ]
+
+        # Normalize weights for each class
+        total_weight = sum(weights)
+        normalized_weights = [w / total_weight for w in weights]
+
+        # Calculate weighted probabilities for each class
+        weighted_probs = {label: 0 for label in labels}
+        for df, weight in zip(ensemble_preds_ls, normalized_weights):
+            for label in labels:
+                weighted_probs[label] += df.loc[idx, label] * weight
+
+        # Select the class with the highest weighted probability
+        predicted_class = max(weighted_probs, key=weighted_probs.get)
+        final_predictions.append(predicted_class)
+
+        # Use the lowest accuracy as the final uncertainty
+        final_uncertainties.append(1 - min(accuracies))
+
+    return final_predictions, final_uncertainties
+
+
+def confidence_weighted_ensemble(ensemble_preds_ls, labels):
+    """Weighted ensemble based on confidence, normalized for all samples per model"""
+    final_predictions = []
+    final_confidences = []
+
+    for idx in ensemble_preds_ls[0].index:
+        class_probabilities = {label: 0 for label in labels}
+        total_confidence = 0
+
+        for df in ensemble_preds_ls:
+            row = df.loc[idx]
+            confidence = 1 - row["uncertainty"]  # confidence score
+            total_confidence += confidence
+
+            for label in labels:
+                class_probabilities[label] += row[label] * confidence
+
+        # Normalize probabilities
+        for label in labels:
+            class_probabilities[label] /= total_confidence
+
+        predicted_class = max(class_probabilities, key=class_probabilities.get)
+        final_predictions.append(predicted_class)
+        final_confidences.append(max(class_probabilities.values()))
+
+    return final_predictions, final_confidences
+
+
+def ensemble_predictions(
+    config_files: List[str], method: str, threshold: float, no_labels: bool
+) -> pd.DataFrame:
     """
     Ensemble predictions from multiple experiments.
 
@@ -31,13 +192,12 @@ def ensemble_predictions(config_files, method, no_labels):
         AssertionError: If the number of config files is less than 2 for majority voting.
 
     """
-    ensemble_preds = []
-    # labels = []
+    ensemble_preds_ls = []
     for config_file in config_files:
         if no_labels:
             # for ensembling results from Nkululeko.demo
-            pred = pd.read_csv(config_file)
-            labels = pred.columns[1:-2]
+            preds = pd.read_csv(config_file)
+            labels = preds.columns[1:-2]
         else:
             # for ensembling results from Nkululeko.nkululeko
             config = configparser.ConfigParser()
@@ -55,42 +215,49 @@ def ensemble_predictions(config_files, method, no_labels):
             labels = expr.util.get_labels()
             # load the experiment
             # get CSV files of predictions
-            pred = expr.util.get_pred_name()
-            print(f"Loading predictions from {pred}")
-            preds = pd.read_csv(pred)
+            pred_name = expr.util.get_pred_name()
+            util.debug(f"Loading predictions from {pred_name}")
+            preds = pd.read_csv(pred_name)
 
-        ensemble_preds.append(preds)
+        ensemble_preds_ls.append(preds)
 
     # pd concate
-    ensemble_preds = pd.concat(ensemble_preds, axis=1)
+    ensemble_preds = pd.concat(ensemble_preds_ls, axis=1)
 
     if method == "majority_voting":
-        # majority voting, get mode, works for odd number of models
-        # raise error when number of configs only two:
         assert (
-            len(config_files) > 2
+            len(ensemble_preds_ls) > 2
         ), "Majority voting only works for more than two models"
-        ensemble_preds["predicted"] = ensemble_preds.mode(axis=1)[0]
-
+        ensemble_preds["predicted"] = majority_voting(ensemble_preds_ls)
     elif method == "mean":
-        for label in labels:
-            ensemble_preds[label] = ensemble_preds[label].mean(axis=1)
-
+        ensemble_preds["predicted"] = mean_ensemble(ensemble_preds, labels)
     elif method == "max":
-        for label in labels:
-            ensemble_preds[label] = ensemble_preds[label].max(axis=1)
-            # get max value from all labels to inver that labels
-
+        ensemble_preds["predicted"] = max_ensemble(ensemble_preds, labels)
     elif method == "sum":
-        for label in labels:
-            ensemble_preds[label] = ensemble_preds[label].sum(axis=1)
-
+        ensemble_preds["predicted"] = sum_ensemble(ensemble_preds, labels)
+    elif method == "max_class":
+        ensemble_preds["predicted"], ensemble_preds["max_probability"] = (
+            max_class_ensemble(ensemble_preds_ls, labels)
+        )
+    elif method == "uncertainty_threshold":
+        ensemble_preds["predicted"] = uncertainty_threshold_ensemble(
+            ensemble_preds_ls, labels, threshold
+        )
+    elif method == "uncertainty_weighted":
+        ensemble_preds["predicted"], ensemble_preds["uncertainty"] = (
+            uncertainty_weighted_ensemble(ensemble_preds_ls, labels)
+        )
+    elif method == "confidence_weighted":
+        ensemble_preds["predicted"], ensemble_preds["confidence"] = (
+            confidence_weighted_ensemble(ensemble_preds_ls, labels)
+        )
     else:
         raise ValueError(f"Unknown ensemble method: {method}")
 
-    # get the highest value from all labels to inver that labels
+    # get the highest value from all labels to infer the label
     # replace the old first predicted column
-    ensemble_preds["predicted"] = ensemble_preds[labels].idxmax(axis=1)
+    if method in ["mean", "max", "sum"]:
+        ensemble_preds["predicted"] = ensemble_preds[labels].idxmax(axis=1)
 
     if no_labels:
         return ensemble_preds
@@ -105,14 +272,14 @@ def ensemble_predictions(config_files, method, no_labels):
 
     truth = ensemble_preds["truth"]
     predicted = ensemble_preds["predicted"]
-    uar = (truth == predicted).mean()
-    Util("ensemble").debug(f"UAR: {uar:.3f}")
+    uar = balanced_accuracy_score(truth, predicted)
+    acc = (truth == predicted).mean()
+    Util("ensemble").debug(f"{method}: UAR: {uar:.3f}, ACC: {acc:.3f}")
 
-    # only return until 'predicted' column
     return ensemble_preds
 
 
-def main(src_dir):
+def main(src_dir: Path) -> None:
     parser = ArgumentParser()
     parser.add_argument(
         "configs",
@@ -122,17 +289,33 @@ def main(src_dir):
     )
     parser.add_argument(
         "--method",
-        default="majority_voting",
-        choices=["majority_voting", "mean", "max", "sum"],
-        help="Ensemble method to use (default: majority_voting)",
+        default=DEFAULT_METHOD,
+        choices=[
+            "majority_voting",
+            "mean",
+            "max",
+            "sum",
+            "max_class",
+            # "uncertainty_lowest",
+            # "entropy",
+            "uncertainty_threshold",
+            "uncertainty_weighted",
+            "confidence_weighted",
+        ],
+        help=f"Ensemble method to use (default: {DEFAULT_METHOD})",
+    )
+    # add threshold if method is uncertainty_threshold
+    parser.add_argument(
+        "--threshold",
+        default=1.0,
+        type=float,
+        help="Threshold for uncertainty_threshold method (default: 1.0, i.e. no threshold)",
     )
     parser.add_argument(
         "--outfile",
-        default="ensemble_result.csv",
-        help="Output file path for the ensemble predictions (default: ensemble_predictions.csv)",
+        default=DEFAULT_OUTFILE,
+        help=f"Output file path for the ensemble predictions (default: {DEFAULT_OUTFILE})",
     )
-
-    # add argument if true label is not available
     parser.add_argument(
         "--no_labels",
         action="store_true",
@@ -143,14 +326,16 @@ def main(src_dir):
 
     start = time.time()
 
-    ensemble_preds = ensemble_predictions(args.configs, args.method, args.no_labels)
+    ensemble_preds = ensemble_predictions(
+        args.configs, args.method, args.threshold, args.no_labels
+    )
 
     # save to csv
     ensemble_preds.to_csv(args.outfile, index=False)
-    print(f"Ensemble predictions saved to: {args.outfile}")
-    print(f"Ensemble done, used {time.time()-start:.2f} seconds")
+    Util("ensemble").debug(f"Ensemble predictions saved to: {args.outfile}")
+    Util("ensemble").debug(f"Ensemble done, used {time.time()-start:.2f} seconds")
 
-    print("DONE")
+    Util("ensemble").debug("DONE")
 
 
 if __name__ == "__main__":

nkululeko/feat_extract/feats_agender.py

@@ -1,13 +1,14 @@
 # feats_agender.py
+
 from nkululeko.feat_extract.featureset import Featureset
 import os
-import pandas as pd
+# import pandas as pd
 import audeer
 import nkululeko.glob_conf as glob_conf
 import audonnx
 import numpy as np
 import audinterface
-
+import torch
 
 class AgenderSet(Featureset):
     """
@@ -32,7 +33,8 @@ class AgenderSet(Featureset):
             archive_path = audeer.download_url(
                 model_url, cache_root, verbose=True)
             audeer.extract_archive(archive_path, model_root)
-        device = self.util.config_val("MODEL", "device", "cpu")
+        cuda = "cuda" if torch.cuda.is_available() else "cpu"
+        device = self.util.config_val("MODEL", "device", cuda)
         self.model = audonnx.load(model_root, device=device)
         self.util.debug(f"initialized agender model")
         self.model_loaded = True

nkululeko/feat_extract/feats_ast.py

@@ -100,10 +100,8 @@ class Ast(Featureset):
             embeddings = torch.mean(last_hidden_state, dim=1)
             embeddings = embeddings.cpu().numpy()
 
-            # convert the same from (768,) to (1, 768)
-            # embeddings = embeddings.reshape(1, -1)
-            print(f"hs shape: {embeddings.shape}")
-            
+            # print(f"hs shape: {embeddings.shape}")
+            # hs shape: (1, 768)
         
         except Exception as e:
             self.util.error(f"Error extracting embeddings for file {file}: {str(e)}, fill with")

nkululeko/feat_extract/feats_spkrec.py

@@ -24,7 +24,7 @@ class Spkrec(Featureset):
     def __init__(self, name, data_df, feat_type):
         """Constructor. is_train is needed to distinguish from test/dev sets,
         because they use the codebook from the training"""
-        super().__init__(name, data_df)
+        super().__init__(name, data_df, feat_type)
         # check if device is not set, use cuda if available
         cuda = "cuda" if torch.cuda.is_available() else "cpu"
         self.device = self.util.config_val("MODEL", "device", cuda)

nkululeko/modelrunner.py

@@ -238,21 +238,21 @@ class Modelrunner:
             if balancing == "ros":
                 from imblearn.over_sampling import RandomOverSampler
 
-                sampler = RandomOverSampler()
+                sampler = RandomOverSampler(random_state=42)
                 X_res, y_res = sampler.fit_resample(
                     self.feats_train, self.df_train[self.target]
                 )
             elif balancing == "smote":
                 from imblearn.over_sampling import SMOTE
 
-                sampler = SMOTE()
+                sampler = SMOTE(random_state=42)
                 X_res, y_res = sampler.fit_resample(
                     self.feats_train, self.df_train[self.target]
                 )
             elif balancing == "adasyn":
                 from imblearn.over_sampling import ADASYN
 
-                sampler = ADASYN()
+                sampler = ADASYN(random_state=42)
                 X_res, y_res = sampler.fit_resample(
                     self.feats_train, self.df_train[self.target]
                 )

nkululeko/models/model_svm.py

@@ -1,5 +1,6 @@
 # model_svm.py
 
+import random
 from sklearn import svm
 from nkululeko.models.model import Model
 
@@ -24,6 +25,7 @@ class SVM_model(Model):
             gamma="scale",
             probability=True,
             class_weight=class_weight,
+            random_state=42,    # for consistent result
         )  # set up the classifier
 
     def set_c(self, c):

nkululeko/reporting/reporter.py

@@ -2,28 +2,32 @@ import ast
 import glob
 import json
 import math
-import os
 
-from confidence_intervals import evaluate_with_conf_int
 import matplotlib.pyplot as plt
 import numpy as np
+
+# from torch import is_tensor
+from audmetric import (
+    accuracy,
+    concordance_cc,
+    mean_absolute_error,
+    mean_squared_error,
+    unweighted_average_recall,
+)
+
+# import os
+from confidence_intervals import evaluate_with_conf_int
 from scipy.special import softmax
-from scipy.stats import entropy
-from scipy.stats import pearsonr
-from sklearn.metrics import ConfusionMatrixDisplay
-from sklearn.metrics import auc
-from sklearn.metrics import classification_report
-from sklearn.metrics import confusion_matrix
-from sklearn.metrics import r2_score
-from sklearn.metrics import roc_auc_score
-from sklearn.metrics import roc_curve
-from torch import is_tensor
-
-from audmetric import accuracy
-from audmetric import concordance_cc
-from audmetric import mean_absolute_error
-from audmetric import mean_squared_error
-from audmetric import unweighted_average_recall
+from scipy.stats import entropy, pearsonr
+from sklearn.metrics import (
+    ConfusionMatrixDisplay,
+    auc,
+    classification_report,
+    confusion_matrix,
+    r2_score,
+    roc_auc_score,
+    roc_curve,
+)
 
 import nkululeko.glob_conf as glob_conf
 from nkululeko.plots import Plots
@@ -166,7 +170,8 @@ class Reporter:
             )
             probas["uncertainty"] = uncertainty
             probas["correct"] = probas.predicted == probas.truth
-            sp = os.path.join(self.util.get_path("store"), "pred_df.csv")
+            sp = self.util.get_pred_name()
+
             self.probas = probas
             probas.to_csv(sp)
             self.util.debug(f"Saved probabilities to {sp}")
@@ -174,7 +179,13 @@ class Reporter:
             ax, caption = plots.plotcatcont(
                 probas, "correct", "uncertainty", "uncertainty", "correct"
             )
-            plots.save_plot(ax, caption, "Uncertainty", "uncertainty", "samples")
+            plots.save_plot(
+                ax,
+                caption,
+                "Uncertainty",
+                "uncertainty_samples",
+                self.util.get_exp_name(),
+            )
 
     def set_id(self, run, epoch):
         """Make the report identifiable with run and epoch index."""

nkululeko/runmanager.py

@@ -50,7 +50,7 @@ class Runmanager:
         self.last_epochs = []  # keep the epoch of best result per run
         # for all runs
         for run in range(int(self.util.config_val("EXP", "runs", 1))):
-            self.util.debug(f"run {run}")
+            self.util.debug(f"run {run} using model {glob_conf.config['MODEL']['type']}")
             # set the run index as global variable for reporting
             self.util.set_config_val("EXP", "run", run)
             self.modelrunner = Modelrunner(

nkululeko/utils/util.py

@@ -1,10 +1,13 @@
 # util.py
 import ast
 import configparser
+import logging
 import os.path
 import pickle
 import sys
 
+# from sysconfig import get_config_h_filename
+# from turtle import setup
 import audeer
 import audformat
 import numpy as np
@@ -32,6 +35,7 @@ class Util:
             self.caller = caller
         else:
             self.caller = ""
+        self.config = None
         if has_config:
             try:
                 import nkululeko.glob_conf as glob_conf
@@ -49,6 +53,30 @@ class Util:
                 self.config = None
                 self.got_data_roots = False
 
+        self.setup_logging()
+        # self.logged_configs = set()
+
+    def setup_logging(self):
+        # Setup logging
+        logger = logging.getLogger(__name__)
+        if not logger.hasHandlers():
+            logger.setLevel(logging.DEBUG)  # Set the desired logging level
+
+            # Create a console handler
+            console_handler = logging.StreamHandler()
+
+            # Create a simple formatter that only shows the message
+            class SimpleFormatter(logging.Formatter):
+                def format(self, record):
+                    return record.getMessage()
+
+            # Set the formatter for the console handler
+            console_handler.setFormatter(SimpleFormatter())
+
+            # Add the console handler to the logger
+            logger.addHandler(console_handler)
+        self.logger = logger
+
     def get_path(self, entry):
         """This method allows the user to get the directory path for the given argument."""
         if self.config is None:
@@ -120,6 +148,7 @@ class Util:
 
     def set_config(self, config):
         self.config = config
+        # self.logged_configs.clear()
 
     def get_save_name(self):
         """Return a relative path to a name to save the experiment"""
@@ -128,7 +157,9 @@ class Util:
 
     def get_pred_name(self):
         store = self.get_path("store")
-        return f"{store}/pred_df.csv"
+        target = self.get_target_name()
+        pred_name = self.get_model_description()
+        return f"{store}/pred_{target}_{pred_name}.csv"
 
     def is_categorical(self, pd_series):
         """Check if a dataframe column is categorical"""
@@ -187,9 +218,14 @@ class Util:
         return_string = f"{ds}"
         if not only_data:
             mt = self.get_model_description()
-            return_string = return_string + "_" + mt
+            target = self.get_target_name()
+            return_string = return_string + "_" + target + "_" + mt
         return return_string.replace("__", "_")
 
+    def get_target_name(self):
+        """Get a string as name from all target sets that are used."""
+        return self.config["DATA"]["target"]
+
     def get_model_type(self):
         return self.config["MODEL"]["type"]
 
@@ -233,6 +269,11 @@ class Util:
             return_string += self._get_value_descript(option[0], option[1]).replace(
                 ".", "-"
             )
+            # prevent double underscores
+            return_string = return_string.replace("__", "_")
+            # remove trailing underscores in the end
+            return_string = return_string.strip("_")
+
         return return_string
 
     def get_plot_name(self):
@@ -249,14 +290,14 @@ class Util:
         return False
 
     def error(self, message):
-        print(f"ERROR {self.caller}: {message}")
+        self.logger.error(f"ERROR: {self.caller}: {message}")
         sys.exit()
 
     def warn(self, message):
-        print(f"WARNING {self.caller}: {message}")
+        self.logger.warning(f"WARNING: {self.caller}: {message}")
 
     def debug(self, message):
-        print(f"DEBUG {self.caller}: {message}")
+        self.logger.debug(f"DEBUG: {self.caller}: {message}")
 
     def set_config_val(self, section, key, value):
         try:
@@ -278,9 +319,13 @@ class Util:
             return self.config[section][key]
         except KeyError:
             if default not in self.stopvals:
-                self.debug(f"value for {key} not found, using default: {default}")
+                self.debug(f"value for {key} is not found, using default: {default}")
             return default
 
+    @classmethod
+    def reset_logged_configs(cls):
+        cls.logged_configs.clear()
+
     def config_val_list(self, section, key, default):
         try:
             return ast.literal_eval(self.config[section][key])

{nkululeko-0.88.3.dist-info → nkululeko-0.88.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.88.3
+Version: 0.88.5
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -68,7 +68,8 @@ A project to detect speaker characteristics by machine learning experiments with
 
 The idea is to have a framework (based on e.g. sklearn and torch) that can be used to rapidly and automatically analyse audio data and explore machine learning models based on that data.
 
-* NEW with nkululeko: [Finetune transformer-models](http://blog.syntheticspeech.de/2024/05/29/nkululeko-how-to-finetune-a-transformer-model/)
+* NEW with nkululek: [Ensemble learning](http://blog.syntheticspeech.de/2024/06/25/nkululeko-ensemble-classifiers-with-late-fusion/)
+* NEW: [Finetune transformer-models](http://blog.syntheticspeech.de/2024/05/29/nkululeko-how-to-finetune-a-transformer-model/)
 * The latest features can be seen in [the ini-file](./ini_file.md) options that are used to control Nkululeko
 * Below is a [Hello World example](#helloworld) that should set you up fastly, also on [Google Colab](https://colab.research.google.com/drive/1GYNBd5cdZQ1QC3Jm58qoeMaJg3UuPhjw?usp=sharing#scrollTo=4G_SjuF9xeQf), and [with Kaggle](https://www.kaggle.com/felixburk/nkululeko-hello-world-example)
 * [Here's a blog post on how to set up nkululeko on your computer.](http://blog.syntheticspeech.de/2021/08/30/how-to-set-up-your-first-nkululeko-project/)
@@ -203,7 +204,8 @@ All of them take *--config <my_config.ini>* as an argument.
 * **nkululeko.nkululeko**: do machine learning experiments combining features and learners
 * **nkululeko.ensemble**: [combine several nkululeko experiments](http://blog.syntheticspeech.de/2024/06/25/nkululeko-ensemble-classifiers-with-late-fusion/) and report on late fusion results
   * *configurations*: which experiments to combine
-  * *--method* (optional): majority_voting, mean, max, sum
+  * *--method* (optional): majority_voting, mean, max, sum, max_class, uncertainty_threshold, uncertainty_weighted, confidence_weighted
+  * *--threshold*: uncertainty threshold (1.0 means no threshold)
   * *--outfile* (optional): name of CSV file for output
   * *--no_labels* (optional): indicate that no ground truth is given 
 * **nkululeko.multidb**: do [multiple experiments](http://blog.syntheticspeech.de/2024/01/02/nkululeko-compare-several-databases/), comparing several databases cross and in itself
@@ -273,6 +275,8 @@ There's my [blog](http://blog.syntheticspeech.de/?s=nkululeko) with tutorials:
 * [Tweak the target variable for database comparison](http://blog.syntheticspeech.de/2024/03/13/nkululeko-how-to-tweak-the-target-variable-for-database-comparison/)
 * [How to run multiple experiments in one go](http://blog.syntheticspeech.de/2022/03/28/how-to-run-multiple-experiments-in-one-go-with-nkululeko/)
 * [How to finetune a transformer-model](http://blog.syntheticspeech.de/2024/05/29/nkululeko-how-to-finetune-a-transformer-model/)
+* [Ensemble (combine) classifiers with late-fusion](http://blog.syntheticspeech.de/2024/06/25/nkululeko-ensemble-classifiers-with-late-fusion/)
+
 
 ### <a name="helloworld">Hello World example</a>
 * NEW: [Here's a Google colab that runs this example out-of-the-box](https://colab.research.google.com/drive/1GYNBd5cdZQ1QC3Jm58qoeMaJg3UuPhjw?usp=sharing#scrollTo=4G_SjuF9xeQf), and here is the same [with Kaggle](https://www.kaggle.com/felixburk/nkululeko-hello-world-example)
@@ -356,6 +360,16 @@ F. Burkhardt, Johannes Wagner, Hagen Wierstorf, Florian Eyben and Björn Schulle
 Changelog
 =========
 
+Version 0.88.5
+--------------
+* add a unique name to the uncertainty plot
+* fix error in speaker embedding (still need speechbrain < 1.0)
+* add get_target_name function in util
+
+Version 0.88.4
+--------------
+* added more ensemble methods, e.g. based on uncertainty
+
 Version 0.88.3
 --------------
 * fixed bug in false uncertainty estimation

{nkululeko-0.88.3.dist-info → nkululeko-0.88.5.dist-info}/RECORD

@@ -2,11 +2,11 @@ nkululeko/__init__.py,sha256=62f8HiEzJ8rG2QlTFJXUCMpvuH3fKI33DoJSj33mscc,63
 nkululeko/aug_train.py,sha256=YhuZnS_WVWnun9G-M6g5n6rbRxoVREz6Zh7k6qprFNQ,3194
 nkululeko/augment.py,sha256=4MG0apTAG5RgkuJrYEjGgDdbodZWi_HweSPNI1JJ5QA,3051
 nkululeko/cacheddataset.py,sha256=lIJ6hUo5LoxSrzXtWV8mzwO7wRtUETWnOQ4ws2XfL1E,969
-nkululeko/constants.py,sha256=nYBLnpl1NpTeGp7KjRHJkxz_Vju3pYEkcrj3CwBX0zI,39
+nkululeko/constants.py,sha256=ir__UePsDhzi0X5DuhvHtXtpIXtZaMIpjU97A7Ljv54,39
 nkululeko/demo.py,sha256=bLuHkeEl5rOfm7ecGHCcWATiPK7-njNbtrGljxzNzFs,5088
 nkululeko/demo_feats.py,sha256=sAeGFojhEj9WEDFtG3SzPBmyYJWLF2rkbpp65m8Ujo4,2025
 nkululeko/demo_predictor.py,sha256=zs1bjhpnKuNCPLJeiyDm19ME1NEDOQT3QNeyVKJq9Yc,4882
-nkululeko/ensemble.py,sha256=huRbXUuabm6QYxGBHjkwEU95e-0qxtO0Z6UdXFgtaMY,4947
+nkululeko/ensemble.py,sha256=rUHg8YmD6L8Ktt2T5M6iwsWVWbpCnfiynhHdN22bLRQ,11873
 nkululeko/experiment.py,sha256=wXZnb_cfOqF8b0Zqzu2bbrEgCCpG_zPkDbD-Usw5sRs,31283
 nkululeko/explore.py,sha256=lDzRoW_Taa5u4BBABZLD89BcQWnYlrftJR4jgt1yyj0,2609
 nkululeko/export.py,sha256=mHeEAAmtZuxdyebLlbSzPrHSi9OMgJHbk35d3DTxRBc,4632
@@ -14,14 +14,14 @@ nkululeko/feature_extractor.py,sha256=UnspIWz3XrNhKnBBhWZkH2bHvD-sROtrQVqB1JvkUy
 nkululeko/file_checker.py,sha256=LoLnL8aHpW-axMQ46qbqrManTs5otG9ShpEZuz9iRSk,3474
 nkululeko/filter_data.py,sha256=w-X2mhKdYr5DxDIz50E5yzO6Jmzk4jjDBoXsgOOVtcA,7222
 nkululeko/glob_conf.py,sha256=KL9YJQTHvTztxo1vr25qRRgaPnx4NTg0XrdbovKGMmw,525
-nkululeko/modelrunner.py,sha256=rpWQRXERiDZ-i_7CwsqynI87vawtsaPihsonDMPe9PU,11151
+nkululeko/modelrunner.py,sha256=cKYD9a7MRoBxfqUy3X8kf6rGTYho-33In8I9YkzMOo8,11196
 nkululeko/multidb.py,sha256=fG3VukEWP1vreVN4gB1IRXxwwg4jLftsSEYtu0o1f78,5634
 nkululeko/nkuluflag.py,sha256=PGWSmZz-PiiHLgcZJAoGOI_Y-sZDVI1ksB8p5r7riWM,3725
 nkululeko/nkululeko.py,sha256=Kn3s2E3yyH8cJ7z6lkMxrnqtCxTu7-qfe9Zr_ONTD5g,1968
 nkululeko/plots.py,sha256=WsI_dtPKfrYPsKymHRmIhqj33aZzTcE8fF_EwLkm_5A,22899
 nkululeko/predict.py,sha256=sF091sSSLnEWcISx9ZcULLie3tY5XeFsQJd6b3vrxFg,2409
 nkululeko/resample.py,sha256=2d9eao_0sLrGZ_KSl8OVKsPor3BkFrlmMhrpB9WelIs,4267
-nkululeko/runmanager.py,sha256=eRMJidkoJhkU5NdIKoozv3vovU-8tqfn-7zqr2JZcnE,7533
+nkululeko/runmanager.py,sha256=xvxL5a9d3jtGFqx0Z3nyyxowA368uNyP0ZitO8kxIIE,7581
 nkululeko/scaler.py,sha256=4nkIqoajkIkuTPK0Z02ifMN_awl6fP_i-GBYdoGYgGM,4101
 nkululeko/segment.py,sha256=YLKckX44tbvTb3LrdgYw9X4guzuF27sutl92z9DkpZU,4835
 nkululeko/syllable_nuclei.py,sha256=Sky-C__MeUDaxqHnDl2TGLLYOYvsahD35TUjWGeG31k,10047
@@ -49,10 +49,10 @@ nkululeko/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/data/dataset.py,sha256=hUD0NqWCfRaSHG8JNs1MsPb0zjUZAf8FJkg_c0ebq0Q,28046
 nkululeko/data/dataset_csv.py,sha256=UGEpi__eT2KFS6Fop6N4HkMrzO-u5VP71gt44kwZavo,4588
 nkululeko/feat_extract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nkululeko/feat_extract/feats_agender.py,sha256=Qm69G4kqAyTVVk7wwRgrXlNwGaDMGRYyKGpuf0vOEgM,3113
+nkululeko/feat_extract/feats_agender.py,sha256=sHyvxxlWXv1QGYXHGHIYEQK7X39eifSVie0tu-zBG3M,3189
 nkululeko/feat_extract/feats_agender_agender.py,sha256=tgH2BnwcxpvuLmOkrMbVdBSX0Onfz2MG12FsddalRKI,3424
 nkululeko/feat_extract/feats_analyser.py,sha256=Y9hMpZ9WsQOrxTP3B1diHnzMeOgwbVpVFWVlIyhHMJs,12722
-nkululeko/feat_extract/feats_ast.py,sha256=LolDaRTfNB9L8-CUqz9tOfkXntL8c9GJs4kqMmg5BSo,4724
+nkululeko/feat_extract/feats_ast.py,sha256=ycJn5eSVOxcEpmeHVk0FPB8q5XiTC8VSKz61L9n0Wa4,4638
 nkululeko/feat_extract/feats_auddim.py,sha256=VlzKKXTXa5kjLgQBWyEFy-daIyU1SkOwCCOIhKsWCvE,3162
 nkululeko/feat_extract/feats_audmodel.py,sha256=VjBNgAoxsHJhwr6Kwt9CxX6SaCM4RK_OV-GU2W5-bhU,3187
 nkululeko/feat_extract/feats_clap.py,sha256=nR6eEIRdsMHcfmD1bNtt5WfDvkxKjvEbukSSrXHm-HU,3489
@@ -65,7 +65,7 @@ nkululeko/feat_extract/feats_oxbow.py,sha256=CmIG9cbHTJTJVnzgCPdQpYpnlewWExpsr5Z
 nkululeko/feat_extract/feats_praat.py,sha256=kZrS6srzH7WoWEd2prp1Dxw6g9JklFQGTNq5zzPpHzg,3105
 nkululeko/feat_extract/feats_snr.py,sha256=9dqZ-4RpK98iJEssM3ttozNd18LWlZYM_QVXvp5xDcs,2829
 nkululeko/feat_extract/feats_spectra.py,sha256=5Pex8awIQC3cjQRHSu4NQFmg4quamG0RL3V3Yd0pJHs,3670
-nkululeko/feat_extract/feats_spkrec.py,sha256=VK4ma3uWzM0YZStsgRTirfkbzjWIfRWSgsYI038QlRY,4803
+nkululeko/feat_extract/feats_spkrec.py,sha256=FugR-X2lDFKLLRRhKnhUYJhz-VIktIj8iVEDLbwNwtw,4814
 nkululeko/feat_extract/feats_squim.py,sha256=Y31YmDmscuG0YozvxyBZIutO3id8t7IZJWCfKucw-6M,4617
 nkululeko/feat_extract/feats_trill.py,sha256=K2ahhdpwpjgg3WZS1POg3UMP2U44i8cLZZvn5Rq7fUI,3228
 nkululeko/feat_extract/feats_wav2vec2.py,sha256=XyxD4NcrF4VFWSeHkXCKWdEOdr8VMzgVUz8N4mwhdyo,5248
@@ -86,7 +86,7 @@ nkululeko/models/model_knn_reg.py,sha256=j7YFfVm6xOR2d9yBYdQiwwqYfqkX0JynX_qLCvk
 nkululeko/models/model_lin_reg.py,sha256=0D7mSnSwK82lNWDMwHYRyq3FmGa6y-DHDGg4qUe85q4,422
 nkululeko/models/model_mlp.py,sha256=VE0CI19qMyRbI-THDkMeJ7JbWf4z7CmZ4MMs1FIQgtM,10557
 nkululeko/models/model_mlp_regression.py,sha256=7oK2zQhhCegSqiBUe6eU7Av8MJ_DPLA9skixJcHaVfg,10232
-nkululeko/models/model_svm.py,sha256=rsME3KvKvNG7bdE5lbvYUu85WZhaASZxxmdNDIVJRZ4,940
+nkululeko/models/model_svm.py,sha256=AzWksBRbIdpUuMbDnAh_YAXebewR5POj9AkB9VC40pI,1010
 nkululeko/models/model_svr.py,sha256=_YZeksqB3eBENGlg3g9RwYFlk9rQQ-XCeNBKLlGGVoE,725
 nkululeko/models/model_tree.py,sha256=rf16faUm4o2LJgkoYpeY998b8DQIvXZ73_m1IS3TnnE,417
 nkululeko/models/model_tree_reg.py,sha256=IgQcPTE-304HQLYSKPF8Z4ot_Ur9dH01fZjS0nXke_M,428
@@ -98,7 +98,7 @@ nkululeko/reporting/defines.py,sha256=IsY1YgKRMaABpylVKjBJgJ5bNCEbGCVA_E6pivraqS
 nkululeko/reporting/latex_writer.py,sha256=qiCRSmB4KOD_za4oHu5x-PhwjZohzfo8wecMOwlXZwc,1886
 nkululeko/reporting/report.py,sha256=W0rcigDdjBvxZQ3pZja_gvToILYvaZ1BFtnN2qFRfYI,1060
 nkululeko/reporting/report_item.py,sha256=siWeGNgo4bAE46YBMNcsdf3jTMTy76BO9Fi6DTvDig4,533
-nkululeko/reporting/reporter.py,sha256=WPevPmtpnzSwiF6lxtczmxpFjmzhNABJspAsNNJgmq4,19237
+nkululeko/reporting/reporter.py,sha256=Gt8tEiDQ9rbsYAKb-EbKGVaogPWKKoAEkwtVeiFoTSA,19119
 nkululeko/reporting/result.py,sha256=G63a2tHCwHhM6NBJgYzsWKWJm4Yu3r4hsCHA2Km7eHU,1073
 nkululeko/segmenting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/segmenting/seg_inaspeechsegmenter.py,sha256=pmLHuXsaqvcdYxB4PSW9l1mbQWZZBJFhi_CGabqydas,1947
@@ -106,9 +106,9 @@ nkululeko/segmenting/seg_silero.py,sha256=lLytS38KzARS17omwv8VBw-zz60RVSXGSvZ5Ev
 nkululeko/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/utils/files.py,sha256=UiGAtZRWYjHSvlmPaTMtzyNNGE6qaLaxQkybctS7iRM,4021
 nkululeko/utils/stats.py,sha256=eC9dMO-by6CDnGLHDBQu-2B4-BudZNJ0nnWGhKYdUMA,2968
-nkululeko/utils/util.py,sha256=BNd9JpoVakPbyysKBsJSCnqlbPlUKHUrcWYcwEnOdVA,15128
-nkululeko-0.88.3.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
-nkululeko-0.88.3.dist-info/METADATA,sha256=8sPk6npYa7my3UA88uk2373yzdfiRFfb70Zix097KR4,39224
-nkululeko-0.88.3.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
-nkululeko-0.88.3.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
-nkululeko-0.88.3.dist-info/RECORD,,
+nkululeko/utils/util.py,sha256=y-pdrjovT8yGtBTJ3ifIpTcF0fPnoz8UKbuLIZ0efpc,16768
+nkululeko-0.88.5.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
+nkululeko-0.88.5.dist-info/METADATA,sha256=6-AE25u5U0-7CldZfExbSfjBu3_Ey3K7kMEkaNBysTc,39896
+nkululeko-0.88.5.dist-info/WHEEL,sha256=FZ75kcLy9M91ncbIgG8dnpCncbiKXSRGJ_PFILs6SFg,91
+nkululeko-0.88.5.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
+nkululeko-0.88.5.dist-info/RECORD,,

{nkululeko-0.88.3.dist-info → nkululeko-0.88.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.1)
+Generator: setuptools (71.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any