nkululeko 0.88.2__tar.gz → 0.88.4__tar.gz

{nkululeko-0.88.2 → nkululeko-0.88.4}/CHANGELOG.md

@@ -1,6 +1,15 @@
 Changelog
 =========
 
+Version 0.88.4
+--------------
+* added more ensemble methods, e.g. based on uncertainty
+
+Version 0.88.3
+--------------
+* fixed bug in false uncertainty estimation
+* changed demo live recording
+
 Version 0.88.2
 --------------
 * changed combine speaker results to show speakers not samples

{nkululeko-0.88.2/nkululeko.egg-info → nkululeko-0.88.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.88.2
+Version: 0.88.4
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -203,7 +203,8 @@ All of them take *--config <my_config.ini>* as an argument.
 * **nkululeko.nkululeko**: do machine learning experiments combining features and learners
 * **nkululeko.ensemble**: [combine several nkululeko experiments](http://blog.syntheticspeech.de/2024/06/25/nkululeko-ensemble-classifiers-with-late-fusion/) and report on late fusion results
   * *configurations*: which experiments to combine
-  * *--method* (optional): majority_voting, mean, max, sum
+  * *--method* (optional): mean, max, sum, max_class, uncertainty_threshold, uncertainty_weighted, confidence_weighted
+  * *--threshold*: uncertainty threshold (1.0 means no threshold)
   * *--outfile* (optional): name of CSV file for output
   * *--no_labels* (optional): indicate that no ground truth is given 
 * **nkululeko.multidb**: do [multiple experiments](http://blog.syntheticspeech.de/2024/01/02/nkululeko-compare-several-databases/), comparing several databases cross and in itself
@@ -273,6 +274,8 @@ There's my [blog](http://blog.syntheticspeech.de/?s=nkululeko) with tutorials:
 * [Tweak the target variable for database comparison](http://blog.syntheticspeech.de/2024/03/13/nkululeko-how-to-tweak-the-target-variable-for-database-comparison/)
 * [How to run multiple experiments in one go](http://blog.syntheticspeech.de/2022/03/28/how-to-run-multiple-experiments-in-one-go-with-nkululeko/)
 * [How to finetune a transformer-model](http://blog.syntheticspeech.de/2024/05/29/nkululeko-how-to-finetune-a-transformer-model/)
+* [Ensemble (combine) classifiers with late-fusion](http://blog.syntheticspeech.de/2024/06/25/nkululeko-ensemble-classifiers-with-late-fusion/)
+
 
 ### <a name="helloworld">Hello World example</a>
 * NEW: [Here's a Google colab that runs this example out-of-the-box](https://colab.research.google.com/drive/1GYNBd5cdZQ1QC3Jm58qoeMaJg3UuPhjw?usp=sharing#scrollTo=4G_SjuF9xeQf), and here is the same [with Kaggle](https://www.kaggle.com/felixburk/nkululeko-hello-world-example)
@@ -356,6 +359,15 @@ F. Burkhardt, Johannes Wagner, Hagen Wierstorf, Florian Eyben and Björn Schulle
 Changelog
 =========
 
+Version 0.88.4
+--------------
+* added more ensemble methods, e.g. based on uncertainty
+
+Version 0.88.3
+--------------
+* fixed bug in false uncertainty estimation
+* changed demo live recording
+
 Version 0.88.2
 --------------
 * changed combine speaker results to show speakers not samples

{nkululeko-0.88.2 → nkululeko-0.88.4}/README.md

@@ -159,7 +159,8 @@ All of them take *--config <my_config.ini>* as an argument.
 * **nkululeko.nkululeko**: do machine learning experiments combining features and learners
 * **nkululeko.ensemble**: [combine several nkululeko experiments](http://blog.syntheticspeech.de/2024/06/25/nkululeko-ensemble-classifiers-with-late-fusion/) and report on late fusion results
   * *configurations*: which experiments to combine
-  * *--method* (optional): majority_voting, mean, max, sum
+  * *--method* (optional): mean, max, sum, max_class, uncertainty_threshold, uncertainty_weighted, confidence_weighted
+  * *--threshold*: uncertainty threshold (1.0 means no threshold)
   * *--outfile* (optional): name of CSV file for output
   * *--no_labels* (optional): indicate that no ground truth is given 
 * **nkululeko.multidb**: do [multiple experiments](http://blog.syntheticspeech.de/2024/01/02/nkululeko-compare-several-databases/), comparing several databases cross and in itself
@@ -229,6 +230,8 @@ There's my [blog](http://blog.syntheticspeech.de/?s=nkululeko) with tutorials:
 * [Tweak the target variable for database comparison](http://blog.syntheticspeech.de/2024/03/13/nkululeko-how-to-tweak-the-target-variable-for-database-comparison/)
 * [How to run multiple experiments in one go](http://blog.syntheticspeech.de/2022/03/28/how-to-run-multiple-experiments-in-one-go-with-nkululeko/)
 * [How to finetune a transformer-model](http://blog.syntheticspeech.de/2024/05/29/nkululeko-how-to-finetune-a-transformer-model/)
+* [Ensemble (combine) classifiers with late-fusion](http://blog.syntheticspeech.de/2024/06/25/nkululeko-ensemble-classifiers-with-late-fusion/)
+
 
 ### <a name="helloworld">Hello World example</a>
 * NEW: [Here's a Google colab that runs this example out-of-the-box](https://colab.research.google.com/drive/1GYNBd5cdZQ1QC3Jm58qoeMaJg3UuPhjw?usp=sharing#scrollTo=4G_SjuF9xeQf), and here is the same [with Kaggle](https://www.kaggle.com/felixburk/nkululeko-hello-world-example)

{nkululeko-0.88.2 → nkululeko-0.88.4}/nkululeko/constants.py

@@ -1,2 +1,2 @@
-VERSION="0.88.2"
+VERSION="0.88.4"
 SAMPLING_RATE = 16000

{nkululeko-0.88.2 → nkululeko-0.88.4}/nkululeko/demo_predictor.py

@@ -72,9 +72,11 @@ class Demo_predictor:
                 else:
                     self.util.debug(df_res)
         else:
-            while True:
+            answer = input("want to record y/n?")
+            while answer == "y":
                 signal = self.record_audio(3)
                 self.predict_signal(signal, self.sr)
+                answer = input("want to record y/n?")
 
     #            self.play_audio(signal)
 
@@ -109,7 +111,7 @@ class Demo_predictor:
     def record_audio(self, seconds):
         import sounddevice as sd
 
-        print("recording ...")
+        print("recording ...", flush=True)
         y = sd.rec(int(seconds * self.sr), samplerate=self.sr, channels=1)
         sd.wait()
         y = y.T

nkululeko-0.88.4/nkululeko/ensemble.py

@@ -0,0 +1,343 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from typing import List
+import configparser
+import time
+from argparse import ArgumentParser
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import balanced_accuracy_score
+
+from nkululeko.constants import VERSION
+from nkululeko.experiment import Experiment
+from nkululeko.utils.util import Util
+
+# import torch
+
+# Constants
+DEFAULT_METHOD = "mean"
+DEFAULT_OUTFILE = "ensemble_result.csv"
+
+
+def majority_voting(ensemble_preds_ls):
+    all_predictions = pd.concat([df["predicted"] for df in ensemble_preds_ls], axis=1)
+    return all_predictions.mode(axis=1).iloc[:, 0]
+
+
+def mean_ensemble(ensemble_preds, labels):
+    for label in labels:
+        ensemble_preds[label] = ensemble_preds[label].mean(axis=1)
+    return ensemble_preds[labels].idxmax(axis=1)
+
+
+def max_ensemble(ensemble_preds, labels):
+    for label in labels:
+        ensemble_preds[label] = ensemble_preds[label].max(axis=1)
+    return ensemble_preds[labels].idxmax(axis=1)
+
+
+def sum_ensemble(ensemble_preds, labels):
+    for label in labels:
+        ensemble_preds[label] = ensemble_preds[label].sum(axis=1)
+    return ensemble_preds[labels].idxmax(axis=1)
+
+
+def uncertainty_ensemble(ensemble_preds):
+    """Same as uncertainty_threshold with a threshold of 0.1"""
+    final_predictions = []
+    best_uncertainty = []
+    for _, row in ensemble_preds.iterrows():
+        uncertainties = row[["uncertainty"]].values
+        min_uncertainty_idx = np.argmin(uncertainties)
+        final_predictions.append(row["predicted"].iloc[min_uncertainty_idx])
+        best_uncertainty.append(uncertainties[min_uncertainty_idx])
+
+    return final_predictions, best_uncertainty
+
+
+def max_class_ensemble(ensemble_preds_ls, labels):
+    """Compare the highest probabilites of all models across classes (instead of same class as in max_ensemble) and return the highest probability and the class"""
+    final_preds = []
+    final_probs = []
+
+    for _, row in pd.concat(ensemble_preds_ls, axis=1).iterrows():
+        max_probs = []
+        max_classes = []
+
+        for model_df in ensemble_preds_ls:
+            model_probs = row[labels].astype(float)
+            max_prob = model_probs.max()
+            max_class = model_probs.idxmax()
+
+            max_probs.append(max_prob)
+            max_classes.append(max_class)
+
+        best_model_index = np.argmax(max_probs)
+
+        final_preds.append(max_classes[best_model_index])
+        final_probs.append(max_probs[best_model_index])
+
+    return pd.Series(final_preds), pd.Series(final_probs)
+
+
+def uncertainty_threshold_ensemble(ensemble_preds_ls, labels, threshold):
+    final_predictions = []
+    final_uncertainties = []
+
+    for idx in ensemble_preds_ls[0].index:
+        uncertainties = [df.loc[idx, "uncertainty"] for df in ensemble_preds_ls]
+        min_uncertainty_idx = np.argmin(uncertainties)
+        min_uncertainty = uncertainties[min_uncertainty_idx]
+
+        if min_uncertainty <= threshold:
+            # Use the prediction with low uncertainty
+            final_predictions.append(
+                ensemble_preds_ls[min_uncertainty_idx].loc[idx, "predicted"]
+            )
+            final_uncertainties.append(min_uncertainty)
+        else:  # for uncertainty above threshold
+            # Calculate mean of probabilities same class different model
+            mean_probs = np.mean(
+                [df.loc[idx, labels].values for df in ensemble_preds_ls], axis=0
+            )
+            final_predictions.append(labels[np.argmax(mean_probs)])
+            final_uncertainties.append(np.mean(uncertainties))
+
+    return final_predictions
+
+
+def uncertainty_weighted_ensemble(ensemble_preds_ls, labels):
+    """Weighted ensemble based on uncertainty, normalized for each class"""
+    final_predictions = []
+    final_uncertainties = []
+
+    for idx in ensemble_preds_ls[0].index:
+        uncertainties = [df.loc[idx, "uncertainty"] for df in ensemble_preds_ls]
+        # Convert uncertainties to accuracies/confidence
+        accuracies = [1 - uncertainty for uncertainty in uncertainties]
+
+        # Calculate weights (inverse of uncertainties)
+        weights = [
+            1 / uncertainty if uncertainty != 0 else 1e10
+            for uncertainty in uncertainties
+        ]
+
+        # Normalize weights for each class
+        total_weight = sum(weights)
+        normalized_weights = [w / total_weight for w in weights]
+
+        # Calculate weighted probabilities for each class
+        weighted_probs = {label: 0 for label in labels}
+        for df, weight in zip(ensemble_preds_ls, normalized_weights):
+            for label in labels:
+                weighted_probs[label] += df.loc[idx, label] * weight
+
+        # Select the class with the highest weighted probability
+        predicted_class = max(weighted_probs, key=weighted_probs.get)
+        final_predictions.append(predicted_class)
+
+        # Use the lowest accuracy as the final uncertainty
+        final_uncertainties.append(1 - min(accuracies))
+
+    return final_predictions, final_uncertainties
+
+
+def confidence_weighted_ensemble(ensemble_preds_ls, labels):
+    """Weighted ensemble based on confidence, normalized for all samples per model"""
+    final_predictions = []
+    final_confidences = []
+
+    for idx in ensemble_preds_ls[0].index:
+        class_probabilities = {label: 0 for label in labels}
+        total_confidence = 0
+
+        for df in ensemble_preds_ls:
+            row = df.loc[idx]
+            confidence = 1 - row["uncertainty"]  # confidence score
+            total_confidence += confidence
+
+            for label in labels:
+                class_probabilities[label] += row[label] * confidence
+
+        # Normalize probabilities
+        for label in labels:
+            class_probabilities[label] /= total_confidence
+
+        predicted_class = max(class_probabilities, key=class_probabilities.get)
+        final_predictions.append(predicted_class)
+        final_confidences.append(max(class_probabilities.values()))
+
+    return final_predictions, final_confidences
+
+
+def ensemble_predictions(
+    config_files: List[str], method: str, threshold: float, no_labels: bool
+) -> pd.DataFrame:
+    """
+    Ensemble predictions from multiple experiments.
+
+    Args:
+        config_files (list): List of configuration file paths.
+        method (str): Ensemble method to use. Options are 'majority_voting', 'mean', 'max', or 'sum'.
+        no_labels (bool): Flag indicating whether the predictions have labels or not.
+
+    Returns:
+        pandas.DataFrame: The ensemble predictions.
+
+    Raises:
+        ValueError: If an unknown ensemble method is provided.
+        AssertionError: If the number of config files is less than 2 for majority voting.
+
+    """
+    ensemble_preds_ls = []
+    for config_file in config_files:
+        if no_labels:
+            # for ensembling results from Nkululeko.demo
+            preds = pd.read_csv(config_file)
+            labels = preds.columns[1:-2]
+        else:
+            # for ensembling results from Nkululeko.nkululeko
+            config = configparser.ConfigParser()
+            config.read(config_file)
+            expr = Experiment(config)
+            module = "ensemble"
+            expr.set_module(module)
+            util = Util(module, has_config=True)
+            util.debug(
+                f"running {expr.name} from config {config_file}, nkululeko version"
+                f" {VERSION}"
+            )
+
+            # get labels
+            labels = expr.util.get_labels()
+            # load the experiment
+            # get CSV files of predictions
+            pred_name = expr.util.get_pred_name()
+            util.debug(f"Loading predictions from {pred_name}")
+            preds = pd.read_csv(pred_name)
+
+        ensemble_preds_ls.append(preds)
+
+    # pd concate
+    ensemble_preds = pd.concat(ensemble_preds_ls, axis=1)
+
+    if method == "majority_voting":
+        assert (
+            len(ensemble_preds_ls) > 2
+        ), "Majority voting only works for more than two models"
+        ensemble_preds["predicted"] = majority_voting(ensemble_preds_ls)
+    elif method == "mean":
+        ensemble_preds["predicted"] = mean_ensemble(ensemble_preds, labels)
+    elif method == "max":
+        ensemble_preds["predicted"] = max_ensemble(ensemble_preds, labels)
+    elif method == "sum":
+        ensemble_preds["predicted"] = sum_ensemble(ensemble_preds, labels)
+    elif method == "max_class":
+        ensemble_preds["predicted"], ensemble_preds["max_probability"] = (
+            max_class_ensemble(ensemble_preds_ls, labels)
+        )
+    elif method == "uncertainty_threshold":
+        ensemble_preds["predicted"] = uncertainty_threshold_ensemble(
+            ensemble_preds_ls, labels, threshold
+        )
+    elif method == "uncertainty_weighted":
+        ensemble_preds["predicted"], ensemble_preds["uncertainty"] = (
+            uncertainty_weighted_ensemble(ensemble_preds_ls, labels)
+        )
+    elif method == "confidence_weighted":
+        ensemble_preds["predicted"], ensemble_preds["confidence"] = (
+            confidence_weighted_ensemble(ensemble_preds_ls, labels)
+        )
+    else:
+        raise ValueError(f"Unknown ensemble method: {method}")
+
+    # get the highest value from all labels to infer the label
+    # replace the old first predicted column
+    if method in ["mean", "max", "sum"]:
+        ensemble_preds["predicted"] = ensemble_preds[labels].idxmax(axis=1)
+
+    if no_labels:
+        return ensemble_preds
+
+    # Drop start, end columns
+    ensemble_preds = ensemble_preds.drop(columns=["start", "end"])
+
+    # Drop other column except until truth
+    ensemble_preds = ensemble_preds.iloc[:, : len(labels) + 3]
+
+    # calculate UAR from predicted and truth columns
+
+    truth = ensemble_preds["truth"]
+    predicted = ensemble_preds["predicted"]
+    uar = balanced_accuracy_score(truth, predicted)
+    acc = (truth == predicted).mean()
+    Util("ensemble").debug(f"{method}: UAR: {uar:.3f}, ACC: {acc:.3f}")
+
+    return ensemble_preds
+
+
+def main(src_dir: Path) -> None:
+    parser = ArgumentParser()
+    parser.add_argument(
+        "configs",
+        nargs="+",
+        help="Paths to the configuration files of the experiments to ensemble. \
+             Can be INI files for Nkululeko.nkululeo or CSV files from Nkululeko.demo.",
+    )
+    parser.add_argument(
+        "--method",
+        default=DEFAULT_METHOD,
+        choices=[
+            "majority_voting",
+            "mean",
+            "max",
+            "sum",
+            "max_class",
+            # "uncertainty_lowest",
+            # "entropy",
+            "uncertainty_threshold",
+            "uncertainty_weighted",
+            "confidence_weighted",
+        ],
+        help=f"Ensemble method to use (default: {DEFAULT_METHOD})",
+    )
+    # add threshold if method is uncertainty_threshold
+    parser.add_argument(
+        "--threshold",
+        default=1.0,
+        type=float,
+        help="Threshold for uncertainty_threshold method (default: 1.0, i.e. no threshold)",
+    )
+    parser.add_argument(
+        "--outfile",
+        default=DEFAULT_OUTFILE,
+        help=f"Output file path for the ensemble predictions (default: {DEFAULT_OUTFILE})",
+    )
+    parser.add_argument(
+        "--no_labels",
+        action="store_true",
+        help="True if true labels are not available. For Nkululeko.demo results.",
+    )
+
+    args = parser.parse_args()
+
+    start = time.time()
+
+    ensemble_preds = ensemble_predictions(
+        args.configs, args.method, args.threshold, args.no_labels
+    )
+
+    # save to csv
+    ensemble_preds.to_csv(args.outfile, index=False)
+    Util("ensemble").debug(f"Ensemble predictions saved to: {args.outfile}")
+    Util("ensemble").debug(f"Ensemble done, used {time.time()-start:.2f} seconds")
+
+    Util("ensemble").debug("DONE")
+
+
+if __name__ == "__main__":
+    cwd = Path(__file__).parent
+    main(cwd)

{nkululeko-0.88.2 → nkululeko-0.88.4}/nkululeko/feat_extract/feats_ast.py

@@ -100,10 +100,8 @@ class Ast(Featureset):
             embeddings = torch.mean(last_hidden_state, dim=1)
             embeddings = embeddings.cpu().numpy()
 
-            # convert the same from (768,) to (1, 768)
-            # embeddings = embeddings.reshape(1, -1)
-            print(f"hs shape: {embeddings.shape}")
-            
+            # print(f"hs shape: {embeddings.shape}")
+            # hs shape: (1, 768)
         
         except Exception as e:
             self.util.error(f"Error extracting embeddings for file {file}: {str(e)}, fill with")

{nkululeko-0.88.2 → nkululeko-0.88.4}/nkululeko/modelrunner.py

@@ -238,21 +238,21 @@ class Modelrunner:
             if balancing == "ros":
                 from imblearn.over_sampling import RandomOverSampler
 
-                sampler = RandomOverSampler()
+                sampler = RandomOverSampler(random_state=42)
                 X_res, y_res = sampler.fit_resample(
                     self.feats_train, self.df_train[self.target]
                 )
             elif balancing == "smote":
                 from imblearn.over_sampling import SMOTE
 
-                sampler = SMOTE()
+                sampler = SMOTE(random_state=42)
                 X_res, y_res = sampler.fit_resample(
                     self.feats_train, self.df_train[self.target]
                 )
             elif balancing == "adasyn":
                 from imblearn.over_sampling import ADASYN
 
-                sampler = ADASYN()
+                sampler = ADASYN(random_state=42)
                 X_res, y_res = sampler.fit_resample(
                     self.feats_train, self.df_train[self.target]
                 )

{nkululeko-0.88.2 → nkululeko-0.88.4}/nkululeko/models/model_svm.py

@@ -1,5 +1,6 @@
 # model_svm.py
 
+import random
 from sklearn import svm
 from nkululeko.models.model import Model
 
@@ -24,6 +25,7 @@ class SVM_model(Model):
             gamma="scale",
             probability=True,
             class_weight=class_weight,
+            random_state=42,    # for consistent result
         )  # set up the classifier
 
     def set_c(self, c):

{nkululeko-0.88.2 → nkululeko-0.88.4}/nkululeko/reporting/reporter.py

@@ -144,10 +144,10 @@ class Reporter:
             and "uncertainty" not in self.probas
         ):
             probas = self.probas
-            probas["predicted"] = self.preds
-            probas["truth"] = self.truths
             # softmax the probabilities or logits
             uncertainty = probas.apply(softmax, axis=1)
+            probas["predicted"] = self.preds
+            probas["truth"] = self.truths
             try:
                 le = glob_conf.label_encoder
                 mapping = dict(zip(le.classes_, range(len(le.classes_))))
@@ -166,7 +166,8 @@ class Reporter:
             )
             probas["uncertainty"] = uncertainty
             probas["correct"] = probas.predicted == probas.truth
-            sp = os.path.join(self.util.get_path("store"), "pred_df.csv")
+            sp = self.util.get_pred_name()
+            
             self.probas = probas
             probas.to_csv(sp)
             self.util.debug(f"Saved probabilities to {sp}")

{nkululeko-0.88.2 → nkululeko-0.88.4}/nkululeko/runmanager.py

@@ -50,7 +50,7 @@ class Runmanager:
         self.last_epochs = []  # keep the epoch of best result per run
         # for all runs
         for run in range(int(self.util.config_val("EXP", "runs", 1))):
-            self.util.debug(f"run {run}")
+            self.util.debug(f"run {run} using model {glob_conf.config['MODEL']['type']}")
             # set the run index as global variable for reporting
             self.util.set_config_val("EXP", "run", run)
             self.modelrunner = Modelrunner(

{nkululeko-0.88.2 → nkululeko-0.88.4}/nkululeko/utils/util.py

@@ -1,10 +1,13 @@
 # util.py
 import ast
 import configparser
+import logging
 import os.path
 import pickle
 import sys
 
+# from sysconfig import get_config_h_filename
+# from turtle import setup
 import audeer
 import audformat
 import numpy as np
@@ -32,6 +35,7 @@ class Util:
             self.caller = caller
         else:
             self.caller = ""
+        self.config = None
         if has_config:
             try:
                 import nkululeko.glob_conf as glob_conf
@@ -49,6 +53,30 @@ class Util:
                 self.config = None
                 self.got_data_roots = False
 
+        self.setup_logging()
+        # self.logged_configs = set()
+
+    def setup_logging(self):
+        # Setup logging
+        logger = logging.getLogger(__name__)
+        if not logger.hasHandlers():
+            logger.setLevel(logging.DEBUG)  # Set the desired logging level
+
+            # Create a console handler
+            console_handler = logging.StreamHandler()
+
+            # Create a simple formatter that only shows the message
+            class SimpleFormatter(logging.Formatter):
+                def format(self, record):
+                    return record.getMessage()
+
+            # Set the formatter for the console handler
+            console_handler.setFormatter(SimpleFormatter())
+
+            # Add the console handler to the logger
+            logger.addHandler(console_handler)
+        self.logger = logger
+
     def get_path(self, entry):
         """This method allows the user to get the directory path for the given argument."""
         if self.config is None:
@@ -120,6 +148,7 @@ class Util:
 
     def set_config(self, config):
         self.config = config
+        # self.logged_configs.clear()
 
     def get_save_name(self):
         """Return a relative path to a name to save the experiment"""
@@ -128,7 +157,8 @@ class Util:
 
     def get_pred_name(self):
         store = self.get_path("store")
-        return f"{store}/pred_df.csv"
+        pred_name = self.get_model_description()
+        return f"{store}/pred_{pred_name}.csv"
 
     def is_categorical(self, pd_series):
         """Check if a dataframe column is categorical"""
@@ -233,6 +263,11 @@ class Util:
             return_string += self._get_value_descript(option[0], option[1]).replace(
                 ".", "-"
             )
+            # prevent double underscores
+            return_string = return_string.replace("__", "_")
+            # remove trailing underscores in the end
+            return_string = return_string.strip("_")
+
         return return_string
 
     def get_plot_name(self):
@@ -249,14 +284,14 @@ class Util:
         return False
 
     def error(self, message):
-        print(f"ERROR {self.caller}: {message}")
+        self.logger.error(f"ERROR: {self.caller}: {message}")
         sys.exit()
 
     def warn(self, message):
-        print(f"WARNING {self.caller}: {message}")
+        self.logger.warning(f"WARNING: {self.caller}: {message}")
 
     def debug(self, message):
-        print(f"DEBUG {self.caller}: {message}")
+        self.logger.debug(f"DEBUG: {self.caller}: {message}")
 
     def set_config_val(self, section, key, value):
         try:
@@ -278,9 +313,13 @@ class Util:
             return self.config[section][key]
         except KeyError:
             if default not in self.stopvals:
-                self.debug(f"value for {key} not found, using default: {default}")
+                self.debug(f"value for {key} is not found, using default: {default}")
             return default
 
+    @classmethod
+    def reset_logged_configs(cls):
+        cls.logged_configs.clear()
+
     def config_val_list(self, section, key, default):
         try:
             return ast.literal_eval(self.config[section][key])