nkululeko 0.81.2__py3-none-any.whl → 0.81.3__py3-none-any.whl

nkululeko/constants.py

@@ -1,2 +1,2 @@
-VERSION="0.81.2"
+VERSION="0.81.3"
 SAMPLING_RATE = 16000

nkululeko/feat_extract/feats_agender_agender.py

@@ -32,10 +32,11 @@ class AgenderAgenderSet(Featureset):
             audeer.extract_archive(archive_path, model_root)
         device = self.util.config_val("MODEL", "device", "cpu")
         self.model = audonnx.load(model_root, device=device)
-        pytorch_total_params = sum(p.numel() for p in self.model.parameters())
-        self.util.debug(
-            f"initialized agender model with {pytorch_total_params} parameters in total"
-        )
+        #        pytorch_total_params = sum(p.numel() for p in self.model.parameters())
+        # self.util.debug(
+        #     f"initialized agender model with {pytorch_total_params} parameters in total"
+        # )
+        self.util.debug("initialized agender model")
         self.model_loaded = True
 
     def extract(self):

nkululeko/feat_extract/feinberg_praat.py

@@ -1,47 +1,46 @@
-"""
-This is a copy of David R. Feinberg's Praat scripts
+"""This is a copy of David R. Feinberg's Praat scripts.
 https://github.com/drfeinberg/PraatScripts
-taken June 23rd 2022
+taken June 23rd 2022.
 """
 
 #!/usr/bin/env python3
+import math
+import statistics
+
 import numpy as np
 import pandas as pd
-import math
-from tqdm import tqdm
 import parselmouth
-import statistics
-from nkululeko.utils.util import Util
-import audiofile
 from parselmouth.praat import call
 from scipy.stats.mstats import zscore
 from sklearn.decomposition import PCA
-from sklearn.preprocessing import StandardScaler
+from tqdm import tqdm
+
+import audiofile
 
 
 # This is the function to measure source acoustics using default male parameters.
 
 
-def measurePitch(voiceID, f0min, f0max, unit):
-    sound = parselmouth.Sound(voiceID)  # read the sound
+def measure_pitch(voice_id, f0min, f0max, unit):
+    sound = parselmouth.Sound(voice_id)  # read the sound
     duration = call(sound, "Get total duration")  # duration
     pitch = call(sound, "To Pitch", 0.0, f0min, f0max)  # create a praat pitch object
-    meanF0 = call(pitch, "Get mean", 0, 0, unit)  # get mean pitch
-    stdevF0 = call(
+    mean_f0 = call(pitch, "Get mean", 0, 0, unit)  # get mean pitch
+    stdev_f0 = call(
         pitch, "Get standard deviation", 0, 0, unit
     )  # get standard deviation
     harmonicity = call(sound, "To Harmonicity (cc)", 0.01, f0min, 0.1, 1.0)
     hnr = call(harmonicity, "Get mean", 0, 0)
-    pointProcess = call(sound, "To PointProcess (periodic, cc)", f0min, f0max)
-    localJitter = call(pointProcess, "Get jitter (local)", 0, 0, 0.0001, 0.02, 1.3)
-    localabsoluteJitter = call(
-        pointProcess, "Get jitter (local, absolute)", 0, 0, 0.0001, 0.02, 1.3
+    point_process = call(sound, "To PointProcess (periodic, cc)", f0min, f0max)
+    local_jitter = call(point_process, "Get jitter (local)", 0, 0, 0.0001, 0.02, 1.3)
+    localabsolute_jitter = call(
+        point_process, "Get jitter (local, absolute)", 0, 0, 0.0001, 0.02, 1.3
     )
-    rapJitter = call(pointProcess, "Get jitter (rap)", 0, 0, 0.0001, 0.02, 1.3)
-    ppq5Jitter = call(pointProcess, "Get jitter (ppq5)", 0, 0, 0.0001, 0.02, 1.3)
-    ddpJitter = call(pointProcess, "Get jitter (ddp)", 0, 0, 0.0001, 0.02, 1.3)
-    localShimmer = call(
-        [sound, pointProcess],
+    rap_jitter = call(point_process, "Get jitter (rap)", 0, 0, 0.0001, 0.02, 1.3)
+    ppq5_jitter = call(point_process, "Get jitter (ppq5)", 0, 0, 0.0001, 0.02, 1.3)
+    ddp_jitter = call(point_process, "Get jitter (ddp)", 0, 0, 0.0001, 0.02, 1.3)
+    local_shimmer = call(
+        [sound, point_process],
         "Get shimmer (local)",
         0,
         0,
@@ -50,8 +49,8 @@ def measurePitch(voiceID, f0min, f0max, unit):
         1.3,
         1.6,
     )
-    localdbShimmer = call(
-        [sound, pointProcess],
+    localdb_shimmer = call(
+        [sound, point_process],
         "Get shimmer (local_dB)",
         0,
         0,
@@ -60,8 +59,8 @@ def measurePitch(voiceID, f0min, f0max, unit):
         1.3,
         1.6,
     )
-    apq3Shimmer = call(
-        [sound, pointProcess],
+    apq3_shimmer = call(
+        [sound, point_process],
         "Get shimmer (apq3)",
         0,
         0,
@@ -70,8 +69,8 @@ def measurePitch(voiceID, f0min, f0max, unit):
         1.3,
         1.6,
     )
-    aqpq5Shimmer = call(
-        [sound, pointProcess],
+    aqpq5_shimmer = call(
+        [sound, point_process],
         "Get shimmer (apq5)",
         0,
         0,
@@ -80,8 +79,8 @@ def measurePitch(voiceID, f0min, f0max, unit):
         1.3,
         1.6,
     )
-    apq11Shimmer = call(
-        [sound, pointProcess],
+    apq11_shimmer = call(
+        [sound, point_process],
         "Get shimmer (apq11)",
         0,
         0,
@@ -90,26 +89,26 @@ def measurePitch(voiceID, f0min, f0max, unit):
         1.3,
         1.6,
     )
-    ddaShimmer = call(
-        [sound, pointProcess], "Get shimmer (dda)", 0, 0, 0.0001, 0.02, 1.3, 1.6
+    dda_shimmer = call(
+        [sound, point_process], "Get shimmer (dda)", 0, 0, 0.0001, 0.02, 1.3, 1.6
     )
 
     return (
         duration,
-        meanF0,
-        stdevF0,
+        mean_f0,
+        stdev_f0,
         hnr,
-        localJitter,
-        localabsoluteJitter,
-        rapJitter,
-        ppq5Jitter,
-        ddpJitter,
-        localShimmer,
-        localdbShimmer,
-        apq3Shimmer,
-        aqpq5Shimmer,
-        apq11Shimmer,
-        ddaShimmer,
+        local_jitter,
+        localabsolute_jitter,
+        rap_jitter,
+        ppq5_jitter,
+        ddp_jitter,
+        local_shimmer,
+        localdb_shimmer,
+        apq3_shimmer,
+        aqpq5_shimmer,
+        apq11_shimmer,
+        dda_shimmer,
     )
 
 
@@ -120,13 +119,13 @@ def measurePitch(voiceID, f0min, f0max, unit):
 # Adapted from: DOI 10.17605/OSF.IO/K2BHS
 # This function measures formants using Formant Position formula
 # def measureFormants(sound, wave_file, f0min,f0max):
-def measureFormants(sound, f0min, f0max):
+def measure_formants(sound, f0min, f0max):
     sound = parselmouth.Sound(sound)  # read the sound
     #    pitch = call(sound, "To Pitch (cc)", 0, f0min, 15, 'no', 0.03, 0.45, 0.01, 0.35, 0.14, f0max)
-    pointProcess = call(sound, "To PointProcess (periodic, cc)", f0min, f0max)
+    point_process = call(sound, "To PointProcess (periodic, cc)", f0min, f0max)
 
     formants = call(sound, "To Formant (burg)", 0.0025, 5, 5000, 0.025, 50)
-    numPoints = call(pointProcess, "Get number of points")
+    num_points = call(point_process, "Get number of points")
 
     f1_list = []
     f2_list = []
@@ -134,9 +133,9 @@ def measureFormants(sound, f0min, f0max):
     f4_list = []
 
     # Measure formants only at glottal pulses
-    for point in range(0, numPoints):
+    for point in range(0, num_points):
         point += 1
-        t = call(pointProcess, "Get time from index", point)
+        t = call(point_process, "Get time from index", point)
         f1 = call(formants, "Get value at time", 1, t, "Hertz", "Linear")
         f2 = call(formants, "Get value at time", 2, t, "Hertz", "Linear")
         f3 = call(formants, "Get value at time", 3, t, "Hertz", "Linear")
@@ -179,7 +178,7 @@ def measureFormants(sound, f0min, f0max):
 # ## This function runs a 2-factor Principle Components Analysis (PCA) on Jitter and Shimmer
 
 
-def runPCA(df):
+def run_pca(df):
     # z-score the Jitter and Shimmer measurements
     measures = [
         "localJitter",
@@ -211,19 +210,19 @@ def runPCA(df):
     # PCA
     pca = PCA(n_components=2)
     try:
-        principalComponents = pca.fit_transform(x)
-        if np.any(np.isnan(principalComponents)):
+        principal_components = pca.fit_transform(x)
+        if np.any(np.isnan(principal_components)):
             print("pc is nan")
-            print(f"count: {np.count_nonzero(np.isnan(principalComponents))}")
-            print(principalComponents)
-            principalComponents = np.nan_to_num(principalComponents)
+            print(f"count: {np.count_nonzero(np.isnan(principal_components))}")
+            print(principal_components)
+            principal_components = np.nan_to_num(principal_components)
     except ValueError:
         print("need more than one file for pca")
-        principalComponents = [[0, 0]]
-    principalDf = pd.DataFrame(
-        data=principalComponents, columns=["JitterPCA", "ShimmerPCA"]
+        principal_components = [[0, 0]]
+    principal_df = pd.DataFrame(
+        data=principal_components, columns=["JitterPCA", "ShimmerPCA"]
     )
-    return principalDf
+    return principal_df
 
 
 # ## This block of code runs the above functions on all of the '.wav' files in the /audio folder
@@ -231,22 +230,21 @@ def runPCA(df):
 
 def compute_features(file_index):
     # create lists to put the results
-    file_list = []
     duration_list = []
-    mean_F0_list = []
-    sd_F0_list = []
+    mean_f0_list = []
+    sd_f0_list = []
     hnr_list = []
-    localJitter_list = []
-    localabsoluteJitter_list = []
-    rapJitter_list = []
-    ppq5Jitter_list = []
-    ddpJitter_list = []
-    localShimmer_list = []
-    localdbShimmer_list = []
-    apq3Shimmer_list = []
-    aqpq5Shimmer_list = []
-    apq11Shimmer_list = []
-    ddaShimmer_list = []
+    local_jitter_list = []
+    localabsolute_jitter_list = []
+    rap_jitter_list = []
+    ppq5_jitter_list = []
+    ddp_jitter_list = []
+    local_shimmer_list = []
+    localdb_shimmer_list = []
+    apq3_shimmer_list = []
+    aqpq5_shimmer_list = []
+    apq11_shimmer_list = []
+    dda_shimmer_list = []
     f1_mean_list = []
     f2_mean_list = []
     f3_mean_list = []
@@ -268,21 +266,21 @@ def compute_features(file_index):
             sound = parselmouth.Sound(values=signal, sampling_frequency=sampling_rate)
             (
                 duration,
-                meanF0,
-                stdevF0,
+                mean_f0,
+                stdev_f0,
                 hnr,
-                localJitter,
-                localabsoluteJitter,
-                rapJitter,
-                ppq5Jitter,
-                ddpJitter,
-                localShimmer,
-                localdbShimmer,
-                apq3Shimmer,
-                aqpq5Shimmer,
-                apq11Shimmer,
-                ddaShimmer,
-            ) = measurePitch(sound, 75, 300, "Hertz")
+                local_jitter,
+                localabsolute_jitter,
+                rap_jitter,
+                ppq5_jitter,
+                ddp_jitter,
+                local_shimmer,
+                localdb_shimmer,
+                apq3_shimmer,
+                aqpq5_shimmer,
+                apq11_shimmer,
+                dda_shimmer,
+            ) = measure_pitch(sound, 75, 300, "Hertz")
             (
                 f1_mean,
                 f2_mean,
@@ -292,28 +290,28 @@ def compute_features(file_index):
                 f2_median,
                 f3_median,
                 f4_median,
-            ) = measureFormants(sound, 75, 300)
+            ) = measure_formants(sound, 75, 300)
             #        file_list.append(wave_file) # make an ID list
         except (statistics.StatisticsError, parselmouth.PraatError) as errors:
             print(f"error on file {wave_file}: {errors}")
 
         duration_list.append(duration)  # make duration list
-        mean_F0_list.append(meanF0)  # make a mean F0 list
-        sd_F0_list.append(stdevF0)  # make a sd F0 list
+        mean_f0_list.append(mean_f0)  # make a mean F0 list
+        sd_f0_list.append(stdev_f0)  # make a sd F0 list
         hnr_list.append(hnr)  # add HNR data
 
         # add raw jitter and shimmer measures
-        localJitter_list.append(localJitter)
-        localabsoluteJitter_list.append(localabsoluteJitter)
-        rapJitter_list.append(rapJitter)
-        ppq5Jitter_list.append(ppq5Jitter)
-        ddpJitter_list.append(ddpJitter)
-        localShimmer_list.append(localShimmer)
-        localdbShimmer_list.append(localdbShimmer)
-        apq3Shimmer_list.append(apq3Shimmer)
-        aqpq5Shimmer_list.append(aqpq5Shimmer)
-        apq11Shimmer_list.append(apq11Shimmer)
-        ddaShimmer_list.append(ddaShimmer)
+        local_jitter_list.append(local_jitter)
+        localabsolute_jitter_list.append(localabsolute_jitter)
+        rap_jitter_list.append(rap_jitter)
+        ppq5_jitter_list.append(ppq5_jitter)
+        ddp_jitter_list.append(ddp_jitter)
+        local_shimmer_list.append(local_shimmer)
+        localdb_shimmer_list.append(localdb_shimmer)
+        apq3_shimmer_list.append(apq3_shimmer)
+        aqpq5_shimmer_list.append(aqpq5_shimmer)
+        apq11_shimmer_list.append(apq11_shimmer)
+        dda_shimmer_list.append(dda_shimmer)
 
         # add the formant data
         f1_mean_list.append(f1_mean)
@@ -330,20 +328,20 @@ def compute_features(file_index):
         np.column_stack(
             [
                 duration_list,
-                mean_F0_list,
-                sd_F0_list,
+                mean_f0_list,
+                sd_f0_list,
                 hnr_list,
-                localJitter_list,
-                localabsoluteJitter_list,
-                rapJitter_list,
-                ppq5Jitter_list,
-                ddpJitter_list,
-                localShimmer_list,
-                localdbShimmer_list,
-                apq3Shimmer_list,
-                aqpq5Shimmer_list,
-                apq11Shimmer_list,
-                ddaShimmer_list,
+                local_jitter_list,
+                localabsolute_jitter_list,
+                rap_jitter_list,
+                ppq5_jitter_list,
+                ddp_jitter_list,
+                local_shimmer_list,
+                localdb_shimmer_list,
+                apq3_shimmer_list,
+                aqpq5_shimmer_list,
+                apq11_shimmer_list,
+                dda_shimmer_list,
                 f1_mean_list,
                 f2_mean_list,
                 f3_mean_list,
@@ -382,7 +380,7 @@ def compute_features(file_index):
     )
 
     # add pca data
-    pcaData = runPCA(df)  # Run jitter and shimmer PCA
+    pcaData = run_pca(df)  # Run jitter and shimmer PCA
     df = pd.concat([df, pcaData], axis=1)  # Add PCA data
     # reload the data so it's all numbers
     df.to_csv("processed_results.csv", index=False)

nkululeko/models/model.py

@@ -4,7 +4,7 @@ import pandas as pd
 import numpy as np
 import nkululeko.glob_conf as glob_conf
 import sklearn.utils
-from nkululeko.reporter import Reporter
+from nkululeko.reporting.reporter import Reporter
 import ast
 from sklearn.model_selection import GridSearchCV
 import pickle

nkululeko/models/model_cnn.py

@@ -20,7 +20,7 @@ from PIL import Image
 from nkululeko.utils.util import Util
 import nkululeko.glob_conf as glob_conf
 from nkululeko.models.model import Model
-from nkululeko.reporter import Reporter
+from nkululeko.reporting.reporter import Reporter
 from nkululeko.losses.loss_softf1loss import SoftF1Loss
 
 

nkululeko/models/model_mlp.py

@@ -2,7 +2,7 @@
 from nkululeko.utils.util import Util
 import nkululeko.glob_conf as glob_conf
 from nkululeko.models.model import Model
-from nkululeko.reporter import Reporter
+from nkululeko.reporting.reporter import Reporter
 import torch
 import ast
 import numpy as np

nkululeko/models/model_mlp_regression.py

@@ -1,16 +1,20 @@
 # model_mlp.py
-from nkululeko.utils.util import Util
-import nkululeko.glob_conf as glob_conf
-from nkululeko.models.model import Model
-from nkululeko.reporter import Reporter
-import torch
 import ast
-import numpy as np
-from sklearn.metrics import mean_squared_error, mean_absolute_error
 from collections import OrderedDict
-from nkululeko.losses.loss_ccc import ConcordanceCorCoeff
 import os
 
+import numpy as np
+import torch
+
+from audmetric import concordance_cc
+from audmetric import mean_absolute_error
+from audmetric import mean_squared_error
+
+import nkululeko.glob_conf as glob_conf
+from nkululeko.losses.loss_ccc import ConcordanceCorCoeff
+from nkululeko.models.model import Model
+from nkululeko.reporting.reporter import Reporter
+
 
 class MLP_Reg_model(Model):
     """MLP = multi layer perceptron"""
@@ -201,7 +205,7 @@ class MLP_Reg_model(Model):
         elif measure == "mae":
             result = mean_absolute_error(targets.numpy(), predictions.numpy())
         elif measure == "ccc":
-            result = Reporter.ccc(targets.numpy(), predictions.numpy())
+            result = concordance_cc(targets.numpy(), predictions.numpy())
         else:
             self.util.error(f"unknown measure: {measure}")
         return result, targets, predictions

nkululeko/{reporter.py → reporting/reporter.py}

@@ -2,25 +2,27 @@ import ast
 import glob
 import json
 import math
+
+from confidence_intervals import evaluate_with_conf_int
 import matplotlib.pyplot as plt
 import numpy as np
 from scipy.stats import pearsonr
-from sklearn.metrics import (
-    ConfusionMatrixDisplay,
-    accuracy_score,
-    classification_report,
-    confusion_matrix,
-    mean_squared_error,
-    mean_absolute_error,
-    r2_score,
-    recall_score,
-)
-from sklearn.utils import resample
+from sklearn.metrics import ConfusionMatrixDisplay
+from sklearn.metrics import classification_report
+from sklearn.metrics import confusion_matrix
+from sklearn.metrics import r2_score
+from torch import is_tensor
+
+from audmetric import accuracy
+from audmetric import concordance_cc
+from audmetric import mean_absolute_error
+from audmetric import mean_squared_error
+from audmetric import unweighted_average_recall
 
 import nkululeko.glob_conf as glob_conf
-from nkululeko.reporting.report_item import ReportItem
-from nkululeko.result import Result
 from nkululeko.reporting.defines import Header
+from nkululeko.reporting.report_item import ReportItem
+from nkululeko.reporting.result import Result
 from nkululeko.utils.util import Util
 
 
@@ -44,11 +46,11 @@ class Reporter:
                 self.result.measure = self.MEASURE
 
     def __init__(self, truths, preds, run, epoch):
-        """Initialization with ground truth und predictions vector"""
+        """Initialization with ground truth und predictions vector."""
         self.util = Util("reporter")
         self.format = self.util.config_val("PLOT", "format", "png")
-        self.truths = truths
-        self.preds = preds
+        self.truths = np.asarray(truths)
+        self.preds = np.asarray(preds)
         self.result = Result(0, 0, 0, 0, "unknown")
         self.run = run
         self.epoch = epoch
@@ -56,30 +58,57 @@ class Reporter:
         self.cont_to_cat = False
         if len(self.truths) > 0 and len(self.preds) > 0:
             if self.util.exp_is_classification():
-                self.result.test = recall_score(
-                    self.truths, self.preds, average="macro"
+                uar, (upper, lower) = evaluate_with_conf_int(
+                    self.preds,
+                    unweighted_average_recall,
+                    self.truths,
+                    num_bootstraps=1000,
+                    alpha=5,
                 )
-                self.result.loss = 1 - accuracy_score(self.truths, self.preds)
+                self.result.test = uar
+                self.result.set_upper_lower(upper, lower)
+                self.result.loss = 1 - accuracy(self.truths, self.preds)
             else:
                 # regression experiment
                 if self.measure == "mse":
-                    self.result.test = mean_squared_error(self.truths, self.preds)
+                    test_result, (upper, lower) = evaluate_with_conf_int(
+                        self.preds,
+                        mean_squared_error,
+                        self.truths,
+                        num_bootstraps=1000,
+                        alpha=5,
+                    )
                 elif self.measure == "mae":
-                    self.result.test = mean_absolute_error(self.truths, self.preds)
+                    test_result, (upper, lower) = evaluate_with_conf_int(
+                        self.preds,
+                        mean_absolute_error,
+                        self.truths,
+                        num_bootstraps=1000,
+                        alpha=5,
+                    )
                 elif self.measure == "ccc":
-                    self.result.test = self.ccc(self.truths, self.preds)
+                    test_result, (upper, lower) = evaluate_with_conf_int(
+                        self.preds,
+                        concordance_cc,
+                        self.truths,
+                        num_bootstraps=1000,
+                        alpha=5,
+                    )
+
                     if math.isnan(self.result.test):
                         self.util.debug(f"Truth: {self.truths}")
                         self.util.debug(f"Predict.: {self.preds}")
-                        self.util.debug(f"Result is NAN: setting to -1")
+                        self.util.debug("Result is NAN: setting to -1")
                         self.result.test = -1
                 else:
                     self.util.error(f"unknown measure: {self.measure}")
 
+                self.result.test = test_result
+                self.result.set_upper_lower(upper, lower)
                 # train and loss are being set by the model
 
     def set_id(self, run, epoch):
-        """Make the report identifiable with run and epoch index"""
+        """Make the report identifiable with run and epoch index."""
         self.run = run
         self.epoch = epoch
 
@@ -97,9 +126,12 @@ class Reporter:
         self._plot_confmat(self.truths, self.preds, plot_name, epoch)
 
     def plot_per_speaker(self, result_df, plot_name, function):
-        """Plot a confusion matrix with the mode category per speakers
+        """Plot a confusion matrix with the mode category per speakers.
+
         Args:
-            * result_df: a pandas dataframe with columns: preds, truths and speaker
+            result_df: a pandas dataframe with columns: preds, truths and speaker.
+            plot_name: name for the figure.
+            function: either mode or mean.
         """
         speakers = result_df.speaker.unique()
         pred = np.zeros(0)
@@ -128,8 +160,14 @@ class Reporter:
         fig_dir = self.util.get_path("fig_dir")
         labels = glob_conf.labels
         fig = plt.figure()  # figsize=[5, 5]
-        uar = recall_score(truths, preds, average="macro")
-        acc = accuracy_score(truths, preds)
+        uar, (upper, lower) = evaluate_with_conf_int(
+            self.preds,
+            unweighted_average_recall,
+            self.truths,
+            num_bootstraps=1000,
+            alpha=5,
+        )
+        acc = accuracy(truths, preds)
         cm = confusion_matrix(
             truths, preds, normalize=None
         )  # normalize must be one of {'true', 'pred', 'all', None}
@@ -138,6 +176,7 @@ class Reporter:
                 f"mismatch between confmatrix dim ({cm.shape[0]}) and labels"
                 f" length ({len(labels)}: {labels})"
             )
+
         try:
             disp = ConfusionMatrixDisplay(
                 confusion_matrix=cm, display_labels=labels
@@ -150,12 +189,23 @@ class Reporter:
 
         reg_res = ""
         if not self.is_classification:
-            reg_res = f", {self.MEASURE}: {self.result.test:.3f}"
+            reg_res = f"{self.result.test:.3f} {self.MEASURE}"
+
+        uar_str = str(int(uar * 1000) / 1000.0)[1:]
+        acc_str = str(int(acc * 1000) / 1000.0)[1:]
+        up_str = str(int(upper * 1000) / 1000.0)[1:]
+        low_str = str(int(lower * 1000) / 1000.0)[1:]
 
         if epoch != 0:
-            plt.title(f"Confusion Matrix, UAR: {uar:.3f}{reg_res}, Epoch: {epoch}")
+            plt.title(
+                f"Confusion Matrix, UAR: {uar_str} "
+                + f"(+-{up_str}/{low_str}), {reg_res}, Epoch: {epoch}"
+            )
         else:
-            plt.title(f"Confusion Matrix, UAR: {uar:.3f}{reg_res}")
+            plt.title(
+                f"Confusion Matrix, UAR: {uar_str} "
+                + f"(+-{up_str}/{low_str}) {reg_res}"
+            )
         img_path = f"{fig_dir}{plot_name}.{self.format}"
         plt.savefig(img_path)
         fig.clear()
@@ -172,9 +222,10 @@ class Reporter:
         )
 
         res_dir = self.util.get_path("res_dir")
-        uar = int(uar * 1000) / 1000.0
-        acc = int(acc * 1000) / 1000.0
-        rpt = f"epoch: {epoch}, UAR: {uar}, ACC: {acc}"
+        rpt = (
+            f"epoch: {epoch}, UAR: {uar_str}"
+            + f", (+-{up_str}/{low_str}), ACC: {acc_str}"
+        )
         # print(rpt)
         self.util.debug(rpt)
         file_name = f"{res_dir}{self.util.get_exp_name()}_conf.txt"
@@ -182,7 +233,7 @@ class Reporter:
             text_file.write(rpt)
 
     def print_results(self, epoch):
-        """Print all evaluation values to text file"""
+        """Print all evaluation values to text file."""
         res_dir = self.util.get_path("res_dir")
         file_name = f"{res_dir}{self.util.get_exp_name()}_{epoch}.txt"
         if self.util.exp_is_classification():
@@ -279,19 +330,3 @@ class Reporter:
         plt.legend()
         plt.savefig(f"{fig_dir}{out_name}.{self.format}")
         plt.close()
-
-    @staticmethod
-    def ccc(ground_truth, prediction):
-        mean_gt = np.mean(ground_truth, 0)
-        mean_pred = np.mean(prediction, 0)
-        var_gt = np.var(ground_truth, 0)
-        var_pred = np.var(prediction, 0)
-        v_pred = prediction - mean_pred
-        v_gt = ground_truth - mean_gt
-        cor = sum(v_pred * v_gt) / (np.sqrt(sum(v_pred**2)) * np.sqrt(sum(v_gt**2)))
-        sd_gt = np.std(ground_truth)
-        sd_pred = np.std(prediction)
-        numerator = 2 * cor * sd_gt * sd_pred
-        denominator = var_gt + var_pred + (mean_gt - mean_pred) ** 2
-        ccc = numerator / denominator
-        return ccc

nkululeko/{result.py → reporting/result.py}

@@ -12,6 +12,11 @@ class Result:
     def get_result(self):
         return self.test
 
+    def set_upper_lower(self, upper, lower):
+        """Set the upper and lower bound of confidence interval."""
+        self.upper = upper
+        self.lower = lower
+
     def get_test_result(self):
         return f"test: {self.test:.3f} {self.measure}"
 

nkululeko/runmanager.py

@@ -1,6 +1,6 @@
 # runmanager.py
 
-from nkululeko.reporter import Reporter
+from nkululeko.reporting.reporter import Reporter
 from nkululeko.utils.util import Util
 import nkululeko.glob_conf as glob_conf
 from nkululeko.modelrunner import Modelrunner

{nkululeko-0.81.2.dist-info → nkululeko-0.81.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.81.2
+Version: 0.81.3
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -18,7 +18,9 @@ Requires-Dist: audformat
 Requires-Dist: audinterface
 Requires-Dist: audiofile
 Requires-Dist: audiomentations
+Requires-Dist: audmetric
 Requires-Dist: audonnx
+Requires-Dist: confidence-intervals
 Requires-Dist: datasets
 Requires-Dist: imageio
 Requires-Dist: laion-clap
@@ -321,6 +323,10 @@ F. Burkhardt, Johannes Wagner, Hagen Wierstorf, Florian Eyben and Björn Schulle
 Changelog
 =========
 
+Version 0.81.3
+--------------
+* added confidence intervals to result reporting
+
 Version 0.81.2
 --------------
 * added a parselmouth.Praat error if pitch out of range

{nkululeko-0.81.2.dist-info → nkululeko-0.81.3.dist-info}/RECORD

@@ -2,7 +2,7 @@ nkululeko/__init__.py,sha256=62f8HiEzJ8rG2QlTFJXUCMpvuH3fKI33DoJSj33mscc,63
 nkululeko/aug_train.py,sha256=YhuZnS_WVWnun9G-M6g5n6rbRxoVREz6Zh7k6qprFNQ,3194
 nkululeko/augment.py,sha256=4MG0apTAG5RgkuJrYEjGgDdbodZWi_HweSPNI1JJ5QA,3051
 nkululeko/cacheddataset.py,sha256=lIJ6hUo5LoxSrzXtWV8mzwO7wRtUETWnOQ4ws2XfL1E,969
-nkululeko/constants.py,sha256=zujT9J62h5BIBCxzigDt23S5plsfoyutXsGMdK_xkAM,39
+nkululeko/constants.py,sha256=hx9HFHOlApn60yieWI1qr4PbrKeT3EFK1aaDMxlt5xU,39
 nkululeko/demo.py,sha256=me8EdjN-zrzClVy9FEmqbTQyDDON88W8vPpWEE8T0cI,2500
 nkululeko/demo_feats.py,sha256=sAeGFojhEj9WEDFtG3SzPBmyYJWLF2rkbpp65m8Ujo4,2025
 nkululeko/demo_predictor.py,sha256=CQL6DO7QxwmwoB_6DlgDS-pdG1KuvemYJ1NEpMjmMk8,4733
@@ -18,10 +18,8 @@ nkululeko/multidb.py,sha256=4ceCu9LFrMGlrcgtz4pWuOQb2KA3jR5uo3FjZgAEBD4,5732
 nkululeko/nkululeko.py,sha256=Ty8cdusXUec9BHml8Gsp1r7DXuvIBMFXUckMpzILBnQ,1966
 nkululeko/plots.py,sha256=K88ZRPFGX_r03BT742H06Dde20xZYdltv7dxjgUiAFA,23025
 nkululeko/predict.py,sha256=dRXX-sQVESa7cNi_56S6UkUOa_pV1g_K4xYtYVM1SJs,1876
-nkululeko/reporter.py,sha256=Gg0dsZclMmdTRUju7yWM3tBVhEZno9VSKD4Tcu_1pJI,11497
 nkululeko/resample.py,sha256=Yzfr_rInG9afPZFnEjiQ3EKRdMSwyYKVQwt9-yNGJn8,2233
-nkululeko/result.py,sha256=kLeEyHQxPzqgCcTadgwvGd2b8gJGpdaf5feHqshjPH0,574
-nkululeko/runmanager.py,sha256=YNjYLzf4KrtcOyiDLF06YLs3nU3U7n_hY_VH4fYFuh0,7451
+nkululeko/runmanager.py,sha256=JNBm7JJN8QU8qEqfWr4eS6rkPnBWoVdIUTynHctCPpw,7461
 nkululeko/scaler.py,sha256=4nkIqoajkIkuTPK0Z02ifMN_awl6fP_i-GBYdoGYgGM,4101
 nkululeko/segment.py,sha256=YLKckX44tbvTb3LrdgYw9X4guzuF27sutl92z9DkpZU,4835
 nkululeko/syllable_nuclei.py,sha256=Sky-C__MeUDaxqHnDl2TGLLYOYvsahD35TUjWGeG31k,10047
@@ -49,7 +47,7 @@ nkululeko/data/dataset.py,sha256=n6v_vVdA0EsZ-NaTgnYfPlCT4QCcD02mJJb-oD7SaSU,272
 nkululeko/data/dataset_csv.py,sha256=v3lSjF23EVjoP460QOfhdcqbWAlBQWlBOuaYujZoS4s,3407
 nkululeko/feat_extract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/feat_extract/feats_agender.py,sha256=_lAL6IxJDJH2bhIvd7yarTqQryx7FjbQXAgY0mJP-KI,3192
-nkululeko/feat_extract/feats_agender_agender.py,sha256=ckQN8K02vdPLUWIylxg5Z4X145gBlPbSDbyBRjabLD0,3278
+nkululeko/feat_extract/feats_agender_agender.py,sha256=5dA7YA-YGxODovMC7ynMk3bnpPjfs0ApvSfjqvoSZY0,3346
 nkululeko/feat_extract/feats_analyser.py,sha256=_5oz4y-NZCEBgfNP2GZ9WNqQR50Hbykm0TvDVomWP0U,11399
 nkululeko/feat_extract/feats_audmodel.py,sha256=TRCkLqPgnyWN-OAcO69pPZF2FIbBy5ERb5ZY22qh6iA,3108
 nkululeko/feat_extract/feats_audmodel_dim.py,sha256=yg39CSR0b54AJyOAlXO3M1ohyY9Rbrjf18pllsoQ03g,3078
@@ -69,20 +67,20 @@ nkululeko/feat_extract/feats_trill.py,sha256=PpygJK_W6QoBNeSah9npQPiQlJxLWFn6TSO
 nkululeko/feat_extract/feats_wav2vec2.py,sha256=sFf-WkLUgKUQsFxGO9m2hS3uYoGkv95mZavCEZyWFGA,5072
 nkululeko/feat_extract/feats_wavlm.py,sha256=RhI0oWIsknnxTVmdnNS_xJO1NnUUR0CUNDWH1yTpNLk,4683
 nkululeko/feat_extract/featureset.py,sha256=-ynkdor8iX7BFx10aIbB3LfwxrrzPoBGz9kXwyAJO9M,1375
-nkululeko/feat_extract/feinberg_praat.py,sha256=7V1VhVMu4QrXkdcXpmqCbpStXfpmOHtfx5GzxXWukz8,21287
+nkululeko/feat_extract/feinberg_praat.py,sha256=EP9pMALjlKdiYInLQdrZ7MmE499Mq-ISRCgqbqL3Rxc,21304
 nkululeko/losses/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/losses/loss_ccc.py,sha256=NOK0y0fxKUnU161B5geap6Fmn8QzoPl2MqtPiV8IuJE,976
 nkululeko/losses/loss_softf1loss.py,sha256=5gW-PuiqeAZcRgfwjueIOQtMokOjZWgQnVIv59HKTCo,1309
 nkululeko/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nkululeko/models/model.py,sha256=SZ2HQ3KiF5fcmrTcvko1E95EQQeFIaPCG90DvZVHbBA,11638
+nkululeko/models/model.py,sha256=8gjRsjSYLWZvfcyTCWhbZ741rkHhx8lxCS2NlSOLP1Y,11648
 nkululeko/models/model_bayes.py,sha256=wI7-sCwibqXMCHviu349TYjgJXXNXym-Z6ZM83uxlFQ,378
-nkululeko/models/model_cnn.py,sha256=iyXeRsAMVeRST1j_D2AUngE02CtVkg6vWwQc1BOaBl0,9716
+nkululeko/models/model_cnn.py,sha256=j4NTp7quWqInzOPfpiMrTcfMbXkOsdlFF9ns0tW_ld4,9726
 nkululeko/models/model_gmm.py,sha256=onovzGBeguwZ-upXtuDLaBw9sd6fDDQslVBOrz1Z8TE,645
 nkululeko/models/model_knn.py,sha256=5tGqiPo2JTw9VLmD-MXNZKFJ5RTLA6uv_blJDJ9lScA,573
 nkululeko/models/model_knn_reg.py,sha256=Fbuk6Ku6eyrbbMEk7rB5dwfhvQOMsdZk6HI_0T0gYPw,580
 nkululeko/models/model_lin_reg.py,sha256=NBTnY2ULuhUBt5ArYQwskZ2Vq4BBDGkqd9SYBFl7Ql4,392
-nkululeko/models/model_mlp.py,sha256=IjiiupLxm5ddb73-eU5Ad79Gb6enurR1fgGY-7NkbFc,9097
-nkululeko/models/model_mlp_regression.py,sha256=F0SaU1qAjnGmTTg-ti1s-XmFYVUYxSV0TJw0_jMxlKU,10054
+nkululeko/models/model_mlp.py,sha256=lYhGrkqEj6fa6a_tcPrqEoorOpM7t7bjSfFLKEV6pu4,9107
+nkululeko/models/model_mlp_regression.py,sha256=NP1yEsqvpDcDBWWzDq7W4SHnXC1kE4fAo4A9aBCq3cY,10083
 nkululeko/models/model_svm.py,sha256=dqDQbfRCtlW3RNqpHDGVsj3ikc131gKURHj5VzAcCr0,867
 nkululeko/models/model_svr.py,sha256=p-Mb4Bn54yOe1upuHQKNpfj4ttOmQnm9pCB7ECkJkJQ,699
 nkululeko/models/model_tree.py,sha256=soXjV523eRvRZ-jbX7X_3S73Wto1B9bm7ZzzDmgYzTc,390
@@ -94,6 +92,8 @@ nkululeko/reporting/defines.py,sha256=IsY1YgKRMaABpylVKjBJgJ5bNCEbGCVA_E6pivraqS
 nkululeko/reporting/latex_writer.py,sha256=qiCRSmB4KOD_za4oHu5x-PhwjZohzfo8wecMOwlXZwc,1886
 nkululeko/reporting/report.py,sha256=W0rcigDdjBvxZQ3pZja_gvToILYvaZ1BFtnN2qFRfYI,1060
 nkululeko/reporting/report_item.py,sha256=siWeGNgo4bAE46YBMNcsdf3jTMTy76BO9Fi6DTvDig4,533
+nkululeko/reporting/reporter.py,sha256=wwpY0gA-8E8d26XH3DSmXm3X0BkBw2Y0YyEiUiNU_Y0,12670
+nkululeko/reporting/result.py,sha256=nSN5or-Py2GPRWHkWpGRh7UCi1W0er7WLEHz8fYLk-A,742
 nkululeko/segmenting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/segmenting/seg_inaspeechsegmenter.py,sha256=pmLHuXsaqvcdYxB4PSW9l1mbQWZZBJFhi_CGabqydas,1947
 nkululeko/segmenting/seg_silero.py,sha256=lLytS38KzARS17omwv8VBw-zz60RVSXGSvZ5EvWlcWQ,3301
@@ -101,8 +101,8 @@ nkululeko/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/utils/files.py,sha256=UiGAtZRWYjHSvlmPaTMtzyNNGE6qaLaxQkybctS7iRM,4021
 nkululeko/utils/stats.py,sha256=29otJpUp1VqbtDKmlLkPPzBmVfTFiHZ70rUdR4860rM,2788
 nkululeko/utils/util.py,sha256=_Z6OMJ3f-8TdETW9eqJYY5hwNRS5XCt9azzRnqoTTZE,12330
-nkululeko-0.81.2.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
-nkululeko-0.81.2.dist-info/METADATA,sha256=-Oo7DH0SM9gF8F0c65DLjGIt6rnUUPF_Ah_OgJrxDRA,34523
-nkululeko-0.81.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-nkululeko-0.81.2.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
-nkululeko-0.81.2.dist-info/RECORD,,
+nkululeko-0.81.3.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
+nkululeko-0.81.3.dist-info/METADATA,sha256=72Q5q8KeaEP3I0TrVzswdI4g0Fc0hnCG-kPFZke8YM8,34664
+nkululeko-0.81.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+nkululeko-0.81.3.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
+nkululeko-0.81.3.dist-info/RECORD,,