nkululeko 0.83.3__py3-none-any.whl → 0.84.1__py3-none-any.whl

nkululeko/augmenting/resampler.py

@@ -12,16 +12,19 @@ from nkululeko.utils.util import Util
 
 
 class Resampler:
-    def __init__(self, df, not_testing=True):
+    def __init__(self, df, replace, not_testing=True):
         self.SAMPLING_RATE = 16000
         self.df = df
         self.util = Util("resampler", has_config=not_testing)
         self.util.warn(f"all files might be resampled to {self.SAMPLING_RATE}")
         self.not_testing = not_testing
+        self.replace = eval(self.util.config_val(
+            "RESAMPLE", "replace", "False")) if not not_testing else replace
 
     def resample(self):
         files = self.df.index.get_level_values(0).values
-        replace = eval(self.util.config_val("RESAMPLE", "replace", "False"))
+        # replace = eval(self.util.config_val("RESAMPLE", "replace", "False"))
+        replace = self.replace
         if self.not_testing:
             store = self.util.get_path("store")
         else:
@@ -42,7 +45,8 @@ class Resampler:
                 continue
             if org_sr != self.SAMPLING_RATE:
                 self.util.debug(f"resampling {f} (sr = {org_sr})")
-                resampler = torchaudio.transforms.Resample(org_sr, self.SAMPLING_RATE)
+                resampler = torchaudio.transforms.Resample(
+                    org_sr, self.SAMPLING_RATE)
                 signal = resampler(signal)
                 if replace:
                     torchaudio.save(
@@ -59,7 +63,8 @@ class Resampler:
             self.df = self.df.set_index(
                 self.df.index.set_levels(new_files, level="file")
             )
-            target_file = self.util.config_val("RESAMPLE", "target", "resampled.csv")
+            target_file = self.util.config_val(
+                "RESAMPLE", "target", "resampled.csv")
             # remove encoded labels
             target = self.util.config_val("DATA", "target", "emotion")
             if "class_label" in self.df.columns:

nkululeko/constants.py

@@ -1,2 +1,2 @@
-VERSION="0.83.3"
+VERSION="0.84.1"
 SAMPLING_RATE = 16000

nkululeko/demo.py

@@ -2,8 +2,9 @@
 # Demonstration code to use the ML-experiment framework
 # Test the loading of a previously trained model and demo mode
 # needs the project config file to run before
-"""
-This script is used to test the loading of a previously trained model and run it in demo mode.
+"""This script is used to test the loading of a previously trained model.
+
+And run it in demo mode.
 It requires the project config file to be run before.
 
 Usage:  
@@ -20,17 +21,15 @@ import argparse
 import configparser
 import os
 
-import nkululeko.glob_conf as glob_conf
 from nkululeko.constants import VERSION
 from nkululeko.experiment import Experiment
+import nkululeko.glob_conf as glob_conf
 from nkululeko.utils.util import Util
 
 
 def main(src_dir):
-    parser = argparse.ArgumentParser(
-        description="Call the nkululeko DEMO framework.")
-    parser.add_argument("--config", default="exp.ini",
-                        help="The base configuration")
+    parser = argparse.ArgumentParser(description="Call the nkululeko DEMO framework.")
+    parser.add_argument("--config", default="exp.ini", help="The base configuration")
     parser.add_argument(
         "--file", help="A file that should be processed (16kHz mono wav)"
     )

nkululeko/demo_predictor.py

@@ -1,18 +1,19 @@
 # demo_predictor.py
 import os
 
-import audformat
-import audiofile
 import numpy as np
 import pandas as pd
 
+import audformat
+import audiofile
+
 import nkululeko.glob_conf as glob_conf
 from nkululeko.utils.util import Util
 
 
 class Demo_predictor:
     def __init__(self, model, file, is_list, feature_extractor, label_encoder, outfile):
-        """Constructor setting up name and configuration"""
+        """Constructor setting up name and configuration."""
         self.model = model
         self.feature_extractor = feature_extractor
         self.label_encoder = label_encoder

nkululeko/experiment.py

@@ -5,20 +5,22 @@ import pickle
 import random
 import time
 
-import audeer
-import audformat
 import numpy as np
 import pandas as pd
 from sklearn.preprocessing import LabelEncoder
 
-import nkululeko.glob_conf as glob_conf
+import audeer
+import audformat
+
 from nkululeko.data.dataset import Dataset
 from nkululeko.data.dataset_csv import Dataset_CSV
 from nkululeko.demo_predictor import Demo_predictor
 from nkululeko.feat_extract.feats_analyser import FeatureAnalyser
 from nkululeko.feature_extractor import FeatureExtractor
 from nkululeko.file_checker import FileChecker
-from nkululeko.filter_data import DataFilter, filter_min_dur
+from nkululeko.filter_data import DataFilter
+from nkululeko.filter_data import filter_min_dur
+import nkululeko.glob_conf as glob_conf
 from nkululeko.plots import Plots
 from nkululeko.reporting.report import Report
 from nkululeko.runmanager import Runmanager
@@ -101,6 +103,7 @@ class Experiment:
                 self.got_speaker = True
             self.datasets.update({d: data})
         self.target = self.util.config_val("DATA", "target", "emotion")
+        glob_conf.set_target(self.target)
         # print target via debug
         self.util.debug(f"target: {self.target}")
         # print keys/column
@@ -487,11 +490,7 @@ class Experiment:
         return df_ret
 
     def analyse_features(self, needs_feats):
-        """
-        Do a feature exploration
-
-        """
-
+        """Do a feature exploration."""
         plot_feats = eval(
             self.util.config_val("EXPL", "feature_distributions", "False")
         )
@@ -511,7 +510,7 @@ class Experiment:
                 f"unknown sample selection specifier {sample_selection}, should"
                 " be [all | train | test]"
             )
-
+        self.util.debug(f"sampling selection: {sample_selection}")
         if self.util.config_val("EXPL", "value_counts", False):
             self.plot_distribution(df_labels)
 
@@ -537,9 +536,13 @@ class Experiment:
                 f"unknown sample selection specifier {sample_selection}, should"
                 " be [all | train | test]"
             )
+        feat_analyser = FeatureAnalyser(sample_selection, df_labels, df_feats)
+        # check if SHAP features should be analysed
+        shap = eval(self.util.config_val("EXPL", "shap", "False"))
+        if shap:
+            feat_analyser.analyse_shap(self.runmgr.get_best_model())
 
         if plot_feats:
-            feat_analyser = FeatureAnalyser(sample_selection, df_labels, df_feats)
             feat_analyser.analyse()
 
         # check if a scatterplot should be done
@@ -692,7 +695,7 @@ class Experiment:
         if self.runmgr.modelrunner.model.is_ann():
             self.runmgr.modelrunner.model = None
             self.util.warn(
-                "Save experiment: Can't pickle the learning model so saving without it."
+                "Save experiment: Can't pickle the trained model so saving without it. (it should be stored anyway)"
             )
         try:
             f = open(filename, "wb")

nkululeko/explore.py

@@ -12,9 +12,9 @@ from nkululeko.utils.util import Util
 
 def main(src_dir):
     parser = argparse.ArgumentParser(
-        description="Call the nkululeko EXPLORE framework.")
-    parser.add_argument("--config", default="exp.ini",
-                        help="The base configuration")
+        description="Call the nkululeko EXPLORE framework."
+    )
+    parser.add_argument("--config", default="exp.ini", help="The base configuration")
     args = parser.parse_args()
     if args.config is not None:
         config_file = args.config
@@ -43,28 +43,34 @@ def main(src_dir):
         import warnings
 
         warnings.filterwarnings("ignore")
-
-    # load the data
-    expr.load_datasets()
-
-    # split into train and test
-    expr.fill_train_and_tests()
-    util.debug(
-        f"train shape : {expr.df_train.shape}, test shape:{expr.df_test.shape}")
-
-    plot_feats = eval(util.config_val(
-        "EXPL", "feature_distributions", "False"))
-    tsne = eval(util.config_val("EXPL", "tsne", "False"))
-    scatter = eval(util.config_val("EXPL", "scatter", "False"))
-    spotlight = eval(util.config_val("EXPL", "spotlight", "False"))
-    model_type = util.config_val("EXPL", "model", False)
-    plot_tree = eval(util.config_val("EXPL", "plot_tree", "False"))
     needs_feats = False
-    if plot_feats or tsne or scatter or model_type or plot_tree:
-        # these investigations need features to explore
-        expr.extract_feats()
+    try:
+        # load the experiment
+        expr.load(f"{util.get_save_name()}")
         needs_feats = True
-    # explore
+    except FileNotFoundError:
+        # first time: load the data
+        expr.load_datasets()
+
+        # split into train and test
+        expr.fill_train_and_tests()
+        util.debug(
+            f"train shape : {expr.df_train.shape}, test shape:{expr.df_test.shape}"
+        )
+
+        plot_feats = eval(util.config_val("EXPL", "feature_distributions", "False"))
+        tsne = eval(util.config_val("EXPL", "tsne", "False"))
+        scatter = eval(util.config_val("EXPL", "scatter", "False"))
+        spotlight = eval(util.config_val("EXPL", "spotlight", "False"))
+        shap = eval(util.config_val("EXPL", "shap", "False"))
+        model_type = util.config_val("EXPL", "model", False)
+        plot_tree = eval(util.config_val("EXPL", "plot_tree", "False"))
+        needs_feats = False
+        if plot_feats or tsne or scatter or model_type or plot_tree or shap:
+            # these investigations need features to explore
+            expr.extract_feats()
+            needs_feats = True
+        # explore
     expr.analyse_features(needs_feats)
     expr.store_report()
     print("DONE")

nkululeko/feat_extract/feats_analyser.py

@@ -40,6 +40,39 @@ class FeatureAnalyser:
             importance = model.feature_importances_
         return importance
 
+    def analyse_shap(self, model):
+        """Shap analysis.
+
+        Use the best model from a previous run and analyse feature importance with SHAP.
+        https://m.mage.ai/how-to-interpret-and-explain-your-machine-learning-models-using-shap-values-471c2635b78e.
+        """
+        import shap
+
+        name = "my_shap_values"
+        if not self.util.exist_pickle(name):
+
+            explainer = shap.Explainer(
+                model.predict_shap,
+                self.features,
+                output_names=glob_conf.labels,
+                algorithm="permutation",
+                npermutations=5,
+            )
+            self.util.debug("computing SHAP values...")
+            shap_values = explainer(self.features)
+            self.util.to_pickle(shap_values, name)
+        else:
+            shap_values = self.util.from_pickle(name)
+        plt.tight_layout()
+        shap.plots.bar(shap_values)
+        fig_dir = self.util.get_path("fig_dir") + "../"  # one up because of the runs
+        exp_name = self.util.get_exp_name(only_data=True)
+        format = self.util.config_val("PLOT", "format", "png")
+        filename = f"_SHAP_{model.name}"
+        filename = f"{fig_dir}{exp_name}{filename}.{format}"
+        plt.savefig(filename)
+        self.util.debug(f"plotted SHAP feature importance tp {filename}")
+
     def analyse(self):
         models = ast.literal_eval(self.util.config_val("EXPL", "model", "['log_reg']"))
         model_name = "_".join(models)

nkululeko/glob_conf.py

@@ -29,3 +29,8 @@ def set_report(report_obj):
 def set_labels(labels_obj):
     global labels
     labels = labels_obj
+
+
+def set_target(target_obj):
+    global target
+    target = target_obj

nkululeko/models/finetune_model.py

@@ -0,0 +1,181 @@
+import dataclasses
+import typing
+
+import torch
+import transformers
+from transformers.models.wav2vec2.modeling_wav2vec2 import (
+    Wav2Vec2PreTrainedModel,
+    Wav2Vec2Model,
+)
+
+
+class ConcordanceCorCoeff(torch.nn.Module):
+
+    def __init__(self):
+
+        super().__init__()
+
+        self.mean = torch.mean
+        self.var = torch.var
+        self.sum = torch.sum
+        self.sqrt = torch.sqrt
+        self.std = torch.std
+
+    def forward(self, prediction, ground_truth):
+
+        mean_gt = self.mean(ground_truth, 0)
+        mean_pred = self.mean(prediction, 0)
+        var_gt = self.var(ground_truth, 0)
+        var_pred = self.var(prediction, 0)
+        v_pred = prediction - mean_pred
+        v_gt = ground_truth - mean_gt
+        cor = self.sum(v_pred * v_gt) / (
+            self.sqrt(self.sum(v_pred**2)) * self.sqrt(self.sum(v_gt**2))
+        )
+        sd_gt = self.std(ground_truth)
+        sd_pred = self.std(prediction)
+        numerator = 2 * cor * sd_gt * sd_pred
+        denominator = var_gt + var_pred + (mean_gt - mean_pred) ** 2
+        ccc = numerator / denominator
+
+        return 1 - ccc
+
+
+@dataclasses.dataclass
+class ModelOutput(transformers.file_utils.ModelOutput):
+
+    logits_cat: torch.FloatTensor = None
+    hidden_states: typing.Tuple[torch.FloatTensor] = None
+    cnn_features: torch.FloatTensor = None
+
+
+class ModelHead(torch.nn.Module):
+
+    def __init__(self, config, num_labels):
+
+        super().__init__()
+
+        self.dense = torch.nn.Linear(config.hidden_size, config.hidden_size)
+        self.dropout = torch.nn.Dropout(config.final_dropout)
+        self.out_proj = torch.nn.Linear(config.hidden_size, num_labels)
+
+    def forward(self, features, **kwargs):
+
+        x = features
+        x = self.dropout(x)
+        x = self.dense(x)
+        x = torch.tanh(x)
+        x = self.dropout(x)
+        x = self.out_proj(x)
+
+        return x
+
+
+class Model(Wav2Vec2PreTrainedModel):
+
+    def __init__(self, config):
+
+        super().__init__(config)
+
+        self.wav2vec2 = Wav2Vec2Model(config)
+        self.cat = ModelHead(config, 2)
+        self.init_weights()
+
+    def freeze_feature_extractor(self):
+        self.wav2vec2.feature_extractor._freeze_parameters()
+
+    def pooling(
+        self,
+        hidden_states,
+        attention_mask,
+    ):
+
+        if attention_mask is None:  # For evaluation with batch_size==1
+            outputs = torch.mean(hidden_states, dim=1)
+        else:
+            attention_mask = self._get_feature_vector_attention_mask(
+                hidden_states.shape[1],
+                attention_mask,
+            )
+            hidden_states = hidden_states * torch.reshape(
+                attention_mask,
+                (-1, attention_mask.shape[-1], 1),
+            )
+            outputs = torch.sum(hidden_states, dim=1)
+            attention_sum = torch.sum(attention_mask, dim=1)
+            outputs = outputs / torch.reshape(attention_sum, (-1, 1))
+
+        return outputs
+
+    def forward(
+        self,
+        input_values,
+        attention_mask=None,
+        labels=None,
+        return_hidden=False,
+    ):
+
+        outputs = self.wav2vec2(
+            input_values,
+            attention_mask=attention_mask,
+        )
+
+        cnn_features = outputs.extract_features
+        hidden_states_framewise = outputs.last_hidden_state
+        hidden_states = self.pooling(
+            hidden_states_framewise,
+            attention_mask,
+        )
+        logits_cat = self.cat(hidden_states)
+
+        if not self.training:
+            logits_cat = torch.softmax(logits_cat, dim=1)
+
+        if return_hidden:
+
+            # make time last axis
+            cnn_features = torch.transpose(cnn_features, 1, 2)
+
+            return ModelOutput(
+                logits_cat=logits_cat,
+                hidden_states=hidden_states,
+                cnn_features=cnn_features,
+            )
+
+        else:
+
+            return ModelOutput(
+                logits_cat=logits_cat,
+            )
+
+
+class ModelWithPreProcessing(Model):
+
+    def __init__(self, config):
+        super().__init__(config)
+
+    def forward(
+        self,
+        input_values,
+    ):
+        # Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm():
+        # normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)
+
+        mean = input_values.mean()
+
+        # var = input_values.var()
+        # raises: onnxruntime.capi.onnxruntime_pybind11_state.NotImplemented: [ONNXRuntimeError] : 9 : NOT_IMPLEMENTED : Could not find an implementation for the node ReduceProd_3:ReduceProd(11)
+
+        var = torch.square(input_values - mean).mean()
+        input_values = (input_values - mean) / torch.sqrt(var + 1e-7)
+
+        output = super().forward(
+            input_values,
+            return_hidden=True,
+        )
+
+        return (
+            output.hidden_states,
+            output.logits_cat,
+            output.cnn_features,
+        )

nkululeko/models/model.py

@@ -20,6 +20,7 @@ class Model:
 
     def __init__(self, df_train, df_test, feats_train, feats_test):
         """Constructor taking the configuration and all dataframes."""
+        self.name = "undefined"
         self.df_train, self.df_test, self.feats_train, self.feats_test = (
             df_train,
             df_test,

nkululeko/models/model_bayes.py

@@ -12,3 +12,4 @@ class Bayes_model(Model):
     def __init__(self, df_train, df_test, feats_train, feats_test):
         super().__init__(df_train, df_test, feats_train, feats_test)
         self.clf = GaussianNB()  # set up the classifier
+        self.name = "bayes"

nkululeko/models/model_cnn.py

@@ -34,7 +34,8 @@ class CNN_model(Model):
         """Constructor taking the configuration and all dataframes"""
         super().__init__(df_train, df_test, feats_train, feats_test)
         super().set_model_type("ann")
-        self.target = glob_conf.config["DATA"]["target"]
+        self.name = "cnn"
+        self.target = glob_conf.target
         labels = glob_conf.labels
         self.class_num = len(labels)
         # set up loss criterion
@@ -86,8 +87,7 @@ class CNN_model(Model):
         train_set = self.Dataset_image(
             feats_train, df_train, self.target, transformations
         )
-        test_set = self.Dataset_image(
-            feats_test, df_test, self.target, transformations)
+        test_set = self.Dataset_image(feats_test, df_test, self.target, transformations)
         # Define data loaders
         self.trainloader = torch.utils.data.DataLoader(
             train_set,
@@ -140,8 +140,7 @@ class CNN_model(Model):
         losses = []
         for images, labels in self.trainloader:
             logits = self.model(images.to(self.device))
-            loss = self.criterion(logits, labels.to(
-                self.device, dtype=torch.int64))
+            loss = self.criterion(logits, labels.to(self.device, dtype=torch.int64))
             losses.append(loss.item())
             self.optimizer.zero_grad()
             loss.backward()
@@ -169,16 +168,14 @@ class CNN_model(Model):
 
         self.loss_eval = (np.asarray(losses)).mean()
         predictions = logits.argmax(dim=1)
-        uar = recall_score(
-            targets.numpy(), predictions.numpy(), average="macro")
+        uar = recall_score(targets.numpy(), predictions.numpy(), average="macro")
         return uar, targets, predictions
 
     def predict(self):
         _, truths, predictions = self.evaluate_model(
             self.model, self.testloader, self.device
         )
-        uar, _, _ = self.evaluate_model(
-            self.model, self.trainloader, self.device)
+        uar, _, _ = self.evaluate_model(self.model, self.trainloader, self.device)
         report = Reporter(truths, predictions, self.run, self.epoch)
         try:
             report.result.loss = self.loss

nkululeko/models/model_gmm.py

@@ -11,10 +11,9 @@ class GMM_model(Model):
 
     def __init__(self, df_train, df_test, feats_train, feats_test):
         super().__init__(df_train, df_test, feats_train, feats_test)
+        self.name = "gmm"
         n_components = int(self.util.config_val("MODEL", "GMM_components", "4"))
-        covariance_type = self.util.config_val(
-            "MODEL", "GMM_covariance_type", "full"
-        )
+        covariance_type = self.util.config_val("MODEL", "GMM_covariance_type", "full")
         self.clf = mixture.GaussianMixture(
             n_components=n_components, covariance_type=covariance_type
         )

nkululeko/models/model_knn.py

@@ -11,6 +11,7 @@ class KNN_model(Model):
 
     def __init__(self, df_train, df_test, feats_train, feats_test):
         super().__init__(df_train, df_test, feats_train, feats_test)
+        self.name = "knn"
         method = self.util.config_val("MODEL", "KNN_weights", "uniform")
         k = int(self.util.config_val("MODEL", "K_val", "5"))
         self.clf = KNeighborsClassifier(

nkululeko/models/model_knn_reg.py

@@ -11,6 +11,7 @@ class KNN_reg_model(Model):
 
     def __init__(self, df_train, df_test, feats_train, feats_test):
         super().__init__(df_train, df_test, feats_train, feats_test)
+        self.name = "knn_reg"
         method = self.util.config_val("MODEL", "KNN_weights", "uniform")
         k = int(self.util.config_val("MODEL", "K_val", "5"))
         self.clf = KNeighborsRegressor(

nkululeko/models/model_lin_reg.py

@@ -11,4 +11,5 @@ class Lin_reg_model(Model):
 
     def __init__(self, df_train, df_test, feats_train, feats_test):
         super().__init__(df_train, df_test, feats_train, feats_test)
+        self.name = "lin_reg"
         self.clf = LinearRegression()  # set up the classifier

nkululeko/models/model_mlp.py

@@ -1,4 +1,6 @@
 # model_mlp.py
+import pandas as pd
+
 from nkululeko.utils.util import Util
 import nkululeko.glob_conf as glob_conf
 from nkululeko.models.model import Model
@@ -20,6 +22,7 @@ class MLP_model(Model):
         """Constructor taking the configuration and all dataframes"""
         super().__init__(df_train, df_test, feats_train, feats_test)
         super().set_model_type("ann")
+        self.name = "mlp"
         self.target = glob_conf.config["DATA"]["target"]
         labels = glob_conf.labels
         self.class_num = len(labels)
@@ -87,8 +90,7 @@ class MLP_model(Model):
         losses = []
         for features, labels in self.trainloader:
             logits = self.model(features.to(self.device))
-            loss = self.criterion(logits, labels.to(
-                self.device, dtype=torch.int64))
+            loss = self.criterion(logits, labels.to(self.device, dtype=torch.int64))
             losses.append(loss.item())
             self.optimizer.zero_grad()
             loss.backward()
@@ -116,16 +118,14 @@ class MLP_model(Model):
 
         self.loss_eval = (np.asarray(losses)).mean()
         predictions = logits.argmax(dim=1)
-        uar = recall_score(
-            targets.numpy(), predictions.numpy(), average="macro")
+        uar = recall_score(targets.numpy(), predictions.numpy(), average="macro")
         return uar, targets, predictions
 
     def predict(self):
         _, truths, predictions = self.evaluate_model(
             self.model, self.testloader, self.device
         )
-        uar, _, _ = self.evaluate_model(
-            self.model, self.trainloader, self.device)
+        uar, _, _ = self.evaluate_model(self.model, self.trainloader, self.device)
         report = Reporter(truths, predictions, self.run, self.epoch)
         try:
             report.result.loss = self.loss
@@ -176,8 +176,18 @@ class MLP_model(Model):
             x = x.squeeze(dim=1).float()
             return self.linear(x)
 
+    def predict_shap(self, features):
+        # predict outputs for all samples in SHAP format (pd. dataframe)
+        results = []
+        for index, row in features.iterrows():
+            feats = row.values
+            res_dict = self.predict_sample(feats)
+            class_key = max(res_dict, key=res_dict.get)
+            results.append(class_key)
+        return results
+
     def predict_sample(self, features):
-        """Predict one sample"""
+        """Predict one sample."""
         with torch.no_grad():
             features = torch.from_numpy(features)
             features = np.reshape(features, (-1, 1)).T