nkululeko 0.84.0__tar.gz → 0.85.0__tar.gz

{nkululeko-0.84.0 → nkululeko-0.85.0}/CHANGELOG.md

@@ -1,6 +1,14 @@
 Changelog
 =========
 
+Version 0.85.0
+--------------
+* first version with finetuning wav2vec2 layers
+
+Version 0.84.1
+--------------
+* made resample independent of config file
+
 Version 0.84.0
 --------------
 * added SHAP analysis

{nkululeko-0.84.0/nkululeko.egg-info → nkululeko-0.85.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.84.0
+Version: 0.85.0
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -333,6 +333,14 @@ F. Burkhardt, Johannes Wagner, Hagen Wierstorf, Florian Eyben and Björn Schulle
 Changelog
 =========
 
+Version 0.85.0
+--------------
+* first version with finetuning wav2vec2 layers
+
+Version 0.84.1
+--------------
+* made resample independent of config file
+
 Version 0.84.0
 --------------
 * added SHAP analysis

{nkululeko-0.84.0 → nkululeko-0.85.0}/nkululeko/augmenting/resampler.py

@@ -12,16 +12,19 @@ from nkululeko.utils.util import Util
 
 
 class Resampler:
-    def __init__(self, df, not_testing=True):
+    def __init__(self, df, replace, not_testing=True):
         self.SAMPLING_RATE = 16000
         self.df = df
         self.util = Util("resampler", has_config=not_testing)
         self.util.warn(f"all files might be resampled to {self.SAMPLING_RATE}")
         self.not_testing = not_testing
+        self.replace = eval(self.util.config_val(
+            "RESAMPLE", "replace", "False")) if not not_testing else replace
 
     def resample(self):
         files = self.df.index.get_level_values(0).values
-        replace = eval(self.util.config_val("RESAMPLE", "replace", "False"))
+        # replace = eval(self.util.config_val("RESAMPLE", "replace", "False"))
+        replace = self.replace
         if self.not_testing:
             store = self.util.get_path("store")
         else:
@@ -42,7 +45,8 @@ class Resampler:
                 continue
             if org_sr != self.SAMPLING_RATE:
                 self.util.debug(f"resampling {f} (sr = {org_sr})")
-                resampler = torchaudio.transforms.Resample(org_sr, self.SAMPLING_RATE)
+                resampler = torchaudio.transforms.Resample(
+                    org_sr, self.SAMPLING_RATE)
                 signal = resampler(signal)
                 if replace:
                     torchaudio.save(
@@ -59,7 +63,8 @@ class Resampler:
             self.df = self.df.set_index(
                 self.df.index.set_levels(new_files, level="file")
             )
-            target_file = self.util.config_val("RESAMPLE", "target", "resampled.csv")
+            target_file = self.util.config_val(
+                "RESAMPLE", "target", "resampled.csv")
             # remove encoded labels
             target = self.util.config_val("DATA", "target", "emotion")
             if "class_label" in self.df.columns:

{nkululeko-0.84.0 → nkululeko-0.85.0}/nkululeko/constants.py

@@ -1,2 +1,2 @@
-VERSION="0.84.0"
+VERSION="0.85.0"
 SAMPLING_RATE = 16000

{nkululeko-0.84.0 → nkululeko-0.85.0}/nkululeko/experiment.py

@@ -340,7 +340,12 @@ class Experiment:
         df_train, df_test = self.df_train, self.df_test
         feats_name = "_".join(ast.literal_eval(glob_conf.config["DATA"]["databases"]))
         self.feats_test, self.feats_train = pd.DataFrame(), pd.DataFrame()
-        feats_types = self.util.config_val_list("FEATS", "type", ["os"])
+        feats_types = self.util.config_val_list("FEATS", "type", [])
+        # for some models no features are needed
+        if len(feats_types) == 0:
+            self.util.debug("no feature extractor specified.")
+            self.feats_train, self.feats_test = pd.DataFrame(), pd.DataFrame()
+            return
         self.feature_extractor = FeatureExtractor(
             df_train, feats_types, feats_name, "train"
         )

{nkululeko-0.84.0 → nkululeko-0.85.0}/nkululeko/feat_extract/feats_whisper.py

@@ -32,22 +32,19 @@ class Whisper(Featureset):
         model_name = f"openai/{self.feat_type}"
         self.model = WhisperModel.from_pretrained(model_name).to(self.device)
         print(f"intialized Whisper model on {self.device}")
-        self.feature_extractor = AutoFeatureExtractor.from_pretrained(
-            model_name)
+        self.feature_extractor = AutoFeatureExtractor.from_pretrained(model_name)
         self.model_initialized = True
 
     def extract(self):
         """Extract the features or load them from disk if present."""
         store = self.util.get_path("store")
         storage = f"{store}{self.name}.pkl"
-        extract = self.util.config_val(
-            "FEATS", "needs_feature_extraction", False)
+        extract = self.util.config_val("FEATS", "needs_feature_extraction", False)
         no_reuse = eval(self.util.config_val("FEATS", "no_reuse", "False"))
         if extract or no_reuse or not os.path.isfile(storage):
             if not self.model_initialized:
                 self.init_model()
-            self.util.debug(
-                "extracting whisper embeddings, this might take a while...")
+            self.util.debug("extracting whisper embeddings, this might take a while...")
             emb_series = []
             for (file, start, end), _ in audeer.progress_bar(
                 self.data_df.iterrows(),

{nkululeko-0.84.0 → nkululeko-0.85.0}/nkululeko/modelrunner.py

@@ -47,16 +47,12 @@ class Modelrunner:
             highest = 0
         else:
             highest = 100000
-        # for all epochs
-        for epoch in range(epoch_num):
-            if only_test:
-                self.model.load(self.run, epoch)
-                self.util.debug(f"reusing model: {self.model.store_path}")
-                self.model.reset_test(self.df_test, self.feats_test)
-            else:
-                self.model.set_id(self.run, epoch)
-                self.model.train()
+        if self.model.model_type == "finetuned":
+            # epochs are handled by Huggingface API
+            self.model.train()
             report = self.model.predict()
+            # todo: findout the best epoch
+            epoch = epoch_num
             report.set_id(self.run, epoch)
             plot_name = self.util.get_plot_name() + f"_{self.run}_{epoch:03d}_cnf"
             reports.append(report)
@@ -67,32 +63,53 @@ class Modelrunner:
             if plot_epochs:
                 self.util.debug(f"plotting conf matrix to {plot_name}")
                 report.plot_confmatrix(plot_name, epoch)
-            store_models = self.util.config_val("EXP", "save", False)
-            plot_best_model = self.util.config_val("PLOT", "best_model", False)
-            if (store_models or plot_best_model) and (
-                not only_test
-            ):  # in any case the model needs to be stored to disk.
-                self.model.store()
-            if patience:
-                patience = int(patience)
-                result = report.result.get_result()
-                if self.util.high_is_good():
-                    if result > highest:
-                        highest = result
-                        patience_counter = 0
-                    else:
-                        patience_counter += 1
+        else:
+            # for all epochs
+            for epoch in range(epoch_num):
+                if only_test:
+                    self.model.load(self.run, epoch)
+                    self.util.debug(f"reusing model: {self.model.store_path}")
+                    self.model.reset_test(self.df_test, self.feats_test)
                 else:
-                    if result < highest:
-                        highest = result
-                        patience_counter = 0
+                    self.model.set_id(self.run, epoch)
+                    self.model.train()
+                report = self.model.predict()
+                report.set_id(self.run, epoch)
+                plot_name = self.util.get_plot_name() + f"_{self.run}_{epoch:03d}_cnf"
+                reports.append(report)
+                self.util.debug(
+                    f"run: {self.run} epoch: {epoch}: result: "
+                    f"{reports[-1].get_result().get_test_result()}"
+                )
+                if plot_epochs:
+                    self.util.debug(f"plotting conf matrix to {plot_name}")
+                    report.plot_confmatrix(plot_name, epoch)
+                store_models = self.util.config_val("EXP", "save", False)
+                plot_best_model = self.util.config_val("PLOT", "best_model", False)
+                if (store_models or plot_best_model) and (
+                    not only_test
+                ):  # in any case the model needs to be stored to disk.
+                    self.model.store()
+                if patience:
+                    patience = int(patience)
+                    result = report.result.get_result()
+                    if self.util.high_is_good():
+                        if result > highest:
+                            highest = result
+                            patience_counter = 0
+                        else:
+                            patience_counter += 1
                     else:
-                        patience_counter += 1
-                if patience_counter >= patience:
-                    self.util.debug(
-                        f"reached patience ({str(patience)}): early stopping"
-                    )
-                    break
+                        if result < highest:
+                            highest = result
+                            patience_counter = 0
+                        else:
+                            patience_counter += 1
+                    if patience_counter >= patience:
+                        self.util.debug(
+                            f"reached patience ({str(patience)}): early stopping"
+                        )
+                        break
 
         if not plot_epochs:
             # Do at least one confusion matrix plot
@@ -133,6 +150,12 @@ class Modelrunner:
             self.model = Bayes_model(
                 self.df_train, self.df_test, self.feats_train, self.feats_test
             )
+        elif model_type == "finetune":
+            from nkululeko.models.model_tuned import Pretrained_model
+
+            self.model = Pretrained_model(
+                self.df_train, self.df_test, self.feats_train, self.feats_test
+            )
         elif model_type == "gmm":
             from nkululeko.models.model_gmm import GMM_model
 

nkululeko-0.85.0/nkululeko/models/finetune_model.py

@@ -0,0 +1,190 @@
+"""
+Code based on @jwagner
+"""
+
+import dataclasses
+import typing
+
+import torch
+import transformers
+from transformers.models.wav2vec2.modeling_wav2vec2 import (
+    Wav2Vec2PreTrainedModel,
+    Wav2Vec2Model,
+)
+
+
+class ConcordanceCorCoeff(torch.nn.Module):
+
+    def __init__(self):
+
+        super().__init__()
+
+        self.mean = torch.mean
+        self.var = torch.var
+        self.sum = torch.sum
+        self.sqrt = torch.sqrt
+        self.std = torch.std
+
+    def forward(self, prediction, ground_truth):
+
+        mean_gt = self.mean(ground_truth, 0)
+        mean_pred = self.mean(prediction, 0)
+        var_gt = self.var(ground_truth, 0)
+        var_pred = self.var(prediction, 0)
+        v_pred = prediction - mean_pred
+        v_gt = ground_truth - mean_gt
+        cor = self.sum(v_pred * v_gt) / (
+            self.sqrt(self.sum(v_pred**2)) * self.sqrt(self.sum(v_gt**2))
+        )
+        sd_gt = self.std(ground_truth)
+        sd_pred = self.std(prediction)
+        numerator = 2 * cor * sd_gt * sd_pred
+        denominator = var_gt + var_pred + (mean_gt - mean_pred) ** 2
+        ccc = numerator / denominator
+
+        return 1 - ccc
+
+
+@dataclasses.dataclass
+class ModelOutput(transformers.file_utils.ModelOutput):
+
+    logits_cat: torch.FloatTensor = None
+    hidden_states: typing.Tuple[torch.FloatTensor] = None
+    cnn_features: torch.FloatTensor = None
+
+
+class ModelHead(torch.nn.Module):
+
+    def __init__(self, config, num_labels):
+
+        super().__init__()
+
+        self.dense = torch.nn.Linear(config.hidden_size, config.hidden_size)
+        self.dropout = torch.nn.Dropout(config.final_dropout)
+        self.out_proj = torch.nn.Linear(config.hidden_size, num_labels)
+
+    def forward(self, features, **kwargs):
+
+        x = features
+        x = self.dropout(x)
+        x = self.dense(x)
+        x = torch.tanh(x)
+        x = self.dropout(x)
+        x = self.out_proj(x)
+
+        return x
+
+
+class Model(Wav2Vec2PreTrainedModel):
+
+    def __init__(self, config):
+
+        super().__init__(config)
+
+        self.wav2vec2 = Wav2Vec2Model(config)
+        self.cat = ModelHead(config, 2)
+        self.init_weights()
+
+    def freeze_feature_extractor(self):
+        self.wav2vec2.feature_extractor._freeze_parameters()
+
+    def pooling(
+        self,
+        hidden_states,
+        attention_mask,
+    ):
+
+        if attention_mask is None:  # For evaluation with batch_size==1
+            outputs = torch.mean(hidden_states, dim=1)
+        else:
+            attention_mask = self._get_feature_vector_attention_mask(
+                hidden_states.shape[1],
+                attention_mask,
+            )
+            hidden_states = hidden_states * torch.reshape(
+                attention_mask,
+                (-1, attention_mask.shape[-1], 1),
+            )
+            outputs = torch.sum(hidden_states, dim=1)
+            attention_sum = torch.sum(attention_mask, dim=1)
+            outputs = outputs / torch.reshape(attention_sum, (-1, 1))
+
+        return outputs
+
+    def forward(
+        self,
+        input_values,
+        attention_mask=None,
+        labels=None,
+        return_hidden=False,
+    ):
+
+        outputs = self.wav2vec2(
+            input_values,
+            attention_mask=attention_mask,
+        )
+
+        cnn_features = outputs.extract_features
+        hidden_states_framewise = outputs.last_hidden_state
+        hidden_states = self.pooling(
+            hidden_states_framewise,
+            attention_mask,
+        )
+        logits_cat = self.cat(hidden_states)
+
+        if not self.training:
+            logits_cat = torch.softmax(logits_cat, dim=1)
+
+        if return_hidden:
+
+            # make time last axis
+            cnn_features = torch.transpose(cnn_features, 1, 2)
+
+            return ModelOutput(
+                logits_cat=logits_cat,
+                hidden_states=hidden_states,
+                cnn_features=cnn_features,
+            )
+
+        else:
+
+            return ModelOutput(
+                logits_cat=logits_cat,
+            )
+
+    def predict(self, signal):
+        result = self(torch.from_numpy(signal))
+        result = result[0].detach().numpy()[0]
+        return result
+
+
+class ModelWithPreProcessing(Model):
+
+    def __init__(self, config):
+        super().__init__(config)
+
+    def forward(
+        self,
+        input_values,
+    ):
+        # Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm():
+        # normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)
+
+        mean = input_values.mean()
+
+        # var = input_values.var()
+        # raises: onnxruntime.capi.onnxruntime_pybind11_state.NotImplemented: [ONNXRuntimeError] : 9 : NOT_IMPLEMENTED : Could not find an implementation for the node ReduceProd_3:ReduceProd(11)
+
+        var = torch.square(input_values - mean).mean()
+        input_values = (input_values - mean) / torch.sqrt(var + 1e-7)
+
+        output = super().forward(
+            input_values,
+            return_hidden=True,
+        )
+
+        return (
+            output.hidden_states,
+            output.logits_cat,
+            output.cnn_features,
+        )

{nkululeko-0.84.0 → nkululeko-0.85.0}/nkululeko/models/model.py

@@ -39,7 +39,7 @@ class Model:
         self.model_type = type
 
     def is_ann(self):
-        if self.model_type == "ann":
+        if (self.model_type == "ann") or (self.model_type == "finetuned"):
             return True
         else:
             return False