nkululeko 0.85.2__py3-none-any.whl → 0.86.0__py3-none-any.whl

nkululeko/constants.py

@@ -1,2 +1,2 @@
-VERSION="0.85.2"
+VERSION="0.86.0"
 SAMPLING_RATE = 16000

nkululeko/experiment.py

@@ -30,15 +30,14 @@ from nkululeko.utils.util import Util
 
 
 class Experiment:
-    """Main class specifying an experiment"""
+    """Main class specifying an experiment."""
 
     def __init__(self, config_obj):
-        """
-        Parameters
-        ----------
-        config_obj : a config parser object that sets the experiment parameters and being set as a global object.
-        """
+        """Constructor.
 
+        Args:
+            - config_obj : a config parser object that sets the experiment parameters and being set as a global object.
+        """
         self.set_globals(config_obj)
         self.name = glob_conf.config["EXP"]["name"]
         self.root = os.path.join(glob_conf.config["EXP"]["root"], "")
@@ -109,15 +108,13 @@ class Experiment:
         # print keys/column
         dbs = ",".join(list(self.datasets.keys()))
         labels = self.util.config_val("DATA", "labels", False)
-        auto_labels = list(
-            next(iter(self.datasets.values())).df[self.target].unique()
-        )
+        auto_labels = list(next(iter(self.datasets.values())).df[self.target].unique())
         if labels:
             self.labels = ast.literal_eval(labels)
             self.util.debug(f"Target labels (from config): {labels}")
         else:
             self.labels = auto_labels
-        self.util.debug(f"Target labels (from database): {auto_labels}")
+            self.util.debug(f"Target labels (from database): {auto_labels}")
         glob_conf.set_labels(self.labels)
         self.util.debug(f"loaded databases {dbs}")
 
@@ -160,8 +157,7 @@ class Experiment:
                 data.split()
                 data.prepare_labels()
                 self.df_test = pd.concat(
-                    [self.df_test, self.util.make_segmented_index(
-                        data.df_test)]
+                    [self.df_test, self.util.make_segmented_index(data.df_test)]
                 )
                 self.df_test.is_labeled = data.is_labeled
             self.df_test.got_gender = self.got_gender
@@ -262,8 +258,7 @@ class Experiment:
                     test_cats = self.df_test[self.target].unique()
                 else:
                     # if there is no target, copy a dummy label
-                    self.df_test = self._add_random_target(
-                        self.df_test).astype("str")
+                    self.df_test = self._add_random_target(self.df_test).astype("str")
                 train_cats = self.df_train[self.target].unique()
                 # print(f"df_train: {pd.DataFrame(self.df_train[self.target])}")
                 # print(f"train_cats with target {self.target}: {train_cats}")
@@ -271,8 +266,7 @@ class Experiment:
                 if type(test_cats) == np.ndarray:
                     self.util.debug(f"Categories test (nd.array): {test_cats}")
                 else:
-                    self.util.debug(
-                        f"Categories test (list): {list(test_cats)}")
+                    self.util.debug(f"Categories test (list): {list(test_cats)}")
             if type(train_cats) == np.ndarray:
                 self.util.debug(f"Categories train (nd.array): {train_cats}")
             else:
@@ -295,8 +289,7 @@ class Experiment:
 
         target_factor = self.util.config_val("DATA", "target_divide_by", False)
         if target_factor:
-            self.df_test[self.target] = self.df_test[self.target] / \
-                float(target_factor)
+            self.df_test[self.target] = self.df_test[self.target] / float(target_factor)
             self.df_train[self.target] = self.df_train[self.target] / float(
                 target_factor
             )
@@ -319,16 +312,14 @@ class Experiment:
     def plot_distribution(self, df_labels):
         """Plot the distribution of samples and speaker per target class and biological sex"""
         plot = Plots()
-        sample_selection = self.util.config_val(
-            "EXPL", "sample_selection", "all")
+        sample_selection = self.util.config_val("EXPL", "sample_selection", "all")
         plot.plot_distributions(df_labels)
         if self.got_speaker:
             plot.plot_distributions_speaker(df_labels)
 
     def extract_test_feats(self):
         self.feats_test = pd.DataFrame()
-        feats_name = "_".join(ast.literal_eval(
-            glob_conf.config["DATA"]["tests"]))
+        feats_name = "_".join(ast.literal_eval(glob_conf.config["DATA"]["tests"]))
         feats_types = self.util.config_val_list("FEATS", "type", ["os"])
         self.feature_extractor = FeatureExtractor(
             self.df_test, feats_types, feats_name, "test"
@@ -345,8 +336,7 @@ class Experiment:
 
         """
         df_train, df_test = self.df_train, self.df_test
-        feats_name = "_".join(ast.literal_eval(
-            glob_conf.config["DATA"]["databases"]))
+        feats_name = "_".join(ast.literal_eval(glob_conf.config["DATA"]["databases"]))
         self.feats_test, self.feats_train = pd.DataFrame(), pd.DataFrame()
         feats_types = self.util.config_val_list("FEATS", "type", [])
         # for some models no features are needed
@@ -380,20 +370,22 @@ class Experiment:
                 f"test feats ({self.feats_test.shape[0]}) != test labels"
                 f" ({self.df_test.shape[0]})"
             )
-            self.df_test = self.df_test[self.df_test.index.isin(
-                self.feats_test.index)]
-            self.util.warn(f"mew test labels shape: {self.df_test.shape[0]}")
+            self.df_test = self.df_test[self.df_test.index.isin(self.feats_test.index)]
+            self.util.warn(f"new test labels shape: {self.df_test.shape[0]}")
 
         self._check_scale()
+        # store = self.util.get_path("store")
+        # store_format = self.util.config_val("FEATS", "store_format", "pkl")
+        # storage = f"{store}test_feats.{store_format}"
+        # self.util.write_store(self.feats_test, storage, store_format)
+        # storage = f"{store}train_feats.{store_format}"
+        # self.util.write_store(self.feats_train, storage, store_format)
 
     def augment(self):
-        """
-        Augment the selected samples
-        """
+        """Augment the selected samples."""
         from nkululeko.augmenting.augmenter import Augmenter
 
-        sample_selection = self.util.config_val(
-            "AUGMENT", "sample_selection", "all")
+        sample_selection = self.util.config_val("AUGMENT", "sample_selection", "all")
         if sample_selection == "all":
             df = pd.concat([self.df_train, self.df_test])
         elif sample_selection == "train":
@@ -488,8 +480,7 @@ class Experiment:
         """
         from nkululeko.augmenting.randomsplicer import Randomsplicer
 
-        sample_selection = self.util.config_val(
-            "AUGMENT", "sample_selection", "all")
+        sample_selection = self.util.config_val("AUGMENT", "sample_selection", "all")
         if sample_selection == "all":
             df = pd.concat([self.df_train, self.df_test])
         elif sample_selection == "train":
@@ -510,8 +501,7 @@ class Experiment:
         plot_feats = eval(
             self.util.config_val("EXPL", "feature_distributions", "False")
         )
-        sample_selection = self.util.config_val(
-            "EXPL", "sample_selection", "all")
+        sample_selection = self.util.config_val("EXPL", "sample_selection", "all")
         # get the data labels
         if sample_selection == "all":
             df_labels = pd.concat([self.df_train, self.df_test])
@@ -574,8 +564,7 @@ class Experiment:
             for scat_target in scat_targets:
                 if self.util.is_categorical(df_labels[scat_target]):
                     for scatter in scatters:
-                        plots.scatter_plot(
-                            df_feats, df_labels, scat_target, scatter)
+                        plots.scatter_plot(df_feats, df_labels, scat_target, scatter)
                 else:
                     self.util.debug(
                         f"{self.name}: binning continuous variable to categories"
@@ -590,6 +579,8 @@ class Experiment:
                         )
 
     def _check_scale(self):
+        self.util.save_to_store(self.feats_train, "feats_train")
+        self.util.save_to_store(self.feats_test, "feats_test")
         scale_feats = self.util.config_val("FEATS", "scale", False)
         # print the scale
         self.util.debug(f"scaler: {scale_feats}")
@@ -664,8 +655,7 @@ class Experiment:
         preds = best.preds
         speakers = self.df_test.speaker.values
         print(f"{len(truths)} {len(preds)} {len(speakers) }")
-        df = pd.DataFrame(
-            data={"truth": truths, "pred": preds, "speaker": speakers})
+        df = pd.DataFrame(data={"truth": truths, "pred": preds, "speaker": speakers})
         plot_name = "result_combined_per_speaker"
         self.util.debug(
             f"plotting speaker combination ({function}) confusion matrix to"

nkululeko/feat_extract/feats_opensmile.py

@@ -65,28 +65,28 @@ class Opensmileset(Featureset):
         feats = smile.process_signal(signal, sr)
         return feats.to_numpy()
 
-    def filter(self):
-        # use only the features that are indexed in the target dataframes
-        self.df = self.df[self.df.index.isin(self.data_df.index)]
-        try:
-            # use only some features
-            selected_features = ast.literal_eval(
-                glob_conf.config["FEATS"]["os.features"]
-            )
-            self.util.debug(f"selecting features from opensmile: {selected_features}")
-            sel_feats_df = pd.DataFrame()
-            hit = False
-            for feat in selected_features:
-                try:
-                    sel_feats_df[feat] = self.df[feat]
-                    hit = True
-                except KeyError:
-                    pass
-            if hit:
-                self.df = sel_feats_df
-                self.util.debug(
-                    "new feats shape after selecting opensmile features:"
-                    f" {self.df.shape}"
-                )
-        except KeyError:
-            pass
+    # def filter(self):
+    #     # use only the features that are indexed in the target dataframes
+    #     self.df = self.df[self.df.index.isin(self.data_df.index)]
+    #     try:
+    #         # use only some features
+    #         selected_features = ast.literal_eval(
+    #             glob_conf.config["FEATS"]["os.features"]
+    #         )
+    #         self.util.debug(f"selecting features from opensmile: {selected_features}")
+    #         sel_feats_df = pd.DataFrame()
+    #         hit = False
+    #         for feat in selected_features:
+    #             try:
+    #                 sel_feats_df[feat] = self.df[feat]
+    #                 hit = True
+    #             except KeyError:
+    #                 pass
+    #         if hit:
+    #             self.df = sel_feats_df
+    #             self.util.debug(
+    #                 "new feats shape after selecting opensmile features:"
+    #                 f" {self.df.shape}"
+    #             )
+    #     except KeyError:
+    #         pass

nkululeko/feat_extract/featureset.py

@@ -15,7 +15,7 @@ class Featureset:
         self.name = name
         self.data_df = data_df
         self.util = Util("featureset")
-        self.feats_types = feats_type
+        self.feats_type = feats_type
 
     def extract(self):
         pass
@@ -25,8 +25,7 @@ class Featureset:
         self.df = self.df[self.df.index.isin(self.data_df.index)]
         try:
             # use only some features
-            selected_features = ast.literal_eval(
-                glob_conf.config["FEATS"]["features"])
+            selected_features = ast.literal_eval(glob_conf.config["FEATS"]["features"])
             self.util.debug(f"selecting features: {selected_features}")
             sel_feats_df = pd.DataFrame()
             hit = False
@@ -35,11 +34,12 @@ class Featureset:
                     sel_feats_df[feat] = self.df[feat]
                     hit = True
                 except KeyError:
+                    self.util.warn(f"non existent feature in {self.feats_type}: {feat}")
                     pass
             if hit:
                 self.df = sel_feats_df
                 self.util.debug(
-                    f"new feats shape after selecting features: {self.df.shape}"
+                    f"new feats shape after selecting features for {self.feats_type}: {self.df.shape}"
                 )
         except KeyError:
             pass

nkululeko/models/model_tuned.py

@@ -1,6 +1,4 @@
-"""
-Code based on @jwagner.
-"""
+"""Code based on @jwagner."""
 
 import dataclasses
 import json
@@ -27,8 +25,6 @@ from nkululeko.reporting.reporter import Reporter
 
 class TunedModel(BaseModel):
 
-    is_classifier = True
-
     def __init__(self, df_train, df_test, feats_train, feats_test):
         """Constructor taking the configuration and all dataframes."""
         super().__init__(df_train, df_test, feats_train, feats_test)
@@ -37,25 +33,47 @@ class TunedModel(BaseModel):
         self.target = glob_conf.config["DATA"]["target"]
         labels = glob_conf.labels
         self.class_num = len(labels)
-        device = self.util.config_val("MODEL", "device", "cpu")
-        self.batch_size = int(self.util.config_val("MODEL", "batch_size", "8"))
-        if device != "cpu":
-            self.util.debug(f"running on device {device}")
+        device = self.util.config_val("MODEL", "device", False)
+        if not device:
+            self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        else:
+            self.device = device
+        if self.device != "cpu":
             os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-            os.environ["CUDA_VISIBLE_DEVICES"] = device
+            os.environ["CUDA_VISIBLE_DEVICES"] = self.device
+        self.util.debug(f"running on device {self.device}")
+        self.is_classifier = self.util.exp_is_classification()
+        if self.is_classifier:
+            self.measure = "uar"
+        else:
+            self.measure = self.util.config_val("MODEL", "measure", "ccc")
+        self.util.debug(f"evaluation metrics: {self.measure}")
+        self.batch_size = int(self.util.config_val("MODEL", "batch_size", "8"))
+        self.util.debug(f"batch size: {self.batch_size}")
+        self.learning_rate = float(
+            self.util.config_val("MODEL", "learning_rate", 0.0001)
+        )
         self.df_train, self.df_test = df_train, df_test
         self.epoch_num = int(self.util.config_val("EXP", "epochs", 1))
-
+        drop = self.util.config_val("MODEL", "drop", False)
+        self.drop = 0.1
+        if drop:
+            self.drop = float(drop)
+        self.util.debug(f"init: training with dropout: {self.drop}")
         self._init_model()
 
     def _init_model(self):
         model_path = "facebook/wav2vec2-large-robust-ft-swbd-300h"
+        pretrained_model = self.util.config_val("MODEL", "pretrained_model", model_path)
         self.num_layers = None
         self.sampling_rate = 16000
         self.max_duration_sec = 8.0
         self.accumulation_steps = 4
-        # create dataset
 
+        # print finetuning information via debug
+        self.util.debug(f"Finetuning from model: {pretrained_model}")
+
+        # create dataset
         dataset = {}
         target_name = glob_conf.target
         data_sources = {
@@ -76,22 +94,32 @@ class TunedModel(BaseModel):
         self.dataset = datasets.DatasetDict(dataset)
 
         # load pre-trained model
-        le = glob_conf.label_encoder
-        mapping = dict(zip(le.classes_, range(len(le.classes_))))
-        target_mapping = {k: int(v) for k, v in mapping.items()}
-        target_mapping_reverse = {value: key for key, value in target_mapping.items()}
-
-        self.config = transformers.AutoConfig.from_pretrained(
-            model_path,
-            num_labels=len(target_mapping),
-            label2id=target_mapping,
-            id2label=target_mapping_reverse,
-            finetuning_task=target_name,
-        )
+        if self.is_classifier:
+            le = glob_conf.label_encoder
+            mapping = dict(zip(le.classes_, range(len(le.classes_))))
+            target_mapping = {k: int(v) for k, v in mapping.items()}
+            target_mapping_reverse = {
+                value: key for key, value in target_mapping.items()
+            }
+            self.config = transformers.AutoConfig.from_pretrained(
+                model_path,
+                num_labels=len(target_mapping),
+                label2id=target_mapping,
+                id2label=target_mapping_reverse,
+                finetuning_task=target_name,
+            )
+        else:
+            self.config = transformers.AutoConfig.from_pretrained(
+                model_path,
+                num_labels=1,
+                finetuning_task=target_name,
+            )
         if self.num_layers is not None:
             self.config.num_hidden_layers = self.num_layers
+        self.config.final_dropout = self.drop
         setattr(self.config, "sampling_rate", self.sampling_rate)
         setattr(self.config, "data", self.util.get_data_name())
+        setattr(self.config, "is_classifier", self.is_classifier)
 
         vocab_dict = {}
         with open("vocab.json", "w") as vocab_file:
@@ -113,7 +141,7 @@ class TunedModel(BaseModel):
         assert self.processor.feature_extractor.sampling_rate == self.sampling_rate
 
         self.model = Model.from_pretrained(
-            model_path,
+            pretrained_model,
             config=self.config,
         )
         self.model.freeze_feature_extractor()
@@ -170,7 +198,7 @@ class TunedModel(BaseModel):
             return_tensors="pt",
         )
 
-        batch["labels"] = torch.tensor(targets)
+        batch["labels"] = torch.Tensor(targets)
 
         return batch
 
@@ -180,14 +208,25 @@ class TunedModel(BaseModel):
             "UAR": audmetric.unweighted_average_recall,
             "ACC": audmetric.accuracy,
         }
+        metrics_reg = {
+            "PCC": audmetric.pearson_cc,
+            "CCC": audmetric.concordance_cc,
+            "MSE": audmetric.mean_squared_error,
+            "MAE": audmetric.mean_absolute_error,
+        }
 
         # truth = p.label_ids[:, 0].astype(int)
         truth = p.label_ids
         preds = p.predictions
         preds = np.argmax(preds, axis=1)
         scores = {}
-        for name, metric in metrics.items():
-            scores[f"{name}"] = metric(truth, preds)
+        if self.is_classifier:
+            for name, metric in metrics.items():
+                scores[f"{name}"] = metric(truth, preds)
+        else:
+            for name, metric in metrics_reg.items():
+                scores[f"{name}"] = metric(truth, preds)
+
         return scores
 
     def train(self):
@@ -203,23 +242,24 @@ class TunedModel(BaseModel):
             return
         targets = pd.DataFrame(self.dataset["train"]["targets"])
         counts = targets[0].value_counts().sort_index()
-        train_weights = 1 / counts
-        train_weights /= train_weights.sum()
-        self.util.debug("train weights: {train_weights}")
-        criterion = torch.nn.CrossEntropyLoss(
-            weight=torch.Tensor(train_weights).to("cuda"),
-        )
-        # criterion = torch.nn.CrossEntropyLoss()
 
-        class Trainer(transformers.Trainer):
+        if self.is_classifier:
+            train_weights = 1 / counts
+            train_weights /= train_weights.sum()
+            self.util.debug(f"train weights: {train_weights}")
+            criterion = torch.nn.CrossEntropyLoss(
+                weight=torch.Tensor(train_weights).to("cuda"),
+            )
+        else:
+            criterion = ConcordanceCorCoeff()
 
+        class Trainer(transformers.Trainer):
             def compute_loss(
                 self,
                 model,
                 inputs,
                 return_outputs=False,
             ):
-
                 targets = inputs.pop("labels").squeeze()
                 targets = targets.type(torch.long)
 
@@ -236,7 +276,8 @@ class TunedModel(BaseModel):
             // 5
         )
         num_steps = max(1, num_steps)
-        # print(num_steps)
+
+        metrics_for_best_model = self.measure.upper()
 
         training_args = transformers.TrainingArguments(
             output_dir=model_root,
@@ -246,13 +287,14 @@ class TunedModel(BaseModel):
             gradient_accumulation_steps=self.accumulation_steps,
             evaluation_strategy="steps",
             num_train_epochs=self.epoch_num,
-            fp16=True,
+            fp16=self.device == "cuda",
             save_steps=num_steps,
             eval_steps=num_steps,
             logging_steps=num_steps,
-            learning_rate=1e-4,
+            logging_strategy="epoch",
+            learning_rate=self.learning_rate,
             save_total_limit=2,
-            metric_for_best_model="UAR",
+            metric_for_best_model=metrics_for_best_model,
             greater_is_better=True,
             load_best_model_at_end=True,
             remove_unused_columns=False,
@@ -271,6 +313,7 @@ class TunedModel(BaseModel):
         )
         trainer.train()
         trainer.save_model(self.torch_root)
+        self.util.debug(f"saved best model to {self.torch_root}")
         self.load(self.run, self.epoch)
 
     def get_predictions(self):
@@ -305,7 +348,7 @@ class TunedModel(BaseModel):
     def predict_sample(self, signal):
         """Predict one sample"""
         prediction = {}
-        if self.util.exp_is_classification():
+        if self.is_classifier:
             # get the class probabilities
             predictions = self.model.predict(signal)
             # pred = self.clf.predict(features)
@@ -337,8 +380,19 @@ class TunedModel(BaseModel):
 @dataclasses.dataclass
 class ModelOutput(transformers.file_utils.ModelOutput):
 
-    logits_cat: torch.FloatTensor = None
+    logits: torch.FloatTensor = None
+    hidden_states: typing.Tuple[torch.FloatTensor] = None
+    cnn_features: torch.FloatTensor = None
+
+
+@dataclasses.dataclass
+class ModelOutputReg(transformers.file_utils.ModelOutput):
+
+    logits: torch.FloatTensor
     hidden_states: typing.Tuple[torch.FloatTensor] = None
+    attentions: typing.Tuple[torch.FloatTensor] = None
+    logits_framewise: torch.FloatTensor = None
+    hidden_states_framewise: torch.FloatTensor = None
     cnn_features: torch.FloatTensor = None
 
 
@@ -368,10 +422,14 @@ class Model(Wav2Vec2PreTrainedModel):
 
     def __init__(self, config):
 
+        if not hasattr(config, "add_adapter"):
+            setattr(config, "add_adapter", False)
+
         super().__init__(config)
 
         self.wav2vec2 = Wav2Vec2Model(config)
-        self.cat = ModelHead(config)
+        self.head = ModelHead(config)
+        self.is_classifier = config.is_classifier
         self.init_weights()
 
     def freeze_feature_extractor(self):
@@ -407,39 +465,44 @@ class Model(Wav2Vec2PreTrainedModel):
         labels=None,
         return_hidden=False,
     ):
-
         outputs = self.wav2vec2(
             input_values,
             attention_mask=attention_mask,
         )
-
         cnn_features = outputs.extract_features
         hidden_states_framewise = outputs.last_hidden_state
         hidden_states = self.pooling(
             hidden_states_framewise,
             attention_mask,
         )
-        logits_cat = self.cat(hidden_states)
-
+        logits = self.head(hidden_states)
         if not self.training:
-            logits_cat = torch.softmax(logits_cat, dim=1)
+            logits = torch.softmax(logits, dim=1)
 
         if return_hidden:
-
             # make time last axis
             cnn_features = torch.transpose(cnn_features, 1, 2)
-
-            return ModelOutput(
-                logits_cat=logits_cat,
-                hidden_states=hidden_states,
-                cnn_features=cnn_features,
-            )
-
+            if self.is_classifier:
+                return ModelOutput(
+                    logits=logits,
+                    hidden_states=hidden_states,
+                    cnn_features=cnn_features,
+                )
+            else:
+                return ModelOutputReg(
+                    logits=logits,
+                    hidden_states=hidden_states,
+                    cnn_features=cnn_features,
+                )
         else:
-
-            return ModelOutput(
-                logits_cat=logits_cat,
-            )
+            if self.is_classifier:
+                return ModelOutput(
+                    logits=logits,
+                )
+            else:
+                return ModelOutputReg(
+                    logits=logits,
+                )
 
     def predict(self, signal):
         result = self(torch.from_numpy(signal))
@@ -447,33 +510,31 @@ class Model(Wav2Vec2PreTrainedModel):
         return result
 
 
-class ModelWithPreProcessing(Model):
-
-    def __init__(self, config):
-        super().__init__(config)
+class ConcordanceCorCoeff(torch.nn.Module):
 
-    def forward(
-        self,
-        input_values,
-    ):
-        # Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm():
-        # normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)
-
-        mean = input_values.mean()
-
-        # var = input_values.var()
-        # raises: onnxruntime.capi.onnxruntime_pybind11_state.NotImplemented: [ONNXRuntimeError] : 9 : NOT_IMPLEMENTED : Could not find an implementation for the node ReduceProd_3:ReduceProd(11)
-
-        var = torch.square(input_values - mean).mean()
-        input_values = (input_values - mean) / torch.sqrt(var + 1e-7)
-
-        output = super().forward(
-            input_values,
-            return_hidden=True,
+    def __init__(self):
+        super().__init__()
+        self.mean = torch.mean
+        self.var = torch.var
+        self.sum = torch.sum
+        self.sqrt = torch.sqrt
+        self.std = torch.std
+
+    def forward(self, prediction, ground_truth):
+        ground_truth = ground_truth.float()
+        mean_gt = self.mean(ground_truth, 0)
+        mean_pred = self.mean(prediction, 0)
+        var_gt = self.var(ground_truth, 0)
+        var_pred = self.var(prediction, 0)
+        v_pred = prediction - mean_pred
+        v_gt = ground_truth - mean_gt
+        cor = self.sum(v_pred * v_gt) / (
+            self.sqrt(self.sum(v_pred**2)) * self.sqrt(self.sum(v_gt**2))
         )
+        sd_gt = self.std(ground_truth)
+        sd_pred = self.std(prediction)
+        numerator = 2 * cor * sd_gt * sd_pred
+        denominator = var_gt + var_pred + (mean_gt - mean_pred) ** 2
+        ccc = numerator / denominator
 
-        return (
-            output.hidden_states,
-            output.logits_cat,
-            output.cnn_features,
-        )
+        return 1 - ccc

{nkululeko-0.85.2.dist-info → nkululeko-0.86.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.85.2
+Version: 0.86.0
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -333,6 +333,12 @@ F. Burkhardt, Johannes Wagner, Hagen Wierstorf, Florian Eyben and Björn Schulle
 Changelog
 =========
 
+Version 0.86.0
+--------------
+* added regression to finetuning
+* added other transformer models to finetuning
+* added output the train/dev features sets actually used by the model
+
 Version 0.85.2
 --------------
 * added data, and automatic task label detection

{nkululeko-0.85.2.dist-info → nkululeko-0.86.0.dist-info}/RECORD

@@ -2,11 +2,11 @@ nkululeko/__init__.py,sha256=62f8HiEzJ8rG2QlTFJXUCMpvuH3fKI33DoJSj33mscc,63
 nkululeko/aug_train.py,sha256=YhuZnS_WVWnun9G-M6g5n6rbRxoVREz6Zh7k6qprFNQ,3194
 nkululeko/augment.py,sha256=4MG0apTAG5RgkuJrYEjGgDdbodZWi_HweSPNI1JJ5QA,3051
 nkululeko/cacheddataset.py,sha256=lIJ6hUo5LoxSrzXtWV8mzwO7wRtUETWnOQ4ws2XfL1E,969
-nkululeko/constants.py,sha256=l15EMSj8vmejkCKCzQ6jMrgj5PuNrcHIREXt9kbSw7U,39
+nkululeko/constants.py,sha256=hvi1X27m7vcqkB_Rgl7alourAusZB1mjPxdW4ChdVyU,39
 nkululeko/demo.py,sha256=8bl15Kitoesnz8oa8yrs52T6YCSOhWbbq9PnZ8Hj6D0,3232
 nkululeko/demo_feats.py,sha256=sAeGFojhEj9WEDFtG3SzPBmyYJWLF2rkbpp65m8Ujo4,2025
 nkululeko/demo_predictor.py,sha256=es56xbT8ifkS_vnrlb5NTZT54gNmeUtNlA4zVA_gnN8,4757
-nkululeko/experiment.py,sha256=ZsSWdasWUyIBF_4vxb4FxvHs42pytG7ErUOABA-WWTo,30722
+nkululeko/experiment.py,sha256=gUJsBMWuadqxEVzuPVToQzFHC9FRUadptP49kTcBiGs,30962
 nkululeko/explore.py,sha256=lDzRoW_Taa5u4BBABZLD89BcQWnYlrftJR4jgt1yyj0,2609
 nkululeko/export.py,sha256=mHeEAAmtZuxdyebLlbSzPrHSi9OMgJHbk35d3DTxRBc,4632
 nkululeko/feature_extractor.py,sha256=8mssYKmo4LclVI-hiLmJEDZ0ZPyDavFG2YwtXcrGzwM,3976
@@ -58,7 +58,7 @@ nkululeko/feat_extract/feats_hubert.py,sha256=cLoUzSLjSYBkQnftjacSL7ES3O7Ysh_KrP
 nkululeko/feat_extract/feats_import.py,sha256=rj1p8lz19tCAC8hLzzZAwZ0M6gzwH3BzfabFUgal0yw,1622
 nkululeko/feat_extract/feats_mld.py,sha256=Vvu7GZOkn7Vda8eIOXqHjg78zegkFe3vTUaCXyVM0eA,2021
 nkululeko/feat_extract/feats_mos.py,sha256=KXNt7QYEfxkvr6UyVhig2aWQBaIvovlrR4gPuP03gmo,4174
-nkululeko/feat_extract/feats_opensmile.py,sha256=vLY8HCpeOj9NdJXzt_GVI3Vxwsjf9cEfcqJ3IHqlTQY,3978
+nkululeko/feat_extract/feats_opensmile.py,sha256=g6ZsAxjjGGvGfrr5fngWC-NJ8E7CP1kYZwrlodZJzzU,4028
 nkululeko/feat_extract/feats_oxbow.py,sha256=CmIG9cbHTJTJVnzgCPdQpYpnlewWExpsr5ZcK8Malyo,4980
 nkululeko/feat_extract/feats_praat.py,sha256=kZrS6srzH7WoWEd2prp1Dxw6g9JklFQGTNq5zzPpHzg,3105
 nkululeko/feat_extract/feats_snr.py,sha256=9dqZ-4RpK98iJEssM3ttozNd18LWlZYM_QVXvp5xDcs,2829
@@ -69,13 +69,12 @@ nkululeko/feat_extract/feats_trill.py,sha256=K2ahhdpwpjgg3WZS1POg3UMP2U44i8cLZZv
 nkululeko/feat_extract/feats_wav2vec2.py,sha256=9WUMfyddB_3nx79g7mZoQrRynhM1uEBWuOotRq8bxoU,5268
 nkululeko/feat_extract/feats_wavlm.py,sha256=ulxpGjifUFx2ZgGmY32SmBJGIuvkYHoLb2n1LZ8KMwA,4703
 nkululeko/feat_extract/feats_whisper.py,sha256=0N7Vj65OVi2PNoB_NrDjWT5lP6xZNKxFOZZIoxkJvcA,4533
-nkululeko/feat_extract/featureset.py,sha256=HtgW2389rmlRAgFP3F1sSFzq2_iUVr2NhOfIXG9omt0,1448
+nkululeko/feat_extract/featureset.py,sha256=ll7tyKAdr--TDShyOYJg0FB4I9NqBq0Ni1k_kUJ-2Vw,1541
 nkululeko/feat_extract/feinberg_praat.py,sha256=EP9pMALjlKdiYInLQdrZ7MmE499Mq-ISRCgqbqL3Rxc,21304
 nkululeko/losses/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/losses/loss_ccc.py,sha256=NOK0y0fxKUnU161B5geap6Fmn8QzoPl2MqtPiV8IuJE,976
 nkululeko/losses/loss_softf1loss.py,sha256=5gW-PuiqeAZcRgfwjueIOQtMokOjZWgQnVIv59HKTCo,1309
 nkululeko/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nkululeko/models/finetune_model.py,sha256=OMlzDyUFNXZ2xSiqqH8tbzey_KzPJ4jsoYT-4KrWFKM,5091
 nkululeko/models/model.py,sha256=PUCqF2r_dEfmFsZn6Cgr1UIzYvxziLH6nSqZ5-vuN1o,11639
 nkululeko/models/model_bayes.py,sha256=WJFZ8wFKwWATz6MhmjeZIi1Pal1viU549WL_PjXDSy8,406
 nkululeko/models/model_cnn.py,sha256=bJxqwe6FnVR2hFeqN6EXexYGgvKYFED1VOhBXVlLWaE,9954
@@ -89,7 +88,7 @@ nkululeko/models/model_svm.py,sha256=rsME3KvKvNG7bdE5lbvYUu85WZhaASZxxmdNDIVJRZ4
 nkululeko/models/model_svr.py,sha256=_YZeksqB3eBENGlg3g9RwYFlk9rQQ-XCeNBKLlGGVoE,725
 nkululeko/models/model_tree.py,sha256=rf16faUm4o2LJgkoYpeY998b8DQIvXZ73_m1IS3TnnE,417
 nkululeko/models/model_tree_reg.py,sha256=IgQcPTE-304HQLYSKPF8Z4ot_Ur9dH01fZjS0nXke_M,428
-nkululeko/models/model_tuned.py,sha256=WJplfUK3CGLSd2mahUrPSjMvqjPfxLp99KFeZaz2AbU,15098
+nkululeko/models/model_tuned.py,sha256=J5CemIAW_WhZIQgppFgPChrsMJvGYzJlCvJC8O62l9M,18049
 nkululeko/models/model_xgb.py,sha256=Thgx5ESdIok4v72mKh4plxpo4smGcKALWNCJTDScY0M,447
 nkululeko/models/model_xgr.py,sha256=aGBtNGLWjOE_2rICGYGFxmT8DtnHYsIl1lIpMtghHsY,418
 nkululeko/reporting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -106,8 +105,8 @@ nkululeko/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/utils/files.py,sha256=UiGAtZRWYjHSvlmPaTMtzyNNGE6qaLaxQkybctS7iRM,4021
 nkululeko/utils/stats.py,sha256=1yUq0FTOyqkU8TwUocJRYdJaqMU5SlOBBRUun9STo2M,2829
 nkululeko/utils/util.py,sha256=b1IHFucRNuF9Iyv5IJeK4AEg0Rga0xKG80UM5GWWdHA,13816
-nkululeko-0.85.2.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
-nkululeko-0.85.2.dist-info/METADATA,sha256=RVGREhA1jakUtQ707C0ecklnUZwx4skVHV0UbPwEsn0,36671
-nkululeko-0.85.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-nkululeko-0.85.2.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
-nkululeko-0.85.2.dist-info/RECORD,,
+nkululeko-0.86.0.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
+nkululeko-0.86.0.dist-info/METADATA,sha256=KrHrjQ6rc4oGxN4EJ_TuZ0dVGGI-qIxw8dY1RBTCnLo,36852
+nkululeko-0.86.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+nkululeko-0.86.0.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
+nkululeko-0.86.0.dist-info/RECORD,,

nkululeko/models/finetune_model.py

@@ -1,190 +0,0 @@
-"""
-Code based on @jwagner
-"""
-
-import dataclasses
-import typing
-
-import torch
-import transformers
-from transformers.models.wav2vec2.modeling_wav2vec2 import (
-    Wav2Vec2PreTrainedModel,
-    Wav2Vec2Model,
-)
-
-
-class ConcordanceCorCoeff(torch.nn.Module):
-
-    def __init__(self):
-
-        super().__init__()
-
-        self.mean = torch.mean
-        self.var = torch.var
-        self.sum = torch.sum
-        self.sqrt = torch.sqrt
-        self.std = torch.std
-
-    def forward(self, prediction, ground_truth):
-
-        mean_gt = self.mean(ground_truth, 0)
-        mean_pred = self.mean(prediction, 0)
-        var_gt = self.var(ground_truth, 0)
-        var_pred = self.var(prediction, 0)
-        v_pred = prediction - mean_pred
-        v_gt = ground_truth - mean_gt
-        cor = self.sum(v_pred * v_gt) / (
-            self.sqrt(self.sum(v_pred**2)) * self.sqrt(self.sum(v_gt**2))
-        )
-        sd_gt = self.std(ground_truth)
-        sd_pred = self.std(prediction)
-        numerator = 2 * cor * sd_gt * sd_pred
-        denominator = var_gt + var_pred + (mean_gt - mean_pred) ** 2
-        ccc = numerator / denominator
-
-        return 1 - ccc
-
-
-@dataclasses.dataclass
-class ModelOutput(transformers.file_utils.ModelOutput):
-
-    logits_cat: torch.FloatTensor = None
-    hidden_states: typing.Tuple[torch.FloatTensor] = None
-    cnn_features: torch.FloatTensor = None
-
-
-class ModelHead(torch.nn.Module):
-
-    def __init__(self, config, num_labels):
-
-        super().__init__()
-
-        self.dense = torch.nn.Linear(config.hidden_size, config.hidden_size)
-        self.dropout = torch.nn.Dropout(config.final_dropout)
-        self.out_proj = torch.nn.Linear(config.hidden_size, num_labels)
-
-    def forward(self, features, **kwargs):
-
-        x = features
-        x = self.dropout(x)
-        x = self.dense(x)
-        x = torch.tanh(x)
-        x = self.dropout(x)
-        x = self.out_proj(x)
-
-        return x
-
-
-class Model(Wav2Vec2PreTrainedModel):
-
-    def __init__(self, config):
-
-        super().__init__(config)
-
-        self.wav2vec2 = Wav2Vec2Model(config)
-        self.cat = ModelHead(config, 2)
-        self.init_weights()
-
-    def freeze_feature_extractor(self):
-        self.wav2vec2.feature_extractor._freeze_parameters()
-
-    def pooling(
-        self,
-        hidden_states,
-        attention_mask,
-    ):
-
-        if attention_mask is None:  # For evaluation with batch_size==1
-            outputs = torch.mean(hidden_states, dim=1)
-        else:
-            attention_mask = self._get_feature_vector_attention_mask(
-                hidden_states.shape[1],
-                attention_mask,
-            )
-            hidden_states = hidden_states * torch.reshape(
-                attention_mask,
-                (-1, attention_mask.shape[-1], 1),
-            )
-            outputs = torch.sum(hidden_states, dim=1)
-            attention_sum = torch.sum(attention_mask, dim=1)
-            outputs = outputs / torch.reshape(attention_sum, (-1, 1))
-
-        return outputs
-
-    def forward(
-        self,
-        input_values,
-        attention_mask=None,
-        labels=None,
-        return_hidden=False,
-    ):
-
-        outputs = self.wav2vec2(
-            input_values,
-            attention_mask=attention_mask,
-        )
-
-        cnn_features = outputs.extract_features
-        hidden_states_framewise = outputs.last_hidden_state
-        hidden_states = self.pooling(
-            hidden_states_framewise,
-            attention_mask,
-        )
-        logits_cat = self.cat(hidden_states)
-
-        if not self.training:
-            logits_cat = torch.softmax(logits_cat, dim=1)
-
-        if return_hidden:
-
-            # make time last axis
-            cnn_features = torch.transpose(cnn_features, 1, 2)
-
-            return ModelOutput(
-                logits_cat=logits_cat,
-                hidden_states=hidden_states,
-                cnn_features=cnn_features,
-            )
-
-        else:
-
-            return ModelOutput(
-                logits_cat=logits_cat,
-            )
-
-    def predict(self, signal):
-        result = self(torch.from_numpy(signal))
-        result = result[0].detach().numpy()[0]
-        return result
-
-
-class ModelWithPreProcessing(Model):
-
-    def __init__(self, config):
-        super().__init__(config)
-
-    def forward(
-        self,
-        input_values,
-    ):
-        # Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm():
-        # normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)
-
-        mean = input_values.mean()
-
-        # var = input_values.var()
-        # raises: onnxruntime.capi.onnxruntime_pybind11_state.NotImplemented: [ONNXRuntimeError] : 9 : NOT_IMPLEMENTED : Could not find an implementation for the node ReduceProd_3:ReduceProd(11)
-
-        var = torch.square(input_values - mean).mean()
-        input_values = (input_values - mean) / torch.sqrt(var + 1e-7)
-
-        output = super().forward(
-            input_values,
-            return_hidden=True,
-        )
-
-        return (
-            output.hidden_states,
-            output.logits_cat,
-            output.cnn_features,
-        )