nkululeko 0.90.0__py3-none-any.whl → 0.90.2__py3-none-any.whl

nkululeko/models/model_tuned.py

@@ -7,17 +7,18 @@ import os
 import pickle
 import typing
 
+import audeer
+import audiofile
+import audmetric
 import datasets
 import numpy as np
 import pandas as pd
 import torch
 import transformers
-from transformers.models.wav2vec2.modeling_wav2vec2 import Wav2Vec2Model
-from transformers.models.wav2vec2.modeling_wav2vec2 import Wav2Vec2PreTrainedModel
-
-import audeer
-import audiofile
-import audmetric
+from transformers.models.wav2vec2.modeling_wav2vec2 import (
+    Wav2Vec2Model,
+    Wav2Vec2PreTrainedModel,
+)
 
 import nkululeko.glob_conf as glob_conf
 from nkululeko.models.model import Model as BaseModel
@@ -25,15 +26,20 @@ from nkululeko.reporting.reporter import Reporter
 
 
 class TunedModel(BaseModel):
-
     def __init__(self, df_train, df_test, feats_train, feats_test):
         """Constructor taking the configuration and all dataframes."""
         super().__init__(df_train, df_test, feats_train, feats_test)
         super().set_model_type("finetuned")
+        self.df_test, self.df_train, self.feats_test, self.feats_train = (
+            df_test,
+            df_train,
+            feats_test,
+            feats_train,
+        )
         self.name = "finetuned_wav2vec2"
         self.target = glob_conf.config["DATA"]["target"]
-        labels = glob_conf.labels
-        self.class_num = len(labels)
+        self.labels = glob_conf.labels
+        self.class_num = len(self.labels)
         device = self.util.config_val("MODEL", "device", False)
         if not device:
             self.device = "cuda" if torch.cuda.is_available() else "cpu"
@@ -54,8 +60,7 @@ class TunedModel(BaseModel):
         self.learning_rate = float(
             self.util.config_val("MODEL", "learning_rate", "0.0001")
         )
-        self.max_duration = float(
-            self.util.config_val("MODEL", "max_duration", "8.0"))
+        self.max_duration = float(self.util.config_val("MODEL", "max_duration", "8.0"))
         self.df_train, self.df_test = df_train, df_test
         self.epoch_num = int(self.util.config_val("EXP", "epochs", 1))
         self.util.debug(f"num of epochs: {self.epoch_num}")
@@ -65,12 +70,12 @@ class TunedModel(BaseModel):
             self.drop = float(drop)
         self.util.debug(f"init: training with dropout: {self.drop}")
         self.push = eval(self.util.config_val("MODEL", "push_to_hub", "False"))
+        self.balancing = self.util.config_val("MODEL", "balancing", False)
         self._init_model()
 
     def _init_model(self):
         model_path = "facebook/wav2vec2-large-robust-ft-swbd-300h"
-        pretrained_model = self.util.config_val(
-            "MODEL", "pretrained_model", model_path)
+        pretrained_model = self.util.config_val("MODEL", "pretrained_model", model_path)
         self.num_layers = None
         self.sampling_rate = 16000
         self.max_duration_sec = self.max_duration
@@ -94,6 +99,42 @@ class TunedModel(BaseModel):
             df = y.reset_index()
             df.start = df.start.dt.total_seconds()
             df.end = df.end.dt.total_seconds()
+            #     ds = datasets.Dataset.from_pandas(df)
+            #     dataset[split] = ds
+
+            # self.dataset = datasets.DatasetDict(dataset)
+            if split == "train" and self.balancing:
+                if self.balancing == "ros":
+                    from imblearn.over_sampling import RandomOverSampler
+
+                    sampler = RandomOverSampler(random_state=42)
+                elif self.balancing == "smote":
+                    from imblearn.over_sampling import SMOTE
+
+                    sampler = SMOTE(random_state=42)
+                elif self.balancing == "adasyn":
+                    from imblearn.over_sampling import ADASYN
+
+                    sampler = ADASYN(random_state=42)
+                else:
+                    self.util.error(f"Unknown balancing algorithm: {self.balancing}")
+
+                X_resampled, y_resampled = sampler.fit_resample(
+                    df[["start", "end"]], df["targets"]
+                )
+                df = pd.DataFrame(
+                    {
+                        "start": X_resampled["start"],
+                        "end": X_resampled["end"],
+                        "targets": y_resampled,
+                    }
+                )
+
+                # print the before and after class distribution
+                self.util.debug(
+                    f"balanced with: {self.balancing}, new size: {len(df)}, was {len(data_sources[split])}"
+                )
+
             ds = datasets.Dataset.from_pandas(df)
             dataset[split] = ds
 
@@ -101,7 +142,7 @@ class TunedModel(BaseModel):
 
         # load pre-trained model
         if self.is_classifier:
-            self.util.debug(f"Task is classification.")
+            self.util.debug("Task is classification.")
             le = glob_conf.label_encoder
             mapping = dict(zip(le.classes_, range(len(le.classes_))))
             target_mapping = {k: int(v) for k, v in mapping.items()}
@@ -116,7 +157,7 @@ class TunedModel(BaseModel):
                 finetuning_task=target_name,
             )
         else:
-            self.util.debug(f"Task is regression.")
+            self.util.debug("Task is regression.")
             self.config = transformers.AutoConfig.from_pretrained(
                 pretrained_model,
                 num_labels=1,
@@ -215,7 +256,6 @@ class TunedModel(BaseModel):
         return batch
 
     def compute_metrics(self, p: transformers.EvalPrediction):
-
         metrics = {
             "UAR": audmetric.unweighted_average_recall,
             "ACC": audmetric.accuracy,
@@ -268,10 +308,9 @@ class TunedModel(BaseModel):
                 else:
                     criterion = torch.nn.CrossEntropyLoss()
             else:
-                self.util.error(
-                    f"criterion {criterion} not supported for classifier")
+                self.util.error(f"criterion {criterion} not supported for classifier")
         else:
-            self.criterion = self.util.config_val("MODEL", "loss", "ccc")
+            criterion = self.util.config_val("MODEL", "loss", "1-ccc")
             if criterion == "1-ccc":
                 criterion = ConcordanceCorCoeff()
             elif criterion == "mse":
@@ -279,8 +318,7 @@ class TunedModel(BaseModel):
             elif criterion == "mae":
                 criterion = torch.nn.L1Loss()
             else:
-                self.util.error(
-                    f"criterion {criterion} not supported for regressor")
+                self.util.error(f"criterion {criterion} not supported for regressor")
 
         # set push_to_hub value, default false
         # push = eval(self.util.config_val("MODEL", "push_to_hub", "False"))
@@ -319,8 +357,7 @@ class TunedModel(BaseModel):
         elif metrics_for_best_model == "MAE":
             greater_is_better = False
         else:
-            self.util.error(
-                f"unknown metric/measure: {metrics_for_best_model}")
+            self.util.error(f"unknown metric/measure: {metrics_for_best_model}")
 
         training_args = transformers.TrainingArguments(
             output_dir=model_root,
@@ -371,7 +408,7 @@ class TunedModel(BaseModel):
         self.load(self.run, self.epoch)
 
     def get_predictions(self):
-        results = []
+        results = [[]].pop(0)
         for (file, start, end), _ in audeer.progress_bar(
             self.df_test.iterrows(),
             total=len(self.df_test),
@@ -384,18 +421,37 @@ class TunedModel(BaseModel):
                     file, duration=end - start, offset=start, always_2d=True
                 )
             assert sr == self.sampling_rate
-            predictions = self.model.predict(signal)
-            results.append(predictions.argmax())
-        return results
+            prediction = self.model.predict(signal)
+            results.append(prediction)
+            # results.append(predictions.argmax())
+        predictions = np.asarray(results)
+        if self.util.exp_is_classification():
+            # make a dataframe for the class probabilities
+            proba_d = {}
+            for c in range(self.class_num):
+                proba_d[c] = []
+            # get the class probabilities
+            # predictions = self.clf.predict_proba(self.feats_test.to_numpy())
+            # pred = self.clf.predict(features)
+            for i in range(self.class_num):
+                proba_d[i] = list(predictions.T[i])
+            probas = pd.DataFrame(proba_d)
+            probas = probas.set_index(self.df_test.index)
+            predictions = probas.idxmax(axis=1).values
+        else:
+            predictions = predictions.flatten()
+            probas = None
+        return predictions, probas
 
     def predict(self):
         """Predict the whole eval feature set"""
-        predictions = self.get_predictions()
+        predictions, probas = self.get_predictions()
         report = Reporter(
             self.df_test[self.target].to_numpy().astype(float),
             predictions,
             self.run,
             self.epoch_num,
+            probas=probas,
         )
         self._plot_epoch_progression(report)
         return report
@@ -407,6 +463,7 @@ class TunedModel(BaseModel):
         )
         with open(log_file, "r") as file:
             data = file.read()
+        data = data.strip().replace("nan", "0")
         list = ast.literal_eval(data)
         epochs, vals, loss = [], [], []
         for index, tp in enumerate(list):
@@ -453,17 +510,15 @@ class TunedModel(BaseModel):
             self.clf = pickle.load(handle)
 
 
-@ dataclasses.dataclass
+@dataclasses.dataclass
 class ModelOutput(transformers.file_utils.ModelOutput):
-
     logits: torch.FloatTensor = None
     hidden_states: typing.Tuple[torch.FloatTensor] = None
     cnn_features: torch.FloatTensor = None
 
 
-@ dataclasses.dataclass
+@dataclasses.dataclass
 class ModelOutputReg(transformers.file_utils.ModelOutput):
-
     logits: torch.FloatTensor
     hidden_states: typing.Tuple[torch.FloatTensor] = None
     attentions: typing.Tuple[torch.FloatTensor] = None
@@ -473,9 +528,7 @@ class ModelOutputReg(transformers.file_utils.ModelOutput):
 
 
 class ModelHead(torch.nn.Module):
-
     def __init__(self, config):
-
         super().__init__()
 
         self.dense = torch.nn.Linear(config.hidden_size, config.hidden_size)
@@ -483,7 +536,6 @@ class ModelHead(torch.nn.Module):
         self.out_proj = torch.nn.Linear(config.hidden_size, config.num_labels)
 
     def forward(self, features, **kwargs):
-
         x = features
         x = self.dropout(x)
         x = self.dense(x)
@@ -495,9 +547,7 @@ class ModelHead(torch.nn.Module):
 
 
 class Model(Wav2Vec2PreTrainedModel):
-
     def __init__(self, config):
-
         if not hasattr(config, "add_adapter"):
             setattr(config, "add_adapter", False)
 
@@ -516,7 +566,6 @@ class Model(Wav2Vec2PreTrainedModel):
         hidden_states,
         attention_mask,
     ):
-
         if attention_mask is None:  # For evaluation with batch_size==1
             outputs = torch.mean(hidden_states, dim=1)
         else:
@@ -532,8 +581,7 @@ class Model(Wav2Vec2PreTrainedModel):
             attention_sum = torch.sum(attention_mask, dim=1)
 
             epsilon = 1e-6  # to avoid division by zero and numerical instability
-            outputs = outputs / (torch.reshape(attention_sum, (-1, 1)) +
-                                 epsilon)
+            outputs = outputs / (torch.reshape(attention_sum, (-1, 1)) + epsilon)
 
         return outputs
 
@@ -590,7 +638,6 @@ class Model(Wav2Vec2PreTrainedModel):
 
 
 class ConcordanceCorCoeff(torch.nn.Module):
-
     def __init__(self):
         super().__init__()
         self.mean = torch.mean

nkululeko/models/model_xgb.py

@@ -1,6 +1,7 @@
 # xgbmodel.py
 
 from xgboost import XGBClassifier
+
 from nkululeko.models.model import Model
 
 

nkululeko/models/model_xgr.py

@@ -1,6 +1,7 @@
 # xgrmodel.py
 
 from xgboost.sklearn import XGBRegressor
+
 from nkululeko.models.model import Model
 
 

nkululeko/multidb.py

@@ -5,6 +5,7 @@ The `main` function is the entry point of the script, which parses command-line
 
 The `plot_heatmap` function generates a heatmap plot of the results and saves it to a file, along with some summary statistics.
 """
+
 # main.py
 # Demonstration code to use the ML-experiment framework
 

nkululeko/nkululeko.py

@@ -6,8 +6,8 @@ import os.path
 
 import numpy as np
 
-from nkululeko.constants import VERSION
 import nkululeko.experiment as exp
+from nkululeko.constants import VERSION
 from nkululeko.utils.util import Util
 
 

nkululeko/predict.py

@@ -26,9 +26,9 @@ from nkululeko.utils.util import Util
 
 def main(src_dir):
     parser = argparse.ArgumentParser(
-        description="Call the nkululeko PREDICT framework.")
-    parser.add_argument("--config", default="exp.ini",
-                        help="The base configuration")
+        description="Call the nkululeko PREDICT framework."
+    )
+    parser.add_argument("--config", default="exp.ini", help="The base configuration")
     args = parser.parse_args()
     if args.config is not None:
         config_file = args.config
@@ -58,8 +58,7 @@ def main(src_dir):
 
     # split into train and test
     expr.fill_train_and_tests()
-    util.debug(
-        f"train shape : {expr.df_train.shape}, test shape:{expr.df_test.shape}")
+    util.debug(f"train shape : {expr.df_train.shape}, test shape:{expr.df_test.shape}")
 
     # process the data
     df = expr.autopredict()

nkululeko/reporting/defines.py

@@ -1,5 +1,4 @@
 class DefineBase:
-
     @classmethod
     def _assert_has_attribute_value(cls, value):
         valid_values = cls._attribute_values()
@@ -8,17 +7,16 @@ class DefineBase:
 
     @classmethod
     def _attribute_values(cls):
-        attributes = inspect.getmembers(
-            cls, lambda x: not inspect.isroutine(x)
-        )
+        attributes = inspect.getmembers(cls, lambda x: not inspect.isroutine(x))
         return sorted(
             [
-                a[1] for a in attributes
-                if not (a[0].startswith('__') and a[0].endswith('__'))
+                a[1]
+                for a in attributes
+                if not (a[0].startswith("__") and a[0].endswith("__"))
             ]
         )
 
 
 class Header(DefineBase):
-    HEADER_RESULTS = 'Results'
-    HEADER_EXPLORE = 'Data exploration'
+    HEADER_RESULTS = "Results"
+    HEADER_EXPLORE = "Data exploration"

nkululeko/reporting/latex_writer.py

@@ -3,9 +3,9 @@ latex_writer.py
 print out report as latex file and pdf
 """
 
-from pylatex import Document, Section, Subsection, Command, Figure
-from pylatex.utils import italic, NoEscape
-from nkululeko.reporting.report_item import ReportItem
+from pylatex import Command, Document, Figure, Section, Subsection
+from pylatex.utils import NoEscape
+
 from nkululeko.utils.util import Util
 
 

nkululeko/reporting/report.py

@@ -4,9 +4,9 @@ report.py
 Collector class for report items collected during module processing.
 
 """
-from nkululeko.utils.util import Util
-from nkululeko.reporting.report_item import ReportItem
+
 from nkululeko.reporting.latex_writer import LatexWriter
+from nkululeko.utils.util import Util
 
 
 class Report:

nkululeko/reporting/report_item.py

@@ -4,6 +4,7 @@ a basic report snippet.
 
 import os.path
 
+
 class ReportItem:
     def __init__(self, topic, caption, contents, image=None):
         self.topic = topic

nkululeko/reporting/reporter.py

@@ -21,13 +21,13 @@ from scipy.special import softmax
 from scipy.stats import entropy, pearsonr
 from sklearn.metrics import (
     ConfusionMatrixDisplay,
+    RocCurveDisplay,
     auc,
     classification_report,
     confusion_matrix,
     r2_score,
     roc_auc_score,
     roc_curve,
-    RocCurveDisplay,
 )
 
 import nkululeko.glob_conf as glob_conf
@@ -419,24 +419,25 @@ class Reporter:
                 f1_per_class = (
                     f"result per class (F1 score): {c_ress} from epoch: {epoch}"
                 )
-                if len(np.unique(self.truths)) == 2:
-                    fpr, tpr, _ = roc_curve(self.truths, self.preds)
-                    auc_score = auc(fpr, tpr)
-                    plot_path = f"{fig_dir}{self.util.get_exp_name()}_{epoch}{self.filenameadd}_roc.{self.format}"
-                    plt.figure()
-                    display = RocCurveDisplay(
-                        fpr=fpr,
-                        tpr=tpr,
-                        roc_auc=auc_score,
-                        estimator_name=f"{self.model_type} estimator",
-                    )
-                    display.plot(ax=None)
-                    plt.savefig(plot_path)
-                    plt.close()
-                    self.util.debug(f"Saved ROC curve to {plot_path}")
-                    pauc_score = roc_auc_score(self.truths, self.preds, max_fpr=0.1)
-                    auc_pauc = f"auc: {auc_score:.3f}, pauc: {pauc_score:.3f} from epoch: {epoch}"
-                    self.util.debug(auc_pauc)
+                # the following auc is buggy, preds should be probabilities
+                # if len(np.unique(self.truths)) == 2:
+                #     fpr, tpr, _ = roc_curve(self.truths, self.preds)
+                #     auc_score = auc(fpr, tpr)
+                #     plot_path = f"{fig_dir}{self.util.get_exp_name()}_{epoch}{self.filenameadd}_roc.{self.format}"
+                #     plt.figure()
+                #     display = RocCurveDisplay(
+                #         fpr=fpr,
+                #         tpr=tpr,
+                #         roc_auc=auc_score,
+                #         estimator_name=f"{self.model_type} estimator",
+                #     )
+                #     display.plot(ax=None)
+                #     plt.savefig(plot_path)
+                #     plt.close()
+                #     self.util.debug(f"Saved ROC curve to {plot_path}")
+                #     pauc_score = roc_auc_score(self.truths, self.preds, max_fpr=0.1)
+                #     auc_pauc = f"auc: {auc_score:.3f}, pauc: {pauc_score:.3f} from epoch: {epoch}"
+                #     self.util.debug(auc_pauc)
                 self.util.debug(f1_per_class)
                 rpt_str = f"{json.dumps(rpt)}\n{f1_per_class}"
                 # rpt_str += f"\n{auc_auc}"

nkululeko/resample.py

@@ -4,24 +4,23 @@
 import argparse
 import configparser
 import os
-import pandas as pd
+
 import audformat
-from nkululeko.augmenting.resampler import Resampler
-from nkululeko.utils.util import Util
+import pandas as pd
 
+from nkululeko.augmenting.resampler import Resampler
 from nkululeko.constants import VERSION
 from nkululeko.experiment import Experiment
 from nkululeko.utils.files import find_files
+from nkululeko.utils.util import Util
 
 
 def main(src_dir):
     parser = argparse.ArgumentParser(
         description="Call the nkululeko RESAMPLE framework."
     )
-    parser.add_argument("--config", default=None,
-                        help="The base configuration")
-    parser.add_argument("--file", default=None,
-                        help="The input audio file to resample")
+    parser.add_argument("--config", default=None, help="The base configuration")
+    parser.add_argument("--file", default=None, help="The input audio file to resample")
     parser.add_argument(
         "--folder",
         default=None,
@@ -34,9 +33,7 @@ def main(src_dir):
     args = parser.parse_args()
 
     if args.file is None and args.folder is None and args.config is None:
-        print(
-            "ERROR: Either --file, --folder, or --config argument must be provided."
-        )
+        print("ERROR: Either --file, --folder, or --config argument must be provided.")
         exit()
 
     if args.file is not None:
@@ -102,8 +99,7 @@ def main(src_dir):
             f"train shape : {expr.df_train.shape}, test shape:{expr.df_test.shape}"
         )
 
-        sample_selection = util.config_val(
-            "RESAMPLE", "sample_selection", "all")
+        sample_selection = util.config_val("RESAMPLE", "sample_selection", "all")
         if sample_selection == "all":
             df = pd.concat([expr.df_train, expr.df_test])
         elif sample_selection == "train":

nkululeko/resample_cli.py

@@ -0,0 +1,99 @@
+import argparse
+import configparser
+import os
+
+import audformat
+import pandas as pd
+
+from nkululeko.augmenting.resampler import Resampler
+from nkululeko.constants import VERSION
+from nkululeko.experiment import Experiment
+from nkululeko.utils.util import Util
+
+
+def main(src_dir):
+    parser = argparse.ArgumentParser(
+        description="Call the nkululeko RESAMPLE framework."
+    )
+    parser.add_argument("--config", default=None, help="The base configuration")
+    parser.add_argument("--file", default=None, help="The input audio file to resample")
+    parser.add_argument(
+        "--replace", action="store_true", help="Replace the original audio file"
+    )
+
+    args = parser.parse_args()
+
+    if args.file is None and args.config is None:
+        print("ERROR: Either --file or --config argument must be provided.")
+        exit()
+
+    if args.file is not None:
+        # Load the audio file into a DataFrame
+        files = pd.Series([args.file])
+        df_sample = pd.DataFrame(index=files)
+        df_sample.index = audformat.utils.to_segmented_index(
+            df_sample.index, allow_nat=False
+        )
+
+        # Resample the audio file
+        util = Util("resampler", has_config=False)
+        util.debug(f"Resampling audio file: {args.file}")
+        rs = Resampler(df_sample, not_testing=True, replace=args.replace)
+        rs.resample()
+    else:
+        # Existing code for handling INI file
+        config_file = args.config
+
+        # Test if the configuration file exists
+        if not os.path.isfile(config_file):
+            print(f"ERROR: no such file: {config_file}")
+            exit()
+
+        # Load one configuration per experiment
+        config = configparser.ConfigParser()
+        config.read(config_file)
+        # Create a new experiment
+        expr = Experiment(config)
+        module = "resample"
+        expr.set_module(module)
+        util = Util(module)
+        util.debug(
+            f"running {expr.name} from config {config_file}, nkululeko version"
+            f" {VERSION}"
+        )
+
+        if util.config_val("EXP", "no_warnings", False):
+            import warnings
+
+            warnings.filterwarnings("ignore")
+
+        # Load the data
+        expr.load_datasets()
+
+        # Split into train and test
+        expr.fill_train_and_tests()
+        util.debug(
+            f"train shape : {expr.df_train.shape}, test shape:{expr.df_test.shape}"
+        )
+
+        sample_selection = util.config_val("RESAMPLE", "sample_selection", "all")
+        if sample_selection == "all":
+            df = pd.concat([expr.df_train, expr.df_test])
+        elif sample_selection == "train":
+            df = expr.df_train
+        elif sample_selection == "test":
+            df = expr.df_test
+        else:
+            util.error(
+                f"unknown selection specifier {sample_selection}, should be [all |"
+                " train | test]"
+            )
+        util.debug(f"resampling {sample_selection}: {df.shape[0]} samples")
+        replace = util.config_val("RESAMPLE", "replace", "False")
+        rs = Resampler(df, replace=replace)
+        rs.resample()
+
+
+if __name__ == "__main__":
+    cwd = os.path.dirname(os.path.abspath(__file__))
+    main(cwd)

nkululeko/runmanager.py

@@ -50,7 +50,9 @@ class Runmanager:
         self.last_epochs = []  # keep the epoch of best result per run
         # for all runs
         for run in range(int(self.util.config_val("EXP", "runs", 1))):
-            self.util.debug(f"run {run} using model {glob_conf.config['MODEL']['type']}")
+            self.util.debug(
+                f"run {run} using model {glob_conf.config['MODEL']['type']}"
+            )
             # set the run index as global variable for reporting
             self.util.set_config_val("EXP", "run", run)
             self.modelrunner = Modelrunner(

nkululeko/scaler.py

@@ -1,8 +1,8 @@
 # scaler.py
 
 import numpy as np
-from sklearn.preprocessing import StandardScaler, RobustScaler
 import pandas as pd
+from sklearn.preprocessing import RobustScaler, StandardScaler
 
 from nkululeko.utils.util import Util