nkululeko 0.81.3__py3-none-any.whl → 0.81.6__py3-none-any.whl

nkululeko/feature_extractor.py

@@ -1,22 +1,25 @@
-"""
-feature_extractor.py
-
-Helper class to encapsulate feature extraction methods
+"""Extract acoustic features from audio samples.
 
+Extract acoustic features using several feature extractors 
+(appends the features column-wise)
 """
+
 import pandas as pd
 
 from nkululeko.utils.util import Util
 
 
 class FeatureExtractor:
-    """
-    Extract acoustic features from audio samples, using several feature extractors (appends the features column-wise)
+    """Extract acoustic features from audio samples.
+
+    Extract acoustic features using several feature extractors (appends the features column-wise).
+
     Args:
         data_df (pandas.DataFrame): dataframe with audiofile paths as index
-        feats_types (array of strings): designations of acoustic feature extractors to be used
-        data_name (string): names of databases that are extracted (for the caching)
-        feats_designation (string): the type of split (train/test), also is used for the cache name.
+        feats_types (List[str]): designations of acoustic feature extractors to be used
+        data_name (str): name of databases that are extracted (for caching)
+        feats_designation (str): the type of split (train/test), also is used for the cache name.
+
     Returns:
         df (pandas.DataFrame): dataframe with same index as data_df and acoustic features in columns
     """
@@ -25,7 +28,6 @@ class FeatureExtractor:
     df = None
     data_df = None  # dataframe to get audio paths
 
-    # def __init__
     def __init__(self, data_df, feats_types, data_name, feats_designation):
         self.data_df = data_df
         self.data_name = data_name
@@ -34,147 +36,69 @@ class FeatureExtractor:
         self.feats_designation = feats_designation
 
     def extract(self):
-        # feats_types = self.util.config_val_list('FEATS', 'type', ['os'])
-        self.featExtractor = None
         self.feats = pd.DataFrame()
-        _scale = True
         for feats_type in self.feats_types:
             store_name = f"{self.data_name}_{feats_type}"
-            if feats_type == "os":
-                from nkululeko.feat_extract.feats_opensmile import Opensmileset
-
-                self.featExtractor = Opensmileset(
-                    f"{store_name}_{self.feats_designation}", self.data_df
-                )
-            elif feats_type == "spectra":
-                from nkululeko.feat_extract.feats_spectra import Spectraloader
-
-                self.featExtractor = Spectraloader(
-                    f"{store_name}_{self.feats_designation}", self.data_df
-                )
-            elif feats_type == "trill":
-                from nkululeko.feat_extract.feats_trill import TRILLset
-
-                self.featExtractor = TRILLset(
-                    f"{store_name}_{self.feats_designation}", self.data_df
-                )
-            elif feats_type.startswith("wav2vec"):
-                from nkululeko.feat_extract.feats_wav2vec2 import Wav2vec2
-
-                self.featExtractor = Wav2vec2(
-                    f"{store_name}_{self.feats_designation}",
-                    self.data_df,
-                    feats_type,
-                )
-            elif feats_type.startswith("hubert"):
-                from nkululeko.feat_extract.feats_hubert import Hubert
-
-                self.featExtractor = Hubert(
-                    f"{store_name}_{self.feats_designation}",
-                    self.data_df,
-                    feats_type,
-                )
-
-            elif feats_type.startswith("wavlm"):
-                from nkululeko.feat_extract.feats_wavlm import Wavlm
-
-                self.featExtractor = Wavlm(
-                    f"{store_name}_{self.feats_designation}",
-                    self.data_df,
-                    feats_type,
-                )
-
-            elif feats_type.startswith("spkrec"):
-                from nkululeko.feat_extract.feats_spkrec import Spkrec
-
-                self.featExtractor = Spkrec(
-                    f"{store_name}_{self.feats_designation}",
-                    self.data_df,
-                    feats_type,
-                )
-            elif feats_type == "audmodel":
-                from nkululeko.feat_extract.feats_audmodel import AudModelSet
-
-                self.featExtractor = AudModelSet(
-                    f"{store_name}_{self.feats_designation}", self.data_df
-                )
-            elif feats_type == "auddim":
-                from nkululeko.feat_extract.feats_audmodel_dim import (
-                    AudModelDimSet,
-                )
-
-                self.featExtractor = AudModelDimSet(
-                    f"{store_name}_{self.feats_designation}", self.data_df
-                )
-            elif feats_type == "agender":
-                from nkululeko.feat_extract.feats_agender import (
-                    AudModelAgenderSet,
-                )
-
-                self.featExtractor = AudModelAgenderSet(
-                    f"{store_name}_{self.feats_designation}", self.data_df
-                )
-            elif feats_type == "agender_agender":
-                from nkululeko.feat_extract.feats_agender_agender import (
-                    AgenderAgenderSet,
-                )
-
-                self.featExtractor = AgenderAgenderSet(
-                    f"{store_name}_{self.feats_designation}", self.data_df
-                )
-            elif feats_type == "snr":
-                from nkululeko.feat_extract.feats_snr import SNRSet
-
-                self.featExtractor = SNRSet(
-                    f"{store_name}_{self.feats_designation}", self.data_df
-                )
-            elif feats_type == "mos":
-                from nkululeko.feat_extract.feats_mos import MOSSet
-
-                self.featExtractor = MOSSet(
-                    f"{store_name}_{self.feats_designation}", self.data_df
-                )
-            elif feats_type == "squim":
-                from nkululeko.feat_extract.feats_squim import SQUIMSet
-
-                self.featExtractor = SQUIMSet(
-                    f"{store_name}_{self.feats_designation}", self.data_df
-                )
-            elif feats_type == "clap":
-                from nkululeko.feat_extract.feats_clap import Clap
-
-                self.featExtractor = Clap(
-                    f"{store_name}_{self.feats_designation}", self.data_df
-                )
-            elif feats_type == "praat":
-                from nkululeko.feat_extract.feats_praat import Praatset
-
-                self.featExtractor = Praatset(
-                    f"{store_name}_{self.feats_designation}", self.data_df
-                )
-            elif feats_type == "mld":
-                from nkululeko.feat_extract.feats_mld import MLD_set
-
-                self.featExtractor = MLD_set(
-                    f"{store_name}_{self.feats_designation}", self.data_df
-                )
-            elif feats_type == "import":
-                from nkululeko.feat_extract.feats_import import Importset
-
-                self.featExtractor = Importset(
-                    f"{store_name}_{self.feats_designation}", self.data_df
-                )
-            else:
-                self.util.error(f"unknown feats_type: {feats_type}")
-
-            self.featExtractor.extract()
-            self.featExtractor.filter()
-            # remove samples that were not extracted by MLD
-            # self.df_test = self.df_test.loc[self.df_test.index.intersection(featExtractor_test.df.index)]
-            # self.df_train = self.df_train.loc[self.df_train.index.intersection(featExtractor_train.df.index)]
-            self.util.debug(f"{feats_type}: shape : {self.featExtractor.df.shape}")
-            self.feats = pd.concat([self.feats, self.featExtractor.df], axis=1)
+            self.feat_extractor = self._get_feat_extractor(store_name, feats_type)
+            self.feat_extractor.extract()
+            self.feat_extractor.filter()
+            self.feats = pd.concat([self.feats, self.feat_extractor.df], axis=1)
         return self.feats
 
     def extract_sample(self, signal, sr):
-        return self.featExtractor.extract_sample(signal, sr)
+        return self.feat_extractor.extract_sample(signal, sr)
+
+    def _get_feat_extractor(self, store_name, feats_type):
+        feat_extractor_class = self._get_feat_extractor_class(feats_type)
+        if feat_extractor_class is None:
+            self.util.error(f"unknown feats_type: {feats_type}")
+        return feat_extractor_class(
+            f"{store_name}_{self.feats_designation}", self.data_df
+        )
+
+    def _get_feat_extractor_class(self, feats_type):
+        if feats_type == "os":
+            from nkululeko.feat_extract.feats_opensmile import Opensmileset
+
+            return Opensmileset
+        elif feats_type == "spectra":
+            from nkululeko.feat_extract.feats_spectra import Spectraloader
+
+            return Spectraloader
+        elif feats_type == "trill":
+            from nkululeko.feat_extract.feats_trill import TRILLset
+
+            return TRILLset
+        elif feats_type.startswith(("wav2vec", "hubert", "wavlm", "spkrec")):
+            return self._get_feat_extractor_by_prefix(feats_type)
+        elif feats_type in (
+            "audmodel",
+            "auddim",
+            "agender",
+            "agender_agender",
+            "snr",
+            "mos",
+            "squim",
+            "clap",
+            "praat",
+            "mld",
+            "import",
+        ):
+            return self._get_feat_extractor_by_name(feats_type)
+        else:
+            return None
+
+    def _get_feat_extractor_by_prefix(self, feats_type):
+        prefix, _, ext = feats_type.partition("_")
+        from importlib import import_module
+
+        module = import_module(f"nkululeko.feat_extract.feats_{prefix.lower()}")
+        class_name = f"{prefix.capitalize()}{ext.capitalize()}set"
+        return getattr(module, class_name)
+
+    def _get_feat_extractor_by_name(self, feats_type):
+        from importlib import import_module
+
+        module = import_module(f"nkululeko.feat_extract.feats_{feats_type.lower()}")
+        class_name = f"{feats_type.capitalize()}Set"
+        return getattr(module, class_name)

nkululeko/modelrunner.py

@@ -39,7 +39,7 @@ class Modelrunner:
         plot_epochs = self.util.config_val("PLOT", "epochs", False)
         only_test = self.util.config_val("MODEL", "only_test", False)
         epoch_num = int(self.util.config_val("EXP", "epochs", 1))
-        if not self.model.is_ANN() and epoch_num > 1:
+        if not self.model.is_ann() and epoch_num > 1:
             self.util.warn(f"setting epoch num to 1 (was {epoch_num}) if model not ANN")
             epoch_num = 1
             glob_conf.config["EXP"]["epochs"] = "1"
@@ -69,7 +69,7 @@ class Modelrunner:
             if plot_epochs:
                 self.util.debug(f"plotting conf matrix to {plot_name}")
                 report.plot_confmatrix(plot_name, epoch)
-            store_models = self.util.config_val("MODEL", "save", False)
+            store_models = self.util.config_val("EXP", "save", False)
             plot_best_model = self.util.config_val("PLOT", "best_model", False)
             if (store_models or plot_best_model) and (
                 not only_test

nkululeko/models/model.py

@@ -1,23 +1,25 @@
 # model.py
-from nkululeko.utils.util import Util
-import pandas as pd
-import numpy as np
-import nkululeko.glob_conf as glob_conf
-import sklearn.utils
-from nkululeko.reporting.reporter import Reporter
 import ast
-from sklearn.model_selection import GridSearchCV
 import pickle
 import random
+
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import GridSearchCV
 from sklearn.model_selection import LeaveOneGroupOut
 from sklearn.model_selection import StratifiedKFold
+import sklearn.utils
+
+import nkululeko.glob_conf as glob_conf
+from nkululeko.reporting.reporter import Reporter
+from nkululeko.utils.util import Util
 
 
 class Model:
-    """Generic model class for linear (non-neural) algorithms"""
+    """Generic model class for linear (non-neural) algorithms."""
 
     def __init__(self, df_train, df_test, feats_train, feats_test):
-        """Constructor taking the configuration and all dataframes"""
+        """Constructor taking the configuration and all dataframes."""
         self.df_train, self.df_test, self.feats_train, self.feats_test = (
             df_train,
             df_test,
@@ -35,7 +37,7 @@ class Model:
     def set_model_type(self, type):
         self.model_type = type
 
-    def is_ANN(self):
+    def is_ann(self):
         if self.model_type == "ann":
             return True
         else:
@@ -277,8 +279,6 @@ class Model:
         prediction = {}
         if self.util.exp_is_classification():
             # get the class probabilities
-            if not self.get_type() == "xgb":
-                features = [features]
             predictions = self.clf.predict_proba(features)
             # pred = self.clf.predict(features)
             for i in range(len(self.clf.classes_)):
@@ -302,7 +302,7 @@ class Model:
                 self.clf = pickle.load(handle)
         except FileNotFoundError as fe:
             self.util.error(
-                f"did you forget to store your models? needs: \n[MODEL]\nsave=True\n{fe}"
+                f"Did you forget to store your models? needs: \n[MODEL]\nsave=True\n{fe}"
             )
 
     def load_path(self, path, run, epoch):

nkululeko/models/model_svm.py

@@ -25,6 +25,9 @@ class SVM_model(Model):
             class_weight=class_weight,
         )  # set up the classifier
 
-    def set_C(self, c):
-        """Set the C parameter"""
+    def set_c(self, c):
+        """Set the C parameter."""
         self.clf.C = c
+
+    def get_type(self):
+        return "svm"

nkululeko/multidb.py

@@ -3,23 +3,27 @@
 
 import argparse
 import ast
-import seaborn as sn
-import pandas as pd
-import matplotlib.pyplot as plt
+import configparser
+import os
+
 import matplotlib.cm as cm
+import matplotlib.pyplot as plt
 import numpy as np
-import os
+import pandas as pd
+import seaborn as sn
+
+import nkululeko.glob_conf as glob_conf
+from nkululeko.aug_train import doit as aug_train
 from nkululeko.experiment import Experiment
-import configparser
-from nkululeko.utils.util import Util
 from nkululeko.nkululeko import doit as nkulu
-from nkululeko.aug_train import doit as aug_train
-import nkululeko.glob_conf as glob_conf
+from nkululeko.utils.util import Util
 
 
 def main(src_dir):
-    parser = argparse.ArgumentParser(description="Call the nkululeko framework.")
-    parser.add_argument("--config", default="exp.ini", help="The base configuration")
+    parser = argparse.ArgumentParser(
+        description="Call the nkululeko MULTIDB framework.")
+    parser.add_argument("--config", default="exp.ini",
+                        help="The base configuration")
     args = parser.parse_args()
     if args.config is not None:
         config_file = args.config
@@ -54,7 +58,8 @@ def main(src_dir):
                 dataset = datasets[i]
                 print(f"running {dataset}")
                 if extra_trains:
-                    extra_trains_1 = extra_trains.removeprefix("[").removesuffix("]")
+                    extra_trains_1 = extra_trains.removeprefix(
+                        "[").removesuffix("]")
                     config["DATA"]["databases"] = f"['{dataset}', {extra_trains_1}]"
                     extra_trains_2 = ast.literal_eval(extra_trains)
                     for extra_train in extra_trains_2:
@@ -67,7 +72,8 @@ def main(src_dir):
                 test = datasets[j]
                 print(f"running train: {train}, test: {test}")
                 if extra_trains:
-                    extra_trains_1 = extra_trains.removeprefix("[").removesuffix("]")
+                    extra_trains_1 = extra_trains.removeprefix(
+                        "[").removesuffix("]")
                     config["DATA"][
                         "databases"
                     ] = f"['{train}', '{test}', {extra_trains_1}]"

nkululeko/predict.py

@@ -1,17 +1,34 @@
 # predict.py
-# use some model and add automatically predicted labels to train and test splits, than save as a new dataset
+# use some model and add automatically predicted labels to train and test splits
+# then save as a new dataset
+
+"""This script is used to call the nkululeko PREDICT framework. 
+
+It loads a configuration file, creates a new experiment,
+and performs automatic prediction on the train and test datasets. The predicted labels are added to the datasets and
+saved as a new dataset.
+
+Usage: \n
+    python3 -m nkululeko.predict [--config CONFIG_FILE] \n
+
+Arguments: \n
+    --config (str): The path to the base configuration file (default: exp.ini)
+"""
 
-from nkululeko.experiment import Experiment
-import configparser
-from nkululeko.utils.util import Util
-from nkululeko.constants import VERSION
 import argparse
+import configparser
 import os
 
+from nkululeko.constants import VERSION
+from nkululeko.experiment import Experiment
+from nkululeko.utils.util import Util
+
 
 def main(src_dir):
-    parser = argparse.ArgumentParser(description="Call the nkululeko framework.")
-    parser.add_argument("--config", default="exp.ini", help="The base configuration")
+    parser = argparse.ArgumentParser(
+        description="Call the nkululeko PREDICT framework.")
+    parser.add_argument("--config", default="exp.ini",
+                        help="The base configuration")
     args = parser.parse_args()
     if args.config is not None:
         config_file = args.config
@@ -28,7 +45,9 @@ def main(src_dir):
     config.read(config_file)
     # create a new experiment
     expr = Experiment(config)
-    util = Util("predict")
+    module = "predict"
+    expr.set_module(module)
+    util = Util(module)
     util.debug(
         f"running {expr.name} from config {config_file}, nkululeko version"
         f" {VERSION}"
@@ -39,7 +58,8 @@ def main(src_dir):
 
     # split into train and test
     expr.fill_train_and_tests()
-    util.debug(f"train shape : {expr.df_train.shape}, test shape:{expr.df_test.shape}")
+    util.debug(
+        f"train shape : {expr.df_train.shape}, test shape:{expr.df_test.shape}")
 
     # process the data
     df = expr.autopredict()