nkululeko 0.89.2__py3-none-any.whl → 0.90.1__py3-none-any.whl

nkululeko/feat_extract/feats_wavlm.py

@@ -8,11 +8,10 @@ import pandas as pd
 import torch
 import torchaudio
 from tqdm import tqdm
-from transformers import Wav2Vec2FeatureExtractor
-from transformers import WavLMModel
+from transformers import Wav2Vec2FeatureExtractor, WavLMModel
 
-from nkululeko.feat_extract.featureset import Featureset
 import nkululeko.glob_conf as glob_conf
+from nkululeko.feat_extract.featureset import Featureset
 
 
 class Wavlm(Featureset):

nkululeko/feat_extract/feats_whisper.py

@@ -1,16 +1,14 @@
 # feats_whisper.py
 import os
 
-import pandas as pd
-import torch
-from transformers import AutoFeatureExtractor
-from transformers import WhisperModel
-
 import audeer
 import audiofile
+import pandas as pd
+import torch
+from transformers import AutoFeatureExtractor, WhisperModel
 
-from nkululeko.feat_extract.featureset import Featureset
 import nkululeko.glob_conf as glob_conf
+from nkululeko.feat_extract.featureset import Featureset
 
 
 class Whisper(Featureset):

nkululeko/feat_extract/featureset.py

@@ -1,8 +1,10 @@
 # featureset.py
+import ast
+
 import pandas as pd
-from nkululeko.utils.util import Util
+
 import nkululeko.glob_conf as glob_conf
-import ast
+from nkululeko.utils.util import Util
 
 
 class Featureset:

nkululeko/feat_extract/feinberg_praat.py

@@ -7,6 +7,7 @@ taken June 23rd 2022.
 import math
 import statistics
 
+import audiofile
 import numpy as np
 import pandas as pd
 import parselmouth
@@ -15,9 +16,6 @@ from scipy.stats.mstats import zscore
 from sklearn.decomposition import PCA
 from tqdm import tqdm
 
-import audiofile
-
-
 # This is the function to measure source acoustics using default male parameters.
 
 

nkululeko/feat_extract/transformer_feature_extractor.py

@@ -0,0 +1,147 @@
+# transformer_feature_extractor.py
+
+import os
+
+import pandas as pd
+import torch
+import torchaudio
+from tqdm import tqdm
+
+import nkululeko.glob_conf as glob_conf
+from nkululeko.feat_extract.featureset import Featureset
+
+
+class TransformerFeatureExtractor(Featureset):
+    def __init__(self, name, data_df, feat_type):
+        super().__init__(name, data_df, feat_type)
+        cuda = "cuda" if torch.cuda.is_available() else "cpu"
+        self.device = self.util.config_val("MODEL", "device", cuda)
+        self.model_initialized = False
+        self.feat_type = feat_type
+
+    def init_model(self):
+        raise NotImplementedError("Subclasses must implement init_model method")
+
+    def get_embeddings(self, signal, sampling_rate, file):
+        try:
+            with torch.no_grad():
+                # Preprocess the input
+                inputs = self.preprocess_input(signal, sampling_rate)
+
+                # Get model outputs
+                outputs = self.model(**inputs)
+
+                # Extract the relevant hidden states
+                hidden_states = self.extract_hidden_states(outputs)
+
+                # Pool the hidden states
+                embeddings = self.pool_hidden_states(hidden_states)
+
+                # Convert to numpy and flatten
+                embeddings = embeddings.cpu().numpy().ravel()
+
+            return embeddings
+
+        except Exception as e:
+            self.util.error(f"Error extracting embeddings for file {file}: {str(e)}")
+            return np.zeros(self.get_embedding_dim())  # Return zero vector on error
+
+    def preprocess_input(self, signal, sampling_rate):
+        # This method should be implemented by subclasses
+        raise NotImplementedError("Subclasses must implement preprocess_input method")
+
+    def extract_hidden_states(self, outputs):
+        # This method should be implemented by subclasses
+        raise NotImplementedError(
+            "Subclasses must implement extract_hidden_states method"
+        )
+
+    def pool_hidden_states(self, hidden_states):
+        # Default implementation: mean pooling over time dimension
+        return torch.mean(hidden_states, dim=1)
+
+    def get_embedding_dim(self):
+        # This method should be implemented by subclasses
+        raise NotImplementedError("Subclasses must implement get_embedding_dim method")
+
+    def extract(self):
+        store = self.util.get_path("store")
+        storage = f"{store}{self.name}.pkl"
+        extract = self.util.config_val("FEATS", "needs_feature_extraction", False)
+        no_reuse = eval(self.util.config_val("FEATS", "no_reuse", "False"))
+        if extract or no_reuse or not os.path.isfile(storage):
+            if not self.model_initialized:
+                self.init_model()
+            self.util.debug(
+                f"extracting {self.feat_type} embeddings, this might take a while..."
+            )
+            emb_series = pd.Series(index=self.data_df.index, dtype=object)
+            for idx, (file, start, end) in enumerate(
+                tqdm(self.data_df.index.to_list())
+            ):
+                signal, sampling_rate = torchaudio.load(
+                    file,
+                    frame_offset=int(start.total_seconds() * 16000),
+                    num_frames=int((end - start).total_seconds() * 16000),
+                )
+                assert sampling_rate == 16000, f"got {sampling_rate} instead of 16000"
+                emb = self.get_embeddings(signal, sampling_rate, file)
+                emb_series[idx] = emb
+            self.df = pd.DataFrame(emb_series.values.tolist(), index=self.data_df.index)
+            self.df.to_pickle(storage)
+            try:
+                glob_conf.config["DATA"]["needs_feature_extraction"] = "false"
+            except KeyError:
+                pass
+        else:
+            self.util.debug(f"reusing extracted {self.feat_type} embeddings")
+            self.df = pd.read_pickle(storage)
+            if self.df.isnull().values.any():
+                self.util.error(
+                    f"got nan: {self.df.shape} {self.df.isnull().sum().sum()}"
+                )
+
+    def extract_sample(self, signal, sr):
+        self.init_model()
+        feats = self.get_embeddings(signal, sr, "no file")
+        return feats
+
+        # for each feature extractor
+        # feats_ast.py
+
+        # class Ast(TransformerFeatureExtractor):
+        #     def preprocess_input(self, signal, sampling_rate):
+        #         inputs = self.processor(signal.numpy(), sampling_rate=sampling_rate, return_tensors="pt")
+        #         return {k: v.to(self.device) for k, v in inputs.items()}
+
+        #     def extract_hidden_states(self, outputs):
+        #         return outputs.last_hidden_state
+
+        #     def get_embedding_dim(self):
+        #         return self.model.config.hidden_size
+
+        # # feats_wav2vec2.py
+
+        # class Wav2vec2(TransformerFeatureExtractor):
+        #     def preprocess_input(self, signal, sampling_rate):
+        #         inputs = self.processor(signal, sampling_rate=sampling_rate, return_tensors="pt")
+        #         return {k: v.to(self.device) for k, v in inputs.items()}
+
+        #     def extract_hidden_states(self, outputs):
+        #         return outputs.last_hidden_state
+
+        #     def get_embedding_dim(self):
+        #         return self.model.config.hidden_size
+
+        # # feats_wavlm.py
+
+        # class Wavlm(TransformerFeatureExtractor):
+        #     def preprocess_input(self, signal, sampling_rate):
+        #         inputs = self.processor(signal, sampling_rate=sampling_rate, return_tensors="pt")
+        #         return {k: v.to(self.device) for k, v in inputs.items()}
+
+        #     def extract_hidden_states(self, outputs):
+        return outputs.last_hidden_state
+
+    # def get_embedding_dim(self):
+    #     return self.model.config.hidden_size

nkululeko/file_checker.py

@@ -1,7 +1,7 @@
-import pandas as pd
-from nkululeko.utils.util import Util
 import os
 
+from nkululeko.utils.util import Util
+
 
 class FileChecker:
     def __init__(self, df):
@@ -10,7 +10,7 @@ class FileChecker:
         self.util.copy_flags(df, self.df)
         check_vad = self.util.config_val("DATA", "check_vad", False)
         if check_vad:
-            self.util.debug(f"This may take a while downloading the VAD model")
+            self.util.debug("This may take a while downloading the VAD model")
             import torch
 
             torch.set_num_threads(1)

nkululeko/filter_data.py

@@ -1,8 +1,10 @@
+import ast
+
 import audformat
 import pandas as pd
+
 import nkululeko.glob_conf as glob_conf
 from nkululeko.utils.util import Util
-import ast
 
 
 class DataFilter:

nkululeko/fixedsegment.py

@@ -0,0 +1,83 @@
+"""
+Segment audio files in a given directory into smaller chunks with a specified length and overlap. This requires the PyDub library to be installed.
+
+Args:
+    input_dir (str): The directory containing the audio files to be segmented.
+    output_dir (str): The directory where the segmented audio files will be saved.
+    segment_length (int): The length of each audio segment in milliseconds.
+    overlap (int): The overlap between adjacent audio segments in milliseconds.
+
+This function will recursively search the input directory for all .wav audio files, and then segment each file into smaller chunks with the specified length and overlap. The segmented audio files will be saved in the output directory, preserving the relative directory structure from the input directory.
+"""
+
+import argparse
+import glob
+from pathlib import Path
+
+from pydub import AudioSegment
+
+
+# list audio files given a directory
+def segment_audio(input_dir, output_dir, segment_length, overlap):
+    # check if input dir exist
+    if not Path(input_dir).exists():
+        print(f"Directory {input_dir} does not exist.")
+        return
+
+    # check if output dir exist, create if not
+    if not Path(output_dir).exists():
+        Path(output_dir).mkdir(parents=True)
+
+    audio_files = glob.glob(str(Path(input_dir) / "**" / "*.wav"), recursive=True)
+
+    for audio_file in audio_files:
+        # segment into 2 seconds with 1 second overlap (default values)
+        audio = AudioSegment.from_file(audio_file)
+
+        segments = []
+
+        for i in range(0, len(audio), segment_length - overlap):
+            segment = audio[i : i + segment_length]
+            segments.append(segment)
+
+        # Path(output_dir).mkdir(exist_ok=True)
+        for i, segment in enumerate(segments):
+            # get relative path from input_dir
+            relative_path = Path(audio_file).relative_to(input_dir)
+            # make output directory if not exist
+            output_subdir = Path(output_dir) / relative_path.parent
+            output_subdir.mkdir(parents=True, exist_ok=True)
+            segment.export(
+                str(output_subdir / f"{Path(audio_file).stem}_{i}.wav"),
+                format="wav",
+            )
+
+    print("DONE.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input_dir", type=str, default="./")
+    # add argument for output_dir
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./segmented_data/",
+    )
+    parser.add_argument(
+        "--segment_length",
+        type=int,
+        default=2000,
+    )
+    parser.add_argument(
+        "--overlap",
+        type=int,
+        default=1000,
+    )
+    args = parser.parse_args()
+    segment_audio(
+        args.input_dir,
+        args.output_dir,
+        segment_length=args.segment_length,
+        overlap=args.overlap,
+    )

nkululeko/models/model.py

@@ -3,13 +3,11 @@ import ast
 import pickle
 import random
 
-from joblib import parallel_backend
 import numpy as np
 import pandas as pd
-from sklearn.model_selection import GridSearchCV
-from sklearn.model_selection import LeaveOneGroupOut
-from sklearn.model_selection import StratifiedKFold
 import sklearn.utils
+from joblib import parallel_backend
+from sklearn.model_selection import GridSearchCV, LeaveOneGroupOut, StratifiedKFold
 
 import nkululeko.glob_conf as glob_conf
 from nkululeko.reporting.reporter import Reporter
@@ -119,7 +117,7 @@ class Model:
         # get unique list of speakers
         speakers = annos["speaker"].unique()
         # check for folds columns
-        if not "fold" in annos.columns:
+        if "fold" not in annos.columns:
             self.util.debug(f"creating random folds for {logo} groups")
             # create a random dictionary of groups
             sdict = {}

nkululeko/models/model_bayes.py

@@ -1,6 +1,7 @@
 # model_bayes.py
 
 from sklearn.naive_bayes import GaussianNB
+
 from nkululeko.models.model import Model
 
 

nkululeko/models/model_cnn.py

@@ -6,23 +6,21 @@ Inspired by code from Su Lei
 """
 
 import ast
-from collections import OrderedDict
 
 import numpy as np
 import pandas as pd
-from PIL import Image
-from sklearn.metrics import recall_score
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-from torch.utils.data import Dataset
 import torchvision.transforms as transforms
+from PIL import Image
+from sklearn.metrics import recall_score
+from torch.utils.data import Dataset
 
 import nkululeko.glob_conf as glob_conf
 from nkululeko.losses.loss_softf1loss import SoftF1Loss
 from nkululeko.models.model import Model
 from nkululeko.reporting.reporter import Reporter
-from nkululeko.utils.util import Util
 
 
 class CNNModel(Model):
@@ -55,7 +53,7 @@ class CNNModel(Model):
             )
         else:
             self.util.error(f"unknown loss function: {criterion}")
-        self.util.debug(f"using model with cross entropy loss function")
+        self.util.debug("using model with cross entropy loss function")
         # set up the model
         # cuda = "cuda" if torch.cuda.is_available() else "cpu"
         self.device = self.util.config_val("MODEL", "device", "cpu")

nkululeko/models/model_gmm.py

@@ -1,8 +1,10 @@
 # model_gmm.py
 
+import pandas as pd
 from sklearn import mixture
+
 from nkululeko.models.model import Model
-import pandas as pd
+
 
 class GMM_model(Model):
     """An GMM model"""
@@ -15,23 +17,25 @@ class GMM_model(Model):
         self.n_components = int(self.util.config_val("MODEL", "GMM_components", "4"))
         covariance_type = self.util.config_val("MODEL", "GMM_covariance_type", "full")
         self.clf = mixture.GaussianMixture(
-            n_components=self.n_components, 
+            n_components=self.n_components,
             covariance_type=covariance_type,
-            random_state = 42,
+            random_state=42,
         )
         # set up the classifier
 
     def get_predictions(self):
-        """Use the predict_proba method of the GaussianMixture model to get 
-        probabilities. Create a DataFrame with these probabilities and return 
+        """Use the predict_proba method of the GaussianMixture model to get
+        probabilities. Create a DataFrame with these probabilities and return
         it along with the predictions."""
         probs = self.clf.predict_proba(self.feats_test)
         preds = self.clf.predict(self.feats_test)
-        
+
         # Convert predictions to a list
         preds = preds.tolist()
-        
+
         # Create a DataFrame for probabilities
-        proba_df = pd.DataFrame(probs, index=self.feats_test.index, columns=range(self.n_components))
-        
+        proba_df = pd.DataFrame(
+            probs, index=self.feats_test.index, columns=range(self.n_components)
+        )
+
         return preds, proba_df

nkululeko/models/model_knn.py

@@ -1,6 +1,7 @@
 # model_knn.py
 
 from sklearn.neighbors import KNeighborsClassifier
+
 from nkululeko.models.model import Model
 
 

nkululeko/models/model_knn_reg.py

@@ -1,6 +1,7 @@
 # model_knn_reg.py
 
 from sklearn.neighbors import KNeighborsRegressor
+
 from nkululeko.models.model import Model
 
 

nkululeko/models/model_lin_reg.py

@@ -1,6 +1,7 @@
 # model_knn_reg.py
 
 from sklearn.linear_model import LinearRegression
+
 from nkululeko.models.model import Model
 
 

nkululeko/models/model_mlp.py

@@ -4,14 +4,13 @@ from collections import OrderedDict
 
 import numpy as np
 import pandas as pd
-from sklearn.metrics import recall_score
 import torch
+from sklearn.metrics import recall_score
 
 import nkululeko.glob_conf as glob_conf
 from nkululeko.losses.loss_softf1loss import SoftF1Loss
 from nkululeko.models.model import Model
 from nkululeko.reporting.reporter import Reporter
-from nkululeko.utils.util import Util
 
 
 class MLPModel(Model):
@@ -44,7 +43,7 @@ class MLPModel(Model):
             )
         else:
             self.util.error(f"unknown loss function: {criterion}")
-        self.util.debug(f"using model with cross entropy loss function")
+        self.util.debug("using model with cross entropy loss function")
         # set up the model, use GPU if availabe
         cuda = "cuda" if torch.cuda.is_available() else "cpu"
         self.device = self.util.config_val("MODEL", "device", cuda)

nkululeko/models/model_mlp_regression.py

@@ -1,15 +1,10 @@
 # model_mlp.py
 import ast
 from collections import OrderedDict
-import os
 
 import numpy as np
 import torch
-
-from audmetric import concordance_cc
-from audmetric import mean_absolute_error
-from audmetric import mean_squared_error
-from traitlets import default
+from audmetric import concordance_cc, mean_absolute_error, mean_squared_error
 
 import nkululeko.glob_conf as glob_conf
 from nkululeko.losses.loss_ccc import ConcordanceCorCoeff

nkululeko/models/model_svm.py

@@ -1,7 +1,7 @@
 # model_svm.py
 
-import random
 from sklearn import svm
+
 from nkululeko.models.model import Model
 
 
@@ -25,7 +25,7 @@ class SVM_model(Model):
             gamma="scale",
             probability=True,
             class_weight=class_weight,
-            random_state=42,    # for consistent result
+            random_state=42,  # for consistent result
         )  # set up the classifier
 
     def set_c(self, c):

nkululeko/models/model_svr.py

@@ -1,6 +1,7 @@
 # svrmodel.py
 
 from sklearn import svm
+
 from nkululeko.models.model import Model
 
 

nkululeko/models/model_tree.py

@@ -1,6 +1,7 @@
 # model_tree.py
 
 from sklearn.tree import DecisionTreeClassifier
+
 from nkululeko.models.model import Model
 
 
@@ -12,6 +13,4 @@ class Tree_model(Model):
     def __init__(self, df_train, df_test, feats_train, feats_test):
         super().__init__(df_train, df_test, feats_train, feats_test)
         self.name = "tree"
-        self.clf = DecisionTreeClassifier(
-            random_state=42
-        )  # set up the classifier
+        self.clf = DecisionTreeClassifier(random_state=42)  # set up the classifier

nkululeko/models/model_tree_reg.py

@@ -1,6 +1,7 @@
 # model_tree_reg.py
 
 from sklearn.tree import DecisionTreeRegressor
+
 from nkululeko.models.model import Model