nkululeko 0.60.0__tar.gz → 0.61.0__tar.gz

{nkululeko-0.60.0 → nkululeko-0.61.0}/CHANGELOG.md

@@ -1,6 +1,10 @@
 Changelog
 =========
 
+Version 0.61.0
+--------------
+* added HUBERT embeddings
+  
 Version 0.60.0
 --------------
 * some bugfixes

{nkululeko-0.60.0/nkululeko.egg-info → nkululeko-0.61.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.60.0
+Version: 0.61.0
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -228,6 +228,10 @@ Nkululeko can be used under the [MIT license](https://choosealicense.com/license
 Changelog
 =========
 
+Version 0.61.0
+--------------
+* added HUBERT embeddings
+  
 Version 0.60.0
 --------------
 * some bugfixes

nkululeko-0.61.0/nkululeko/constants.py

@@ -0,0 +1 @@
+VERSION = '0.61.0'

nkululeko-0.61.0/nkululeko/feature_extractor.py

@@ -0,0 +1,138 @@
+"""
+feature_extractor.py
+
+Helper class to encapsulate feature extraction methods
+
+"""
+import pandas as pd
+
+from nkululeko.feat_extract.feats_opensmile import Opensmileset
+from nkululeko.util import Util
+
+
+class FeatureExtractor:
+    """
+    Extract acoustic features from audio samples, using several feature extractors (appends the features column-wise)
+    Args:
+        data_df (pandas.DataFrame): dataframe with audiofile paths as index
+        feats_types (array of strings): designations of acoustic feature extractors to be used
+        data_name (string): names of databases that are extracted (for the caching)
+        feats_designation (string): the type of split (train/test), also is used for the cache name.
+    Returns:
+        df (pandas.DataFrame): dataframe with same index as data_df and acoustic features in columns
+    """
+
+    # pandas dataframe to store the features (and indexed with the data from the sets)
+    df = None
+    data_df = None  # dataframe to get audio paths
+
+    # def __init__
+    def __init__(self, data_df, feats_types, data_name, feats_designation):
+        self.data_df = data_df
+        self.data_name = data_name
+        self.feats_types = feats_types
+        self.util = Util("feature_extractor")
+        self.feats_designation = feats_designation
+
+    def extract(self):
+        # feats_types = self.util.config_val_list('FEATS', 'type', ['os'])
+        self.featExtractor = None
+        self.feats = pd.DataFrame()
+        _scale = True
+        for feats_type in self.feats_types:
+            store_name = f"{self.data_name}_{feats_type}"
+            if feats_type == "os":
+                self.featExtractor = Opensmileset(
+                    f"{store_name}_{self.feats_designation}", self.data_df
+                )
+            elif feats_type == "trill":
+                from nkululeko.feat_extract.feats_trill import TRILLset
+                self.featExtractor = TRILLset(
+                    f"{store_name}_{self.feats_designation}", self.data_df
+                )
+            elif feats_type == "wav2vec":
+                from nkululeko.feat_extract.feats_wav2vec2 import Wav2vec2
+                self.featExtractor = Wav2vec2(
+                    f"{store_name}_{self.feats_designation}", self.data_df
+                )
+            elif feats_type in ("hubert", "hubert_ft", "hubert_large", 
+                                "hubert_xlarge", "hubert_xlarge_ft"):
+                from nkululeko.feat_extract.feats_hubert import Hubert
+                self.featExtractor = Hubert(
+                    f"{store_name}_{self.feats_designation}", self.data_df,
+                    feats_type
+                )
+ 
+            elif feats_type == "audmodel":
+                from nkululeko.feat_extract.feats_audmodel import AudModelSet
+                self.featExtractor = AudModelSet(
+                    f"{store_name}_{self.feats_designation}", self.data_df
+                )
+            elif feats_type == "auddim":
+                from nkululeko.feat_extract.feats_audmodel_dim import \
+                    AudModelDimSet
+                self.featExtractor = AudModelDimSet(
+                    f"{store_name}_{self.feats_designation}", self.data_df
+                )
+            elif feats_type == "agender":
+                from nkululeko.feat_extract.feats_agender import \
+                    AudModelAgenderSet
+                self.featExtractor = AudModelAgenderSet(
+                    f"{store_name}_{self.feats_designation}", self.data_df
+                )
+            elif feats_type == "agender_agender":
+                from nkululeko.feat_extract.feats_agender_agender import \
+                    AgenderAgenderSet
+                self.featExtractor = AgenderAgenderSet(
+                    f"{store_name}_{self.feats_designation}", self.data_df
+                )
+            elif feats_type == "snr":
+                from nkululeko.feat_extract.feats_snr import SNRSet
+                self.featExtractor = SNRSet(
+                    f"{store_name}_{self.feats_designation}", self.data_df
+                )
+            elif feats_type == "mos":
+                from nkululeko.feat_extract.feats_mos import MOSSet
+                self.featExtractor = MOSSet(
+                    f"{store_name}_{self.feats_designation}", self.data_df
+                )
+            elif feats_type == "squim":
+                from nkululeko.feat_extract.feats_squim import SQUIMSet
+                self.featExtractor = SQUIMSet(
+                    f"{store_name}_{self.feats_designation}", self.data_df
+                )
+            elif feats_type == "clap":
+                from nkululeko.feat_extract.feats_clap import Clap
+                self.featExtractor = Clap(
+                    f"{store_name}_{self.feats_designation}", self.data_df
+                )
+            elif feats_type == "praat":
+                from nkululeko.feat_extract.feats_praat import Praatset
+                self.featExtractor = Praatset(
+                    f"{store_name}_{self.feats_designation}", self.data_df
+                )
+            elif feats_type == "mld":
+                from nkululeko.feat_extract.feats_mld import MLD_set
+                self.featExtractor = MLD_set(
+                    f"{store_name}_{self.feats_designation}", self.data_df
+                )
+            elif feats_type == "import":
+                from nkululeko.feat_extract.feats_import import Importset
+                self.featExtractor = Importset(
+                    f"{store_name}_{self.feats_designation}", self.data_df
+                )
+            else:
+                self.util.error(f"unknown feats_type: {feats_type}")
+
+            self.featExtractor.extract()
+            self.featExtractor.filter()
+            # remove samples that were not extracted by MLD
+            # self.df_test = self.df_test.loc[self.df_test.index.intersection(featExtractor_test.df.index)]
+            # self.df_train = self.df_train.loc[self.df_train.index.intersection(featExtractor_train.df.index)]
+            self.util.debug(
+                f"{feats_type}: shape : {self.featExtractor.df.shape}")
+            self.feats = pd.concat([self.feats, self.featExtractor.df], axis=1)
+        return self.feats
+
+    def extract_sample(self, signal, sr):
+        return self.featExtractor.extract_sample(signal, sr)

{nkululeko-0.60.0 → nkululeko-0.61.0}/nkululeko/plots.py

@@ -95,6 +95,17 @@ class Plots():
             else:
                 self.util.error(f'plot value counts: the plot distribution descriptor for {att} has more than 2 values')
 
+    def plot_durations(self, df, filename, sample_selection):
+        fig_dir = self.util.get_path('fig_dir')+'../' # one up because of the runs 
+        ax = sns.histplot(df, x='duration', hue='class_label', kde=True)
+        ax.set_title(f'{sample_selection} {df.shape[0]}')
+        ax.set_xlabel(f'duration')
+        ax.set_ylabel(f'number of samples')
+        fig = ax.figure
+        plt.tight_layout()
+        plt.savefig(f'{fig_dir}{filename}_{sample_selection}.{self.format}')
+        plt.close(fig)
+        fig.clear()   
 
     def describe_df(self, name, df, target, filename):
         """Make a stacked barplot of samples and speakers per sex and target values. speaker, gender and target columns must be present"""

{nkululeko-0.60.0 → nkululeko-0.61.0}/nkululeko/segment.py

@@ -7,25 +7,8 @@ from nkululeko.util import Util
 from  nkululeko.constants import VERSION
 import argparse
 import os
-import torch
-import audformat
-from audformat.utils import to_filewise_index
-from audformat import segmented_index
 import pandas as pd 
 
-# initialize the VAD model
-SAMPLING_RATE = 16000
-torch.set_num_threads(1)
-vad_model, vad_utils = torch.hub.load(repo_or_dir='snakers4/silero-vad',
-                    model='silero_vad',
-                    force_reload=False,
-                    onnx=False)
-(get_speech_timestamps,
-save_audio,
-read_audio,
-VADIterator,
-collect_chunks) = vad_utils
-
 def main(src_dir):
     parser = argparse.ArgumentParser(description='Call the nkululeko framework.')
     parser.add_argument('--config', default='exp.ini', help='The base configuration')
@@ -60,7 +43,7 @@ def main(src_dir):
     util.debug(f'train shape : {expr.df_train.shape}, test shape:{expr.df_test.shape}')
 
     # segment
-    segment_target = util.config_val('DATA', 'segment_target', '_seg')
+    segment_target = util.config_val('SEGMENT', 'target', '_seg')
     # this if a specific dataset is to be segmented
     # segment_db = util.config_val('DATA', 'segment', False)
     # if segment_db:
@@ -72,7 +55,8 @@ def main(src_dir):
     #             name = f'{dataset}{segment_target}'
     #             df_seg.to_csv(f'{expr.data_dir}/{name}.csv')
 
-    sample_selection = util.config_val('DATA', 'segment', 'all')
+    segmenter = util.config_val('SEGMENT', 'method', 'silero')
+    sample_selection = util.config_val('SEGMENT', 'sample_selection', 'all')
     if sample_selection=='all':
         df = pd.concat([expr.df_train, expr.df_test])
     elif sample_selection=='train':
@@ -81,26 +65,38 @@ def main(src_dir):
         df = expr.df_test
     else:
         util.error(f'unknown segmentation selection specifier {sample_selection}, should be [all | train | test]')
+    # if "duration" not in df.columns:
+    #     df = df.drop(columns=['duration'], inplace=True)
+    util.debug(f'segmenting {sample_selection}: {df.shape[0]} samples with {segmenter}') 
+    if segmenter=='silero':
+        from nkululeko.segmenting.seg_silero import Silero_segmenter
+        segmenter = Silero_segmenter()
+        df_seg = segmenter.segment_dataframe(df)
+
+    else:
+        util.error(f'unkown segmenter: {segmenter}')
 
-    if "duration" not in df.columns:
-        df = df.drop(columns=['duration'], inplace=True)
-    util.debug(f'segmenting train and test set: {df.shape[0]} samples') 
-    df_seg = segment_dataframe(df) 
     def calc_dur(x):
+        from datetime import datetime
         starts = x[1]
         ends = x[2]
         return (ends - starts).total_seconds()
     df_seg['duration'] = df_seg.index.to_series().map(lambda x:calc_dur(x)) 
+    if "duration" not in df.columns:
+        df['duration'] = df.index.to_series().map(lambda x:calc_dur(x)) 
+    num_before = df.shape[0]
+    num_after = df_seg.shape[0]
     dataname = '_'.join(expr.datasets.keys())
     name = f'{dataname}{segment_target}'
     df_seg.to_csv(f'{expr.data_dir}/{name}.csv')
+    from nkululeko.plots import Plots
+    plots = Plots()
+    plots.plot_durations(df, 'original_durations', sample_selection)
+    plots.plot_durations(df_seg, 'segmented_durations', sample_selection)
     print('')
-    util.debug(f'saved {name}.csv to {expr.data_dir}, {df_seg.shape[0]} samples')
+    util.debug(f'saved {name}.csv to {expr.data_dir}, {num_after} samples (was {num_before})')
     print('DONE')
 
-
-
-
 def get_segmentation(file):
 #    print(f'segmenting {file[0]}')
     print('.', end='')

{nkululeko-0.60.0 → nkululeko-0.61.0/nkululeko.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.60.0
+Version: 0.61.0
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -228,6 +228,10 @@ Nkululeko can be used under the [MIT license](https://choosealicense.com/license
 Changelog
 =========
 
+Version 0.61.0
+--------------
+* added HUBERT embeddings
+  
 Version 0.60.0
 --------------
 * some bugfixes

nkululeko-0.60.0/nkululeko/constants.py

@@ -1 +0,0 @@
-VERSION = '0.60.0'

nkululeko-0.60.0/nkululeko/feature_extractor.py

@@ -1,95 +0,0 @@
-"""
-feature_extractor.py
-
-Helper class to encapsulate feature extraction methods
-
-"""
-import pandas as pd
-from nkululeko.util import Util 
-from nkululeko.feat_extract.feats_opensmile import Opensmileset
-
-
-class FeatureExtractor:
-    """
-    Extract acoustic features from audio samples, using several feature extractors (appends the features column-wise)
-    Args:
-        data_df (pandas.DataFrame): dataframe with audiofile paths as index
-        feats_types (array of strings): designations of acoustic feature extractors to be used
-        data_name (string): names of databases that are extracted (for the caching)
-        feats_designation (string): the type of split (train/test), also is used for the cache name.    
-    Returns:
-        df (pandas.DataFrame): dataframe with same index as data_df and acoustic features in columns  
-    """
-    df = None # pandas dataframe to store the features (and indexed with the data from the sets)
-    data_df = None # dataframe to get audio paths
-
-# def __init__
-    def __init__(self, data_df, feats_types, data_name, feats_designation):
-        self.data_df = data_df
-        self.data_name = data_name
-        self.feats_types = feats_types
-        self.util = Util('feature_extractor')
-        self.feats_designation = feats_designation
-    
-    def extract(self):
-        # feats_types = self.util.config_val_list('FEATS', 'type', ['os'])
-        self.featExtractor = None
-        self.feats= pd.DataFrame()
-        _scale = True
-        for feats_type in self.feats_types:
-            store_name = f'{self.data_name}_{feats_type}'   
-            if feats_type=='os':
-                self.featExtractor = Opensmileset(f'{store_name}_{self.feats_designation}', self.data_df)
-            elif feats_type=='trill':
-                from nkululeko.feat_extract.feats_trill import TRILLset
-                self.featExtractor = TRILLset(f'{store_name}_{self.feats_designation}', self.data_df)
-            elif feats_type=='wav2vec':
-                from nkululeko.feat_extract.feats_wav2vec2 import Wav2vec2
-                self.featExtractor = Wav2vec2(f'{store_name}_{self.feats_designation}', self.data_df)
-            elif feats_type=='audmodel':
-                from nkululeko.feat_extract.feats_audmodel import AudModelSet
-                self.featExtractor = AudModelSet(f'{store_name}_{self.feats_designation}', self.data_df)
-            elif feats_type=='auddim':
-                from nkululeko.feat_extract.feats_audmodel_dim import AudModelDimSet
-                self.featExtractor = AudModelDimSet(f'{store_name}_{self.feats_designation}', self.data_df)
-            elif feats_type=='agender':
-                from nkululeko.feat_extract.feats_agender import AudModelAgenderSet
-                self.featExtractor = AudModelAgenderSet(f'{store_name}_{self.feats_designation}', self.data_df)
-            elif feats_type=='agender_agender':
-                from nkululeko.feat_extract.feats_agender_agender import AgenderAgenderSet
-                self.featExtractor = AgenderAgenderSet(f'{store_name}_{self.feats_designation}', self.data_df)
-            elif feats_type=='snr':
-                from nkululeko.feat_extract.feats_snr import SNRSet
-                self.featExtractor = SNRSet(f'{store_name}_{self.feats_designation}', self.data_df)
-            elif feats_type=='mos':
-                from nkululeko.feat_extract.feats_mos import MOSSet
-                self.featExtractor = MOSSet(f'{store_name}_{self.feats_designation}', self.data_df)
-            elif feats_type=='squim':
-                from nkululeko.feat_extract.feats_squim import SQUIMSet
-                self.featExtractor = SQUIMSet(f'{store_name}_{self.feats_designation}', self.data_df)
-            elif feats_type=='clap':
-                from nkululeko.feat_extract.feats_clap import Clap
-                self.featExtractor = Clap(f'{store_name}_{self.feats_designation}', self.data_df)
-            elif feats_type=='praat':
-                from nkululeko.feat_extract.feats_praat import Praatset
-                self.featExtractor = Praatset(f'{store_name}_{self.feats_designation}', self.data_df)
-            elif feats_type=='mld':
-                from nkululeko.feat_extract.feats_mld import MLD_set
-                self.featExtractor = MLD_set(f'{store_name}_{self.feats_designation}', self.data_df)
-            elif feats_type=='import':
-                from nkululeko.feat_extract.feats_import import Importset
-                self.featExtractor = Importset(f'{store_name}_{self.feats_designation}', self.data_df)
-            else:
-                self.util.error(f'unknown feats_type: {feats_type}')
-
-            self.featExtractor.extract()
-            self.featExtractor.filter()
-            # remove samples that were not extracted by MLD
-            #self.df_test = self.df_test.loc[self.df_test.index.intersection(featExtractor_test.df.index)]
-            #self.df_train = self.df_train.loc[self.df_train.index.intersection(featExtractor_train.df.index)]
-            self.util.debug(f'{feats_type}: shape : {self.featExtractor.df.shape}')
-            self.feats = pd.concat([self.feats, self.featExtractor.df], axis = 1)
-        return self.feats
-
-    def extract_sample(self, signal, sr):
-        return self.featExtractor.extract_sample(signal, sr)