nkululeko 0.59.1__py3-none-any.whl → 0.61.0__py3-none-any.whl

nkululeko/feats_oxbow.py

@@ -1,87 +0,0 @@
-# feats_oxbow.py
-
-from nkululeko.util import Util
-from nkululeko.featureset import Featureset
-import os
-import pandas as pd
-import opensmile
-
-class Openxbow(Featureset):
-    """Class to extract openXBOW processed opensmile features (https://github.com/openXBOW)"""
-
-    def __init__(self, name, data_df, is_train = False):
-        """Constructor. is_train is needed to distinguish from test/dev sets, because they use the codebook from the training"""
-        super().__init__(name, data_df)
-        self.is_train = is_train
-
-    def extract(self):
-        """Extract the features or load them from disk if present."""
-        self.featset = self.util.config_val('FEATS', 'set', 'eGeMAPSv02')
-        self.feature_set = eval(f'opensmile.FeatureSet.{self.featset}')
-        store = self.util.get_path('store')
-        storage = f'{store}{self.name}_{self.featset}.pkl'
-        extract = self.util.config_val('FEATS', 'needs_feature_extraction', False)
-        no_reuse = eval(self.util.config_val('FEATS', 'no_reuse', 'False'))
-        if extract or no_reuse or not os.path.isfile(storage):
-            # extract smile features first
-            self.util.debug('extracting openSmile features, this might take a while...')
-            smile = opensmile.Smile(
-                feature_set= self.feature_set,
-                feature_level=opensmile.FeatureLevel.LowLevelDescriptors,
-                num_workers=5,)
-            if isinstance(self.data_df.index, pd.MultiIndex):
-                is_multi_index = True
-                smile_df = smile.process_index(self.data_df.index)
-            else:
-                smile_df = smile.process_files(self.data_df.index)
-            smile_df.index = smile_df.index.droplevel(1)
-            smile_df.index = smile_df.index.droplevel(1)
-            # compute xbow features
-            # set some file names on disk
-            lld_name, xbow_name, codebook_name = 'llds.csv', 'xbow.csv', 'xbow_codebook'
-            # save the smile features
-            smile_df.to_csv(lld_name, sep=';', header=False)
-            # get the path of the xbow java jar file
-            xbow_path = self.util.config_val('FEATS', 'xbow.model', '../openXBOW/')
-            # get the size of the codebook
-            size = self.util.config_val('FEATS', 'size', 500)
-            # get the number of assignements
-            assignments = self.util.config_val('FEATS', 'assignments', 10)
-            # differentiate between train and test
-            if self.is_train:
-                # store the codebook
-                os.system(f'java -jar {xbow_path}openXBOW.jar -i {lld_name} -standardizeInput -log \
-                    -o {xbow_name} -size {size} -a {assignments} -B {codebook_name}')
-            else:
-                # use the codebook
-                os.system(f'java -jar {xbow_path}openXBOW.jar -i {lld_name} \
-                    -o {xbow_name} -b {codebook_name}')
-            # read in the result from disk
-            xbow_df = pd.read_csv(xbow_name, sep=';', header=None)
-            # set the index
-            xbow_df = xbow_df.set_index(self.data_df.index)
-            # check if smile features should be added
-            with_os = self.util.config_val('FEATS', 'with_os', False)
-            if with_os:
-                # extract smile functionals
-                self.util.debug('extracting openSmile functionals, this might take a while...')
-                smile = opensmile.Smile(
-                    feature_set= opensmile.FeatureSet.eGeMAPSv02, # always use eGemaps for this
-                    feature_level=opensmile.FeatureLevel.Functionals,
-                    num_workers=5,)
-                if isinstance(self.data_df.index, pd.MultiIndex):
-                    is_multi_index = True
-                    smile_df = smile.process_index(self.data_df.index)
-                else:
-                    smile_df = smile.process_files(self.data_df.index)
-                # drop the multi index
-                smile_df.index = smile_df.index.droplevel(1)
-                smile_df.index = smile_df.index.droplevel(1)
-                xbow_df = xbow_df.join(smile_df)
-            # in any case, store to disk for later use
-            xbow_df.to_pickle(storage) 
-            # and assign to be the "official" feature set
-            self.df = xbow_df           
-        else:
-            self.util.debug('reusing extracted OS features.')
-            self.df = pd.read_pickle(storage)

nkululeko/feats_praat.py

@@ -1,72 +0,0 @@
-# feats_praat.py
-from nkululeko.featureset import Featureset
-import os
-import pandas as pd
-import nkululeko.glob_conf as glob_conf
-from nkululeko import feinberg_praat
-import ast
-
-class Praatset(Featureset):
-    """
-    a feature extractor for the Praat software, based on 
-    David R. Feinberg's Praat scripts for the parselmouth python interface.
-    https://osf.io/6dwr3/
-
-    """
-    def __init__(self, name, data_df):
-        super().__init__(name, data_df)
-
-    def extract(self):
-        """Extract the features based on the initialized dataset or re-open them when found on disk."""
-        store = self.util.get_path('store')
-        store_format = self.util.config_val('FEATS', 'store_format', 'pkl')
-        storage = f'{store}{self.name}.{store_format}'
-        extract = self.util.config_val('FEATS', 'needs_feature_extraction', False)
-        no_reuse = eval(self.util.config_val('FEATS', 'no_reuse', 'False'))
-        if extract or no_reuse or not os.path.isfile(storage):
-            self.util.debug('extracting Praat features, this might take a while...')
-            self.df = feinberg_praat.compute_features(self.data_df.index)
-            self.df = self.df.set_index(self.data_df.index)
-            for i, col in enumerate(self.df.columns):
-                if self.df[col].isnull().values.any():
-                    self.util.debug(f'{col} includes {self.df[col].isnull().sum()} nan, inserting mean values')
-                    self.df[col] = self.df[col].fillna(self.df[col].mean())
-
-            self.util.write_store(self.df, storage, store_format)
-            try:
-                glob_conf.config['DATA']['needs_feature_extraction'] = 'false'
-            except KeyError:
-                pass
-        else:
-            self.util.debug(f'reusing extracted Praat features: {storage}.')
-            self.df = self.util.get_store(storage, store_format)
-        self.util.debug(f'praat feature names: {self.df.columns}')
-        self.df = self.df.astype(float)
-
-
-
-    def extract_sample(self, signal, sr):
-        self.util.error('feats_praat: extracting single samples not implemented yet')
-        feats = None
-        return feats
-    
-    def filter(self):
-        # use only the features that are indexed in the target dataframes
-        self.df = self.df[self.df.index.isin(self.data_df.index)]
-        try: 
-            # use only some features
-            selected_features = ast.literal_eval(glob_conf.config['FEATS']['praat.features'])
-            self.util.debug(f'selecting features from Praat: {selected_features}')
-            sel_feats_df = pd.DataFrame()
-            hit = False
-            for feat in selected_features:
-                try:
-                    sel_feats_df[feat] = self.df[feat]
-                    hit = True
-                except KeyError:
-                    pass
-            if hit:
-                self.df = sel_feats_df
-                self.util.debug(f'new feats shape after selecting Praat features: {self.df.shape}')
-        except KeyError:
-            pass

nkululeko/feats_snr.py

@@ -1,63 +0,0 @@
-""" feats_snr.py
-    Estimate snr (signal to noise ratio as acoustic features)
-"""
-from nkululeko.util import Util
-from nkululeko.featureset import Featureset
-from nkululeko.estimate_snr import SNREstimator
-import os
-import pandas as pd
-import os
-import nkululeko.glob_conf as glob_conf
-import audiofile
-
-class SNRSet(Featureset):
-    """Class to estimate snr"""
-
-    def __init__(self, name, data_df):
-        """Constructor. """
-        super().__init__(name, data_df)
-
-    def extract(self):
-        """Estimate the features or load them from disk if present."""
-        store = self.util.get_path('store')
-        store_format = self.util.config_val('FEATS', 'store_format', 'pkl')
-        storage = f'{store}{self.name}.{store_format}'
-        extract = self.util.config_val('FEATS', 'needs_feature_extraction', False)
-        no_reuse = eval(self.util.config_val('FEATS', 'no_reuse', 'False'))
-        if extract or no_reuse or not os.path.isfile(storage):
-            self.util.debug('estimating SNR, this might take a while...')
-            snr_series = pd.Series(index = self.data_df.index, dtype=object)
-            for idx, (file, start, end) in enumerate(self.data_df.index.to_list()):
-                signal, sampling_rate = audiofile.read(file, offset=start.total_seconds(), duration=(end-start).total_seconds(), always_2d=True)
-                snr = self.get_snr(signal[0], sampling_rate)
-                snr_series[idx] = snr
-                if idx%10==0:
-                    print('.', end='')
-            print('')
-            self.df = pd.DataFrame(snr_series.values.tolist(), index=self.data_df.index) 
-            self.df.columns = ['snr']
-            self.util.write_store(self.df, storage, store_format)
-            try:
-                glob_conf.config['DATA']['needs_feature_extraction'] = 'false'
-            except KeyError:
-                pass
-        else:
-            self.util.debug('reusing estimated SNR values')
-            self.df = self.util.get_store(storage, store_format)
-
-    def get_snr(self, signal, sampling_rate):
-        r"""Estimate SNR from raw audio signal.
-        Args:
-            signal: audio signal
-            sampling_rate: sample rate
-        Returns
-            snr: estimated signal to noise ratio
-        """
-        snr_estimator = SNREstimator(signal, sampling_rate)
-        estimated_snr, log_energies, energy_threshold_low, energy_threshold_high = snr_estimator.estimate_snr()
-        return estimated_snr
-    
-    def extract_sample(self, signal, sr):
-        self.init_model()
-        feats = self.get_snr(signal, sr)
-        return feats

nkululeko/feats_squim.py

@@ -1,99 +0,0 @@
-""" feats_squim.py
-predict SQUIM ( SPEECH QUALITY AND INTELLIGIBILITY
-MEASURES) features
-
-
-    Wideband Perceptual Estimation of Speech Quality (PESQ) [2]
-    Short-Time Objective Intelligibility (STOI) [3]
-    Scale-Invariant Signal-to-Distortion Ratio (SI-SDR) [4]
-
-
-adapted from
-from https://pytorch.org/audio/main/tutorials/squim_tutorial.html#sphx-glr-tutorials-squim-tutorial-py
-paper: https://arxiv.org/pdf/2304.01448.pdf
-
-needs 
-pip uninstall -y torch torchvision torchaudio
-pip install --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cpu
-
-"""
-from nkululeko.util import Util
-from nkululeko.featureset import Featureset
-import os
-import pandas as pd
-import os
-import nkululeko.glob_conf as glob_conf
-import audiofile
-import torch
-import torchaudio
-from torchaudio.pipelines import SQUIM_OBJECTIVE
-
-class SQUIMSet(Featureset):
-    """Class to predict SQUIM features
-    
-    """
-
-    def __init__(self, name, data_df):
-        """Constructor. is_train is needed to distinguish from test/dev sets, because they use the codebook from the training"""
-        super().__init__(name, data_df)
-        self.device = self.util.config_val('MODEL', 'device', 'cpu')
-        self.model_initialized = False
-
-
-    def init_model(self):
-        # load model
-        self.util.debug('loading model...')
-        self.objective_model = SQUIM_OBJECTIVE.get_model()
-        self.model_initialized = True
-
-
-    def extract(self):
-        """Extract the features or load them from disk if present."""
-        store = self.util.get_path('store')
-        store_format = self.util.config_val('FEATS', 'store_format', 'pkl')
-        storage = f'{store}{self.name}.{store_format}'
-        extract = self.util.config_val('FEATS', 'needs_feature_extraction', False)
-        no_reuse = eval(self.util.config_val('FEATS', 'no_reuse', 'False'))
-        if extract or no_reuse or not os.path.isfile(storage):
-            if not self.model_initialized:
-                self.init_model()
-            self.util.debug('predicting SQUIM, this might take a while...')
-            emb_series = pd.Series(index = self.data_df.index, dtype=object)
-            length = len(self.data_df.index)
-            for idx, (file, start, end) in enumerate(self.data_df.index.to_list()):
-                signal, sampling_rate = audiofile.read(file, offset=start.total_seconds(), duration=(end-start).total_seconds(), always_2d=True)
-                emb = self.get_embeddings(signal, sampling_rate)
-                emb_series[idx] = emb
-                if idx%10==0:
-                    self.util.debug(f'SQUIM: {idx} of {length} done')
-            self.df = pd.DataFrame(emb_series.values.tolist(), index=self.data_df.index) 
-            self.df.columns = ['pesq', 'sdr', 'stoi']
-            self.util.write_store(self.df, storage, store_format)
-            try:
-                glob_conf.config['DATA']['needs_feature_extraction'] = 'false'
-            except KeyError:
-                pass
-        else:
-            self.util.debug('reusing predicted SQUIM values')
-            self.df = self.util.get_store(storage, store_format)
-            if self.df.isnull().values.any():
-                nanrows = self.df.columns[self.df.isna().any()].tolist()
-                print(nanrows)
-                self.util.error(f'got nan: {self.df.shape} {self.df.isnull().sum().sum()}')
-
-
-    def get_embeddings(self, signal, sampling_rate):
-        tmp_audio_name = 'squim_audio_tmp.wav'
-        audiofile.write(tmp_audio_name, signal, sampling_rate)
-        WAVEFORM_SPEECH, SAMPLE_RATE_SPEECH = torchaudio.load(tmp_audio_name)
-        with torch.no_grad():
-            stoi_hyp, pesq_hyp, si_sdr_hyp = self.objective_model(WAVEFORM_SPEECH)
-        pesq = float(pesq_hyp[0].numpy())
-        stoi = float(stoi_hyp[0].numpy())
-        sdr = float(si_sdr_hyp[0].numpy())
-        return pesq, sdr, stoi
- 
-    def extract_sample(self, signal, sr):
-        self.init_model()
-        feats = self.get_embeddings(signal, sr)
-        return feats

nkululeko/feats_trill.py

@@ -1,74 +0,0 @@
-# feats_trill.py
-from numpy.core.numeric import tensordot
-from nkululeko.featureset import Featureset
-import pandas as pd
-from nkululeko.util import Util 
-import nkululeko.glob_conf as glob_conf
-import audiofile as af
-import os
-import tensorflow as tf
-# Import TF 2.X and make sure we're running eager.
-assert tf.executing_eagerly()
-import tensorflow_hub as hub
-
-class TRILLset(Featureset):
-    """A feature extractor for the Google TRILL embeddings"""
-    """https://ai.googleblog.com/2020/06/improving-speech-representations-and.html"""
-
-    # Initialization of the class
-    def __init__(self, name, data_df):
-        """
-        Initialize the class with name, data and Util instance
-        Also loads the model from hub
-        
-        :param name: Name of the class
-        :type name: str
-        :param data_df: Data of the class
-        :type data_df: DataFrame
-        :return: None
-        """
-        super().__init__(name, data_df)   
-        # Load the model from the configured path
-        model_path = self.util.config_val('FEATS', 'trill.model', \
-            'https://tfhub.dev/google/nonsemantic-speech-benchmark/trill/3')
-        self.module = hub.load(model_path)
-
-
-    def extract(self):
-        store = self.util.get_path('store')
-        storage = f'{store}{self.name}.pkl'
-        extract = self.util.config_val('FEATS', 'needs_feature_extraction', False)
-        no_reuse = eval(self.util.config_val('FEATS', 'no_reuse', 'False'))
-        if extract or no_reuse or not os.path.isfile(storage):
-            self.util.debug('extracting TRILL embeddings, this might take a while...')
-            emb_series = pd.Series(index = self.data_df.index, dtype=object)
-            length = len(self.data_df.index)
-            for idx, file in enumerate(self.data_df.index.get_level_values(0)):
-                emb = self.getEmbeddings(file)
-                emb_series[idx] = emb
-                if idx%10==0:
-                    self.util.debug(f'TRILL: {idx} of {length} done')
-            self.df = pd.DataFrame(emb_series.values.tolist(), index=self.data_df.index) 
-            self.df.to_pickle(storage)
-            try:
-                glob_conf.config['DATA']['needs_feature_extraction'] = 'false'
-            except KeyError:
-                pass
-        else:
-            self.util.debug('reusing extracted TRILL embeddings')
-            self.df = pd.read_pickle(storage)
-
-    def embed_wav(self, wav): 
-        if len(wav.shape) > 1:
-            wav = tf.reduce_mean(wav, axis=0)
-            
-        emb_dict = self.module(samples=wav, sample_rate=tf.constant(16000))
-        return emb_dict['embedding']
-
-    def getEmbeddings(self, file):
-        wav = af.read(file)[0]
-        wav = tf.convert_to_tensor(wav)
-        emb_short = self.embed_wav(wav)
-        # you get one embedding per frame, we use the mean for all the frames
-        emb_short = emb_short.numpy().mean(axis=0)
-        return emb_short

nkululeko/feats_wav2vec2.py

@@ -1,94 +0,0 @@
-# feats_wav2vec2.py
-
-from nkululeko.util import Util
-from nkululeko.featureset import Featureset
-import os
-import pandas as pd
-import os
-import nkululeko.glob_conf as glob_conf
-import transformers
-from transformers.models.wav2vec2.modeling_wav2vec2 import Wav2Vec2Model
-import torch
-
-import audiofile
-
-class Wav2vec2(Featureset):
-    """Class to extract wav2vec2 embeddings (https://huggingface.co/facebook/wav2vec2-large-robust-ft-swbd-300h)"""
-
-    def __init__(self, name, data_df):
-        """Constructor. is_train is needed to distinguish from test/dev sets, because they use the codebook from the training"""
-        super().__init__(name, data_df)
-        self.device = self.util.config_val('MODEL', 'device', 'cpu')
-        self.model_initialized = False
-
-
-
-    def init_model(self):
-        # load model
-        self.util.debug('loading wav2vec model...')
-        model_path = self.util.config_val('FEATS', 'wav2vec.model', 'wav2vec2-large-robust-ft-swbd-300h')
-        self.processor = transformers.Wav2Vec2Processor.from_pretrained(model_path)
-        self.model = Wav2Vec2Model.from_pretrained(model_path).to(self.device)
-        print(f'intialized vec model on {self.device}')
-        self.model.eval()
-        self.model_initialized = True
-
-
-    def extract(self):
-        """Extract the features or load them from disk if present."""
-        store = self.util.get_path('store')
-        storage = f'{store}{self.name}.pkl'
-        extract = self.util.config_val('FEATS', 'needs_feature_extraction', False)
-        no_reuse = eval(self.util.config_val('FEATS', 'no_reuse', 'False'))
-        if extract or no_reuse or not os.path.isfile(storage):
-            if not self.model_initialized:
-                self.init_model()
-            self.util.debug('extracting wav2vec2 embeddings, this might take a while...')
-            emb_series = pd.Series(index = self.data_df.index, dtype=object)
-            length = len(self.data_df.index)
-            for idx, (file, start, end) in enumerate(self.data_df.index.to_list()):
-                signal, sampling_rate = audiofile.read(file, offset=start.total_seconds(), duration=(end-start).total_seconds(), always_2d=True)
-                #signal, sampling_rate = audiofile.read(audio_path, always_2d=True)
-                emb = self.get_embeddings(signal, sampling_rate)
-                emb_series[idx] = emb
-                if idx%10==0:
-                    self.util.debug(f'Wav2vec2: {idx} of {length} done')
-            self.df = pd.DataFrame(emb_series.values.tolist(), index=self.data_df.index) 
-            self.df.to_pickle(storage)
-            try:
-                glob_conf.config['DATA']['needs_feature_extraction'] = 'false'
-            except KeyError:
-                pass
-        else:
-            self.util.debug('reusing extracted wav2vec2 embeddings')
-            self.df = pd.read_pickle(storage)
-            if self.df.isnull().values.any():
-                nanrows = self.df.columns[self.df.isna().any()].tolist()
-                print(nanrows)
-                self.util.error(f'got nan: {self.df.shape} {self.df.isnull().sum().sum()}')
-
-
-    def get_embeddings(self, signal, sampling_rate):
-        r"""Extract embeddings from raw audio signal."""
-        with torch.no_grad():
-            # run through processor to normalize signal
-            # always returns a batch, so we just get the first entry
-            # then we put it on the device
-            y = self.processor(signal, sampling_rate=sampling_rate)
-            y = y['input_values'][0]
-            y = torch.from_numpy(y).to(self.device)
-        
-            # run through model
-            # first entry contains hidden state
-            y = self.model(y)[0]
-        
-            # pool result and convert to numpy
-            y = torch.mean(y, dim=1)
-            y = y.detach().cpu().numpy()
-    
-        return y.flatten()
-
-    def extract_sample(self, signal, sr):
-        self.init_model()
-        feats = self.get_embeddings(signal, sr)
-        return feats

nkululeko/featureset.py

@@ -1,41 +0,0 @@
-# featureset.py
-import pandas as pd
-from nkululeko.util import Util 
-import nkululeko.glob_conf as glob_conf
-import ast
-
-class Featureset:
-    name = '' # designation
-    df = None # pandas dataframe to store the features (and indexed with the data from the sets)
-    data_df = None # dataframe to get audio paths
-
-
-    def __init__(self, name, data_df):
-        self.name = name
-        self.data_df = data_df
-        self.util = Util('featureset')
-
-    def extract(self):
-        pass
-
-    def filter(self):
-        # use only the features that are indexed in the target dataframes
-        self.df = self.df[self.df.index.isin(self.data_df.index)]
-        try: 
-            # use only some features
-            selected_features = ast.literal_eval(glob_conf.config['FEATS']['features'])
-            self.util.debug(f'selecting features: {selected_features}')
-            sel_feats_df = pd.DataFrame()
-            hit = False
-            for feat in selected_features:
-                try:
-                    sel_feats_df[feat] = self.df[feat]
-                    hit = True
-                except KeyError:
-                    pass
-            if hit:
-                self.df = sel_feats_df
-                self.util.debug(f'new feats shape after selecting features: {self.df.shape}')
-        except KeyError:
-            pass
-