nkululeko 0.91.2__tar.gz → 0.92.0__tar.gz

{nkululeko-0.91.2 → nkululeko-0.92.0}/CHANGELOG.md

@@ -1,6 +1,14 @@
 Changelog
 =========
 
+Version 0.92.0
+--------------
+* added first version of automatic speaker prediction/segmentation
+
+Version 0.91.3
+--------------
+* some additions for robustness
+
 Version 0.91.2
 --------------
 * making lint work by excluding constants from check

{nkululeko-0.91.2/nkululeko.egg-info → nkululeko-0.92.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.91.2
+Version: 0.92.0
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -355,6 +355,14 @@ F. Burkhardt, Johannes Wagner, Hagen Wierstorf, Florian Eyben and Björn Schulle
 Changelog
 =========
 
+Version 0.92.0
+--------------
+* added first version of automatic speaker prediction/segmentation
+
+Version 0.91.3
+--------------
+* some additions for robustness
+
 Version 0.91.2
 --------------
 * making lint work by excluding constants from check

nkululeko-0.92.0/nkululeko/autopredict/ap_sid.py

@@ -0,0 +1,48 @@
+""""
+A predictor for sid - Speaker ID.
+"""
+
+import numpy as np
+from pyannote.audio import Pipeline
+import torch
+
+from nkululeko.feature_extractor import FeatureExtractor
+import nkululeko.glob_conf as glob_conf
+from nkululeko.utils.util import Util
+
+
+class SIDPredictor:
+    """SIDPredictor.
+
+    predicting speaker id.
+    """
+
+    def __init__(self, df):
+        self.df = df
+        self.util = Util("sidPredictor")
+        hf_token = self.util.config_val("Model", "hf_token", None)
+        if hf_token is None:
+            self.util.error(
+                "speaker id prediction needs huggingface token: [MODEL][hf_token]"
+            )
+        self.pipeline = Pipeline.from_pretrained(
+            "pyannote/speaker-diarization-3.1",
+            use_auth_token=hf_token,
+        )
+        device = self.util.config_val("Model", "device", "cpu")
+        self.pipeline.to(torch.device(device))
+
+    def predict(self, split_selection):
+        self.util.debug(f"estimating speaker id for {split_selection} samples")
+        return_df = self.df.copy()
+        # @todo
+        # 1) concat all audio files
+        # 2) get segmentations with pyannote
+        # 3) map pyannote segments with orginal ones and assign speaker id
+
+        return return_df
+
+    def concat_files(self, df):
+        pass
+        # todo
+        # please use https://audeering.github.io/audiofile/usage.html#read-a-file

{nkululeko-0.91.2 → nkululeko-0.92.0}/nkululeko/constants.py

@@ -1,2 +1,2 @@
-VERSION="0.91.2"
+VERSION="0.92.0"
 SAMPLING_RATE = 16000

{nkululeko-0.91.2 → nkululeko-0.92.0}/nkululeko/data/dataset_csv.py

@@ -39,6 +39,8 @@ class Dataset_CSV(Dataset):
             df = audformat.utils.read_csv(data_file)
         if isinstance(df, pd.Series):
             df = df.to_frame()
+        elif isinstance(df, pd.Index):
+            df = pd.DataFrame(index=df)
         rename_cols = self.util.config_val_data(self.name, "colnames", False)
         if rename_cols:
             col_dict = ast.literal_eval(rename_cols)
@@ -78,7 +80,11 @@ class Dataset_CSV(Dataset):
 
         self.df = df
         self.db = None
-        self.got_target = True
+        target = self.util.config_val("DATA", "target", None)
+        if target is not None:
+            self.got_target = True
+        else:
+            self.got_target = False
         self.is_labeled = self.got_target
         self.start_fresh = eval(self.util.config_val("DATA", "no_reuse", "False"))
         is_index = False

{nkululeko-0.91.2 → nkululeko-0.92.0}/nkululeko/experiment.py

@@ -125,7 +125,15 @@ class Experiment:
         # df = pd.read_csv(storage, header=0, index_col=[0,1,2])
         # df.index.set_levels(pd.to_timedelta(df.index.levels[1]), level=1)
         # df.index.set_levels(pd.to_timedelta(df.index.levels[2]), level=2)
-        df = audformat.utils.read_csv(storage)
+        try:
+            df = audformat.utils.read_csv(storage)
+        except ValueError:
+            # split might be empty
+            return pd.DataFrame()
+        if isinstance(df, pd.Series):
+            df = df.to_frame()
+        elif isinstance(df, pd.Index):
+            df = pd.DataFrame(index=df)
         df.is_labeled = True if self.target in df else False
         # print(df.head())
         return df
@@ -431,7 +439,12 @@ class Experiment:
             )
         targets = self.util.config_val_list("PREDICT", "targets", ["gender"])
         for target in targets:
-            if target == "gender":
+            if target == "speaker":
+                from nkululeko.autopredict.ap_sid import SIDPredictor
+
+                predictor = SIDPredictor(df)
+                df = predictor.predict(sample_selection)
+            elif target == "gender":
                 from nkululeko.autopredict.ap_gender import GenderPredictor
 
                 predictor = GenderPredictor(df)

{nkululeko-0.91.2 → nkululeko-0.92.0}/nkululeko/plots.py

@@ -305,6 +305,9 @@ class Plots:
         except AttributeError as ae:
             self.util.warn(ae)
             ax = sns.histplot(df, x="duration", kde=True)
+        except ValueError as error:
+            self.util.warn(error)
+            ax = sns.histplot(df, x="duration", kde=True)
         min = self.util.to_3_digits(df.duration.min())
         max = self.util.to_3_digits(df.duration.max())
         title = f"Duration distr. for {sample_selection} {df.shape[0]}. min={min}, max={max}"

{nkululeko-0.91.2 → nkululeko-0.92.0}/nkululeko/predict.py

@@ -2,7 +2,7 @@
 # use some model and add automatically predicted labels to train and test splits
 # then save as a new dataset
 
-"""This script is used to call the nkululeko PREDICT framework. 
+r"""This script is used to call the nkululeko PREDICT framework.
 
 It loads a configuration file, creates a new experiment,
 and performs automatic prediction on the train and test datasets. The predicted labels are added to the datasets and

{nkululeko-0.91.2 → nkululeko-0.92.0}/nkululeko/segment.py

@@ -1,5 +1,4 @@
-"""
-Segments the samples in the dataset into chunks based on voice activity detection using SILERO VAD [1].
+"""Segments the samples in the dataset into chunks based on voice activity detection using SILERO VAD [1].
 
 The segmentation results are saved to a file, and the distributions of the original and
 segmented durations are plotted.
@@ -15,7 +14,7 @@ Example:
 
 References:
     [1] https://github.com/snakers4/silero-vad
-    
+    [2] https://github.com/pyannote/pyannote-audio
 """
 
 import argparse
@@ -83,12 +82,15 @@ def main():
 
         segmenter = Silero_segmenter()
         df_seg = segmenter.segment_dataframe(df)
+    elif segmenter == "pyannote":
+        from nkululeko.segmenting.seg_pyannote import Pyannote_segmenter
 
+        segmenter = Pyannote_segmenter(config)
+        df_seg = segmenter.segment_dataframe(df)
     else:
-        util.error(f"unkown segmenter: {segmenter}")
+        util.error(f"unknown segmenter: {segmenter}")
 
     def calc_dur(x):
-
         starts = x[1]
         ends = x[2]
         return (ends - starts).total_seconds()
@@ -110,13 +112,11 @@ def main():
     )
     print("")
     # remove encoded labels
-    target = util.config_val("DATA", "target", "emotion")
+    target = util.config_val("DATA", "target", None)
     if "class_label" in df_seg.columns:
         df_seg = df_seg.drop(columns=[target])
         df_seg = df_seg.rename(columns={"class_label": target})
     # save file
-    # dataname = "_".join(expr.datasets.keys())
-    # name = f"{dataname}{segment_target}"
     df_seg.to_csv(f"{expr.data_dir}/{segmented_file}")
     util.debug(
         f"saved {segmented_file} to {expr.data_dir}, {num_after} samples (was"

nkululeko-0.92.0/nkululeko/segmenting/seg_pyannote.py

@@ -0,0 +1,129 @@
+"""seg_pyannote.py.
+
+Segment a dataset with the Pyannote segmenter.
+Also adds speaker ids to the segments.
+
+"""
+
+import pandas as pd
+from pyannote.audio import Pipeline
+import torch
+from tqdm import tqdm
+
+import audformat
+from audformat import segmented_index
+
+from nkululeko.utils.util import Util
+
+
+SAMPLING_RATE = 16000
+
+
+class Pyannote_segmenter:
+    def __init__(self, not_testing=True):
+        # initialize the VAD model
+        torch.set_num_threads(1)
+        self.no_testing = not_testing
+        self.util = Util("pyannote_segmenter")
+        hf_token = self.util.config_val("MODEL", "hf_token", None)
+        if hf_token is None:
+            self.util.error(
+                "speaker id prediction needs huggingface token: [MODEL][hf_token]"
+            )
+        self.pipeline = Pipeline.from_pretrained(
+            "pyannote/speaker-diarization-3.1",
+            use_auth_token=hf_token,
+        )
+        device = self.util.config_val("MODEL", "device", "cpu")
+        if device == "cpu":
+            self.util.warn(
+                "running pyannote on CPU can be really slow, consider using a GPU"
+            )
+        self.pipeline.to(torch.device(device))
+
+    def get_segmentation_simple(self, file):
+
+        annotation = self.pipeline(file[0])
+
+        speakers, starts, ends, files = [], [], [], []
+        # print the result
+        for turn, _, speaker in annotation.itertracks(yield_label=True):
+            start = turn.start
+            end = turn.end
+            speakers.append(speaker)
+            starts.append(start)
+            files.append(file[0])
+            ends.append(end)
+        seg_index = segmented_index(files, starts, ends)
+        return seg_index, speakers
+
+    def get_segmentation(self, file, min_length, max_length):
+        annotation = self.pipeline(file)
+        files, starts, ends, speakers = [], [], [], []
+        for turn, _, speaker in annotation.itertracks(yield_label=True):
+            start = turn.start
+            end = turn.end
+            new_end = end
+            handled = False
+            while end - start > max_length:
+                new_end = start + max_length
+                if end - new_end < min_length:
+                    new_end = end
+                files.append(file[0])
+                starts.append(start)
+                ends.append(new_end)
+                speakers.append(speaker)
+                start += max_length
+                handled = True
+            if not handled and end - start > min_length:
+                files.append(file[0])
+                starts.append(start)
+                ends.append(end)
+                speakers.append(speaker)
+        seg_index = segmented_index(files, starts, ends)
+        return seg_index, speakers
+
+    def segment_dataframe(self, df):
+        dfs = []
+        max_length = eval(self.util.config_val("SEGMENT", "max_length", "False"))
+        if max_length:
+            if self.no_testing:
+                min_length = float(self.util.config_val("SEGMENT", "min_length", 2))
+            else:
+                min_length = 2
+            self.util.debug(f"segmenting with max length: {max_length+min_length}")
+        for file, values in tqdm(df.iterrows()):
+            if max_length:
+                index, speakers = self.get_segmentation(file, min_length, max_length)
+            else:
+                index, speakers = self.get_segmentation_simple(file)
+            df = pd.DataFrame(
+                values.to_dict(),
+                index,
+            )
+            df["speaker"] = speakers
+            dfs.append(df)
+        return audformat.utils.concat(dfs)
+
+
+def main():
+    files = pd.Series(["test_wavs/very_long.wav"])
+    df_sample = pd.DataFrame(index=files)
+    df_sample["target"] = "anger"
+    df_sample.index = audformat.utils.to_segmented_index(
+        df_sample.index, allow_nat=False
+    )
+    segmenter = Pyannote_segmenter(not_testing=False)
+    df_seg = segmenter.segment_dataframe(df_sample)
+
+    def calc_dur(x):
+        starts = x[1]
+        ends = x[2]
+        return (ends - starts).total_seconds()
+
+    df_seg["duration"] = df_seg.index.to_series().map(lambda x: calc_dur(x))
+    print(df_seg.head(100))
+
+
+if __name__ == "__main__":
+    main()

{nkululeko-0.91.2 → nkululeko-0.92.0/nkululeko.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.91.2
+Version: 0.92.0
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -355,6 +355,14 @@ F. Burkhardt, Johannes Wagner, Hagen Wierstorf, Florian Eyben and Björn Schulle
 Changelog
 =========
 
+Version 0.92.0
+--------------
+* added first version of automatic speaker prediction/segmentation
+
+Version 0.91.3
+--------------
+* some additions for robustness
+
 Version 0.91.2
 --------------
 * making lint work by excluding constants from check

{nkululeko-0.91.2 → nkululeko-0.92.0}/nkululeko.egg-info/SOURCES.txt

@@ -100,6 +100,7 @@ nkululeko/autopredict/ap_gender.py
 nkululeko/autopredict/ap_mos.py
 nkululeko/autopredict/ap_pesq.py
 nkululeko/autopredict/ap_sdr.py
+nkululeko/autopredict/ap_sid.py
 nkululeko/autopredict/ap_snr.py
 nkululeko/autopredict/ap_stoi.py
 nkululeko/autopredict/ap_valence.py
@@ -162,6 +163,7 @@ nkululeko/reporting/reporter.py
 nkululeko/reporting/result.py
 nkululeko/segmenting/__init__.py
 nkululeko/segmenting/seg_inaspeechsegmenter.py
+nkululeko/segmenting/seg_pyannote.py
 nkululeko/segmenting/seg_silero.py
 nkululeko/utils/__init__.py
 nkululeko/utils/files.py