nkululeko 0.91.3__py3-none-any.whl → 0.92.1__py3-none-any.whl

nkululeko/autopredict/ap_sid.py

@@ -2,13 +2,12 @@
 A predictor for sid - Speaker ID.
 """
 
-from pyannote.audio import Pipeline
-
-
 import numpy as np
+from pyannote.audio import Pipeline
+import torch
 
-import nkululeko.glob_conf as glob_conf
 from nkululeko.feature_extractor import FeatureExtractor
+import nkululeko.glob_conf as glob_conf
 from nkululeko.utils.util import Util
 
 
@@ -21,23 +20,29 @@ class SIDPredictor:
     def __init__(self, df):
         self.df = df
         self.util = Util("sidPredictor")
+        hf_token = self.util.config_val("Model", "hf_token", None)
+        if hf_token is None:
+            self.util.error(
+                "speaker id prediction needs huggingface token: [MODEL][hf_token]"
+            )
         self.pipeline = Pipeline.from_pretrained(
             "pyannote/speaker-diarization-3.1",
-            use_auth_token="HUGGINGFACE_ACCESS_TOKEN_GOES_HERE",
+            use_auth_token=hf_token,
         )
+        device = self.util.config_val("Model", "device", "cpu")
+        self.pipeline.to(torch.device(device))
 
     def predict(self, split_selection):
-        self.util.debug(f"estimating PESQ for {split_selection} samples")
+        self.util.debug(f"estimating speaker id for {split_selection} samples")
         return_df = self.df.copy()
-        feats_name = "_".join(ast.literal_eval(glob_conf.config["DATA"]["databases"]))
-        self.feature_extractor = FeatureExtractor(
-            self.df, ["squim"], feats_name, split_selection
-        )
-        result_df = self.feature_extractor.extract()
-        # replace missing values by 0
-        result_df = result_df.fillna(0)
-        result_df = result_df.replace(np.nan, 0)
-        result_df.replace([np.inf, -np.inf], 0, inplace=True)
-        pred_vals = result_df.pesq * 100
-        return_df["pesq_pred"] = pred_vals.astype("int") / 100
+        # @todo
+        # 1) concat all audio files
+        # 2) get segmentations with pyannote
+        # 3) map pyannote segments with orginal ones and assign speaker id
+
         return return_df
+
+    def concat_files(self, df):
+        pass
+        # todo
+        # please use https://audeering.github.io/audiofile/usage.html#read-a-file

nkululeko/constants.py

@@ -1,2 +1,2 @@
-VERSION="0.91.3"
+VERSION="0.92.1"
 SAMPLING_RATE = 16000

nkululeko/experiment.py

@@ -439,7 +439,12 @@ class Experiment:
             )
         targets = self.util.config_val_list("PREDICT", "targets", ["gender"])
         for target in targets:
-            if target == "gender":
+            if target == "speaker":
+                from nkululeko.autopredict.ap_sid import SIDPredictor
+
+                predictor = SIDPredictor(df)
+                df = predictor.predict(sample_selection)
+            elif target == "gender":
                 from nkululeko.autopredict.ap_gender import GenderPredictor
 
                 predictor = GenderPredictor(df)

nkululeko/plots.py

@@ -4,14 +4,14 @@ import ast
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
-import seaborn as sns
 from scipy import stats
+import seaborn as sns
 from sklearn.manifold import TSNE
 
 import nkululeko.glob_conf as glob_conf
-import nkululeko.utils.stats as su
 from nkululeko.reporting.defines import Header
 from nkululeko.reporting.report_item import ReportItem
+import nkululeko.utils.stats as su
 from nkululeko.utils.util import Util
 
 
@@ -30,8 +30,6 @@ class Plots:
             df_speaker["samplenum"] = df_speaker.shape[0]
             df_speakers = pd.concat([df_speakers, df_speaker.head(1)])
         # plot the distribution of samples per speaker
-        # one up because of the runs
-        fig_dir = self.util.get_path("fig_dir") + "../"
         self.util.debug("plotting samples per speaker")
         if "gender" in df_speakers:
             filename = "samples_value_counts"
@@ -319,6 +317,31 @@ class Plots:
         img_path = f"{fig_dir}{filename}_{sample_selection}.{self.format}"
         plt.savefig(img_path)
         plt.close(fig)
+        self.util.debug(f"plotted durations to {img_path}")
+        glob_conf.report.add_item(
+            ReportItem(
+                Header.HEADER_EXPLORE,
+                caption,
+                title,
+                img_path,
+            )
+        )
+
+    def plot_speakers(self, df, sample_selection):
+        filename = "speakers"
+        caption = "speakers"
+        # one up because of the runs
+        fig_dir = self.util.get_path("fig_dir") + "../"
+        sns.set_style("whitegrid")  # Set style for chart
+        ax = df["speaker"].value_counts().plot(kind="pie")
+        title = f"Speaker distr. for {sample_selection} {df.shape[0]}."
+        ax.set_title(title)
+        fig = ax.figure
+        # plt.tight_layout()
+        img_path = f"{fig_dir}{filename}_{sample_selection}.{self.format}"
+        plt.savefig(img_path)
+        plt.close(fig)
+        self.util.debug(f"plotted speakers to {img_path}")
         glob_conf.report.add_item(
             ReportItem(
                 Header.HEADER_EXPLORE,

nkululeko/predict.py

@@ -2,7 +2,7 @@
 # use some model and add automatically predicted labels to train and test splits
 # then save as a new dataset
 
-"""This script is used to call the nkululeko PREDICT framework. 
+r"""This script is used to call the nkululeko PREDICT framework.
 
 It loads a configuration file, creates a new experiment,
 and performs automatic prediction on the train and test datasets. The predicted labels are added to the datasets and

nkululeko/segment.py

@@ -1,5 +1,4 @@
-"""
-Segments the samples in the dataset into chunks based on voice activity detection using SILERO VAD [1].
+"""Segments the samples in the dataset into chunks based on voice activity detection using SILERO VAD [1].
 
 The segmentation results are saved to a file, and the distributions of the original and
 segmented durations are plotted.
@@ -15,7 +14,7 @@ Example:
 
 References:
     [1] https://github.com/snakers4/silero-vad
-    
+    [2] https://github.com/pyannote/pyannote-audio
 """
 
 import argparse
@@ -64,7 +63,7 @@ def main():
     # segment
     segmented_file = util.config_val("SEGMENT", "result", "segmented.csv")
 
-    segmenter = util.config_val("SEGMENT", "method", "silero")
+    method = util.config_val("SEGMENT", "method", "silero")
     sample_selection = util.config_val("SEGMENT", "sample_selection", "all")
     if sample_selection == "all":
         df = pd.concat([expr.df_train, expr.df_test])
@@ -77,18 +76,21 @@ def main():
             f"unknown segmentation selection specifier {sample_selection},"
             " should be [all | train | test]"
         )
-    util.debug(f"segmenting {sample_selection}: {df.shape[0]} samples with {segmenter}")
-    if segmenter == "silero":
+    util.debug(f"segmenting {sample_selection}: {df.shape[0]} samples with {method}")
+    if method == "silero":
         from nkululeko.segmenting.seg_silero import Silero_segmenter
 
         segmenter = Silero_segmenter()
         df_seg = segmenter.segment_dataframe(df)
+    elif method == "pyannote":
+        from nkululeko.segmenting.seg_pyannote import Pyannote_segmenter
 
+        segmenter = Pyannote_segmenter(config)
+        df_seg = segmenter.segment_dataframe(df)
     else:
-        util.error(f"unkown segmenter: {segmenter}")
+        util.error(f"unknown segmenter: {method}")
 
     def calc_dur(x):
-
         starts = x[1]
         ends = x[2]
         return (ends - starts).total_seconds()
@@ -108,6 +110,9 @@ def main():
     plots.plot_durations(
         df_seg, "segmented_durations", sample_selection, caption="Segmented durations"
     )
+    if method == "pyannote":
+        plots.plot_speakers(df_seg, sample_selection)
+
     print("")
     # remove encoded labels
     target = util.config_val("DATA", "target", None)
@@ -115,8 +120,6 @@ def main():
         df_seg = df_seg.drop(columns=[target])
         df_seg = df_seg.rename(columns={"class_label": target})
     # save file
-    # dataname = "_".join(expr.datasets.keys())
-    # name = f"{dataname}{segment_target}"
     df_seg.to_csv(f"{expr.data_dir}/{segmented_file}")
     util.debug(
         f"saved {segmented_file} to {expr.data_dir}, {num_after} samples (was"

nkululeko/segmenting/seg_pyannote.py

@@ -0,0 +1,129 @@
+"""seg_pyannote.py.
+
+Segment a dataset with the Pyannote segmenter.
+Also adds speaker ids to the segments.
+
+"""
+
+import pandas as pd
+from pyannote.audio import Pipeline
+import torch
+from tqdm import tqdm
+
+import audformat
+from audformat import segmented_index
+
+from nkululeko.utils.util import Util
+
+
+SAMPLING_RATE = 16000
+
+
+class Pyannote_segmenter:
+    def __init__(self, not_testing=True):
+        # initialize the VAD model
+        torch.set_num_threads(1)
+        self.no_testing = not_testing
+        self.util = Util("pyannote_segmenter")
+        hf_token = self.util.config_val("MODEL", "hf_token", None)
+        if hf_token is None:
+            self.util.error(
+                "speaker id prediction needs huggingface token: [MODEL][hf_token]"
+            )
+        self.pipeline = Pipeline.from_pretrained(
+            "pyannote/speaker-diarization-3.1",
+            use_auth_token=hf_token,
+        )
+        device = self.util.config_val("MODEL", "device", "cpu")
+        if device == "cpu":
+            self.util.warn(
+                "running pyannote on CPU can be really slow, consider using a GPU"
+            )
+        self.pipeline.to(torch.device(device))
+
+    def get_segmentation_simple(self, file):
+
+        annotation = self.pipeline(file[0])
+
+        speakers, starts, ends, files = [], [], [], []
+        # print the result
+        for turn, _, speaker in annotation.itertracks(yield_label=True):
+            start = turn.start
+            end = turn.end
+            speakers.append(speaker)
+            starts.append(start)
+            files.append(file[0])
+            ends.append(end)
+        seg_index = segmented_index(files, starts, ends)
+        return seg_index, speakers
+
+    def get_segmentation(self, file, min_length, max_length):
+        annotation = self.pipeline(file)
+        files, starts, ends, speakers = [], [], [], []
+        for turn, _, speaker in annotation.itertracks(yield_label=True):
+            start = turn.start
+            end = turn.end
+            new_end = end
+            handled = False
+            while end - start > max_length:
+                new_end = start + max_length
+                if end - new_end < min_length:
+                    new_end = end
+                files.append(file[0])
+                starts.append(start)
+                ends.append(new_end)
+                speakers.append(speaker)
+                start += max_length
+                handled = True
+            if not handled and end - start > min_length:
+                files.append(file[0])
+                starts.append(start)
+                ends.append(end)
+                speakers.append(speaker)
+        seg_index = segmented_index(files, starts, ends)
+        return seg_index, speakers
+
+    def segment_dataframe(self, df):
+        dfs = []
+        max_length = eval(self.util.config_val("SEGMENT", "max_length", "False"))
+        if max_length:
+            if self.no_testing:
+                min_length = float(self.util.config_val("SEGMENT", "min_length", 2))
+            else:
+                min_length = 2
+            self.util.debug(f"segmenting with max length: {max_length+min_length}")
+        for file, values in tqdm(df.iterrows()):
+            if max_length:
+                index, speakers = self.get_segmentation(file, min_length, max_length)
+            else:
+                index, speakers = self.get_segmentation_simple(file)
+            df = pd.DataFrame(
+                values.to_dict(),
+                index,
+            )
+            df["speaker"] = speakers
+            dfs.append(df)
+        return audformat.utils.concat(dfs)
+
+
+def main():
+    files = pd.Series(["test_wavs/very_long.wav"])
+    df_sample = pd.DataFrame(index=files)
+    df_sample["target"] = "anger"
+    df_sample.index = audformat.utils.to_segmented_index(
+        df_sample.index, allow_nat=False
+    )
+    segmenter = Pyannote_segmenter(not_testing=False)
+    df_seg = segmenter.segment_dataframe(df_sample)
+
+    def calc_dur(x):
+        starts = x[1]
+        ends = x[2]
+        return (ends - starts).total_seconds()
+
+    df_seg["duration"] = df_seg.index.to_series().map(lambda x: calc_dur(x))
+    print(df_seg.head(100))
+
+
+if __name__ == "__main__":
+    main()

{nkululeko-0.91.3.dist-info → nkululeko-0.92.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.91.3
+Version: 0.92.1
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -355,6 +355,14 @@ F. Burkhardt, Johannes Wagner, Hagen Wierstorf, Florian Eyben and Björn Schulle
 Changelog
 =========
 
+Version 0.92.1
+--------------
+* added a speaker plot to pyannote results 
+
+Version 0.92.0
+--------------
+* added first version of automatic speaker prediction/segmentation
+
 Version 0.91.3
 --------------
 * some additions for robustness

{nkululeko-0.91.3.dist-info → nkululeko-0.92.1.dist-info}/RECORD

@@ -2,13 +2,13 @@ nkululeko/__init__.py,sha256=62f8HiEzJ8rG2QlTFJXUCMpvuH3fKI33DoJSj33mscc,63
 nkululeko/aug_train.py,sha256=FoMbBrfyOZd4QAw7oIHl3X6-UpsqAKWVDIolCA7qOWs,3196
 nkululeko/augment.py,sha256=3RzaxB3gRxovgJVjHXi0glprW01J7RaHhUkqotW2T3U,2955
 nkululeko/cacheddataset.py,sha256=XFpWZmbJRg0pvhnIgYf0TkclxllD-Fctu-Ol0PF_00c,969
-nkululeko/constants.py,sha256=tT5Su7q7ufkiVtsEMOV5FgJVkE2U8idmrVxrNi4gFmc,39
+nkululeko/constants.py,sha256=XCM_WklnS2apgRW6vmIdjyrrkWT4PRKQCPhILJsDKKM,39
 nkululeko/demo-ft.py,sha256=iD9Pzp9QjyAv31q1cDZ75vPez7Ve8A4Cfukv5yfZdrQ,770
 nkululeko/demo.py,sha256=4Yzhg6pCPBYPGJrP7JX2TysVosl_R1llpVDKc2P_gUA,4955
 nkululeko/demo_feats.py,sha256=BvZjeNFTlERIRlq34OHM4Z96jdDQAhB01BGQAUcX9dM,2026
 nkululeko/demo_predictor.py,sha256=lDF-xOxRdEAclOmbepAYg-BQXQdGkHfq2n74PTIoop8,4872
 nkululeko/ensemble.py,sha256=71V-rre61H3J4sh7lu-OTo4I2_g7mm_rQxwW1ARDHgY,12782
-nkululeko/experiment.py,sha256=kRz3diGyupXneXFBLmmUm6BV-sGSqQJv44Zojn0Bhcs,31649
+nkululeko/experiment.py,sha256=h3DS-k6vk5juXa3HJXI7Z4vvnNspO4qj5SJ1o1Z3PIk,31860
 nkululeko/explore.py,sha256=Y5lPPychnI-7fyP8zvwVb9P09fvprbUPOofOppuABYQ,3658
 nkululeko/export.py,sha256=U-V4acxtuL6qKt6oAsVcM5TTeWogYUJ3GU-lA6rq6d4,4336
 nkululeko/feature_extractor.py,sha256=UnspIWz3XrNhKnBBhWZkH2bHvD-sROtrQVqB1JvkUyw,4088
@@ -20,12 +20,12 @@ nkululeko/modelrunner.py,sha256=lJy-xM4QfDDWeL0dLTE_VIb4sYrnd_Z_yJRK3wwohQA,1119
 nkululeko/multidb.py,sha256=sO6OwJn8sn1-C-ig3thsIL8QMWHdV9SnJhDodKjeKrI,6876
 nkululeko/nkuluflag.py,sha256=PGWSmZz-PiiHLgcZJAoGOI_Y-sZDVI1ksB8p5r7riWM,3725
 nkululeko/nkululeko.py,sha256=M7baIq2nAoi6dEoBL4ATEuqAs5U1fvl_hyqAl5DybAQ,2040
-nkululeko/plots.py,sha256=sR061gOsyvuh8UBYS52FINSal4CYNQgvq3B4WOSimDw,23092
-nkululeko/predict.py,sha256=b35YOqovGb5PLDz0nDuhJGykEAPq2Y45R9lzxJZMuMU,2083
+nkululeko/plots.py,sha256=UvAsppf9cFVifx2u0V4dgbejsp7UDeqPGKTHwa02dis,23916
+nkululeko/predict.py,sha256=MLnHEyFmSiHLLs-HDczag8Vu3zKF5T1rXLKdZZJ6py8,2083
 nkululeko/resample.py,sha256=akSAjJ3qn-O5NAyLJHVHdsK7MUZPGaZUvM2TwMSmj2M,5194
 nkululeko/runmanager.py,sha256=AswmORVUkCIH0gTx6zEyufvFATQBS8C5TXo2erSNdVg,7611
 nkululeko/scaler.py,sha256=7VOZ4sREMoQtahfETt9RyuR29Fb7PCwxlYVjBbdCVFc,4101
-nkululeko/segment.py,sha256=PAc5sVLoqKzOVENd9A5ybaKrJOvD9WEPwDdJGTv6OIM,4319
+nkululeko/segment.py,sha256=oAyXVnFt_fRc6fbrIQE1fOo2tiMF6GAY3oB_admKtT8,4545
 nkululeko/syllable_nuclei.py,sha256=5w_naKxNxz66a_qLkraemi2fggM-gWesiiBPS47iFcE,9931
 nkululeko/test.py,sha256=1w624vo5KTzmFC8BUStGlLDmIEAFuJUz7J0W-gp7AxI,1677
 nkululeko/test_predictor.py,sha256=DEHE_D3A6m6KJTrpDKceA1n655t_UZV3WQd57K4a3Ho,2863
@@ -43,7 +43,7 @@ nkululeko/autopredict/ap_gender.py,sha256=b6oTqHKVwOnYh4YlKbuMflssS4HJqs_c1ayusa
 nkululeko/autopredict/ap_mos.py,sha256=e4hmgb0Yf1_AbC5P0CqXJIvufjhbTrqmI5goARxrY0Y,1107
 nkululeko/autopredict/ap_pesq.py,sha256=mRt3Loucaoy4vJxwfuxUt0fP88bMGvkmrLCEpKEXWp0,1140
 nkululeko/autopredict/ap_sdr.py,sha256=VQ2UkxOO3ipqYNNjFwKgEaGCk8IzLI5lX_2tZFLIvTY,1188
-nkululeko/autopredict/ap_sid.py,sha256=1TYJg-Bg_LJGPzIWF-oYtXmD5Otvi0fW_f8uzEVZY5g,1309
+nkululeko/autopredict/ap_sid.py,sha256=87LXMHzJ8jt2q9dUtPJd_nJi_XOcFoqpbva-BT4UJN0,1393
 nkululeko/autopredict/ap_snr.py,sha256=AiTU8-7CMEeowmYkMO19lw1HCb1yTXC6KeulNf8gOqw,1110
 nkululeko/autopredict/ap_stoi.py,sha256=UEQg1ZV0meAsxgdWB8ieRs9GPXHqArmsaOyCGRwpcnA,1187
 nkululeko/autopredict/ap_valence.py,sha256=WrW4Ltqi_odW49_4QEVKkfnrcztLIVZ4cXIEHu4dBN8,1026
@@ -106,14 +106,15 @@ nkululeko/reporting/reporter.py,sha256=4OlYZAParkfJKO_aAyxqVpLc21zxZ-jDhtJKIMeUs
 nkululeko/reporting/result.py,sha256=G63a2tHCwHhM6NBJgYzsWKWJm4Yu3r4hsCHA2Km7eHU,1073
 nkululeko/segmenting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/segmenting/seg_inaspeechsegmenter.py,sha256=b3t0zdpJYofKWMyKRMtMMX91xeR-k8d5pbnNaQHcsOE,1902
+nkululeko/segmenting/seg_pyannote.py,sha256=6IPbgjnGOz9juzEKDTZN3PSipX4t6Mz-DILAx3rp5do,4216
 nkululeko/segmenting/seg_silero.py,sha256=ulodnvtRq5MLHDxy_RmAK4tJg6h1d-mPq-uCPFkGVKg,4258
 nkululeko/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/utils/files.py,sha256=UiGAtZRWYjHSvlmPaTMtzyNNGE6qaLaxQkybctS7iRM,4021
 nkululeko/utils/stats.py,sha256=vCRzhCR0Gx5SiJyAGbj1TIto8ocGz58CM5Pr3LltagA,2948
 nkululeko/utils/util.py,sha256=XFZdhCc_LM4EmoZ5tKKaBCQLXclcNmvHwhfT_CXB98c,16723
-nkululeko-0.91.3.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
-nkululeko-0.91.3.dist-info/METADATA,sha256=FI1hcOEFJbfAHNQi1SCFhPv7sC06vdWa75lIw2Ix4YA,41584
-nkululeko-0.91.3.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-nkululeko-0.91.3.dist-info/entry_points.txt,sha256=lNTkFEdh6Kjo5o95ZAWf_0Lq-4ztGoAoMVSDuPtuyS0,442
-nkululeko-0.91.3.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
-nkululeko-0.91.3.dist-info/RECORD,,
+nkululeko-0.92.1.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
+nkululeko-0.92.1.dist-info/METADATA,sha256=faUT2RL5nLgqu-1TRgmKLc3tEZKEO4hXG0NUWTxwiGo,41757
+nkululeko-0.92.1.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+nkululeko-0.92.1.dist-info/entry_points.txt,sha256=lNTkFEdh6Kjo5o95ZAWf_0Lq-4ztGoAoMVSDuPtuyS0,442
+nkululeko-0.92.1.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
+nkululeko-0.92.1.dist-info/RECORD,,