nkululeko 0.65.2__tar.gz → 0.65.5__tar.gz

{nkululeko-0.65.2 → nkululeko-0.65.5}/CHANGELOG.md

@@ -1,6 +1,19 @@
 Changelog
 =========
 
+Version 0.65.5
+--------------
+* added fill_na in plot effect size
+
+Version 0.65.4
+--------------
+* added datasets to distribution
+* changes in wav2vec2
+
+Version 0.65.3
+--------------
+* various bugfixes
+
 Version 0.65.2
 --------------
 * fixed bug in dataset.csv that prevented correct paths for relative files

{nkululeko-0.65.2/nkululeko.egg-info → nkululeko-0.65.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.65.2
+Version: 0.65.5
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -255,6 +255,19 @@ Nkululeko can be used under the [MIT license](https://choosealicense.com/license
 Changelog
 =========
 
+Version 0.65.5
+--------------
+* added fill_na in plot effect size
+
+Version 0.65.4
+--------------
+* added datasets to distribution
+* changes in wav2vec2
+
+Version 0.65.3
+--------------
+* various bugfixes
+
 Version 0.65.2
 --------------
 * fixed bug in dataset.csv that prevented correct paths for relative files

nkululeko-0.65.5/nkululeko/constants.py

@@ -0,0 +1,2 @@
+VERSION="0.65.5"
+SAMPLING_RATE=16000

{nkululeko-0.65.2 → nkululeko-0.65.5}/nkululeko/data/dataset.py

@@ -159,7 +159,7 @@ class Dataset:
             self.df["duration"] = (end - start).total_seconds()
 
         # Perform some filtering if desired
-        required = eval(self.util.config_val_data(self.name, "required", "False"))
+        required = self.util.config_val_data(self.name, "required", False)
         if required:
             pre = self.df.shape[0]
             self.df = self.df[self.df[required].notna()]
@@ -204,7 +204,10 @@ class Dataset:
                 pass
             try:
                 # try to get the gender values
-                df_local["gender"] = source_df["gender"]
+                if "gender" in source_df:
+                    df_local["gender"] = source_df["gender"]
+                else:
+                    df_local["gender"] = source_df["sex"]
                 got_gender = True
             except (KeyError, ValueError, audformat.errors.BadKeyError) as e:
                 pass

{nkululeko-0.65.2 → nkululeko-0.65.5}/nkululeko/data/dataset_csv.py

@@ -48,21 +48,23 @@ class Dataset_CSV(Dataset):
         self.start_fresh = eval(self.util.config_val("DATA", "no_reuse", "False"))
         if self.is_labeled and not "class_label" in self.df.columns:
             self.df["class_label"] = self.df[self.target]
-        if "gender" in df.columns:
+        if "gender" in self.df.columns:
             self.got_gender = True
-        if "age" in df.columns:
+        elif "sex" in self.df.columns:
+            self.df = self.df.rename(columns={'sex':'gender'})            
+            self.got_gender = True
+        if "age" in self.df.columns:
             self.got_age = True
-        if "speaker" in df.columns:
+        if "speaker" in self.df.columns:
             self.got_speaker = True
-            ns = df["speaker"].nunique()
-            self.util.debug(f"num of speakers: {ns}")
         speaker_num = 0
         if self.got_speaker:
             speaker_num = self.df.speaker.nunique()
         self.util.debug(
             f"Loaded database {self.name} with {df.shape[0]} "
             f"samples: got targets: {self.got_target}, got speakers: {self.got_speaker} ({speaker_num}), "
-            f"got sexes: {self.got_gender}"
+            f"got sexes: {self.got_gender}, "
+            f"got age: {self.got_age}"
         )
 
     def prepare(self):

{nkululeko-0.65.2 → nkululeko-0.65.5}/nkululeko/export.py

@@ -10,7 +10,7 @@ import audiofile
 from nkululeko.experiment import Experiment
 from nkululeko.util import Util
 from nkululeko.constants import VERSION
-
+import shutil
 
 def main(src_dir):
     parser = argparse.ArgumentParser(description="Call the nkululeko framework.")
@@ -54,6 +54,7 @@ def main(src_dir):
     target_root = util.config_val("EXPORT", "root", "./exported_data/")
     orig_root = util.config_val("EXPORT", "orig_root", None)
     data_name = util.config_val("EXPORT", "data_name", "export")
+    segments_as_files = eval(util.config_val("EXPORT", "segments_as_files", "False"))
     audeer.mkdir(target_root)
     splits = {"train": df_train, "test": df_test}
     df_all = pd.DataFrame()
@@ -61,25 +62,40 @@ def main(src_dir):
         files = []
         df = splits[split]
         for idx, (file, start, end) in enumerate(df.index.to_list()):
-            signal, sampling_rate = audiofile.read(
-                file,
-                offset=start.total_seconds(),
-                duration=(end - start).total_seconds(),
-                always_2d=True,
-            )
             file_dir = os.path.dirname(file)
-            file_name = os.path.basename(file)
-            wav_folder = (
-                f"{target_root}/{os.path.basename(os.path.normpath(orig_root))}"
-            )
-            audeer.mkdir(wav_folder)
-            new_rel_path = file_dir[file_dir.index(orig_root)+1+len(orig_root):]
-            new_file_path = f"{wav_folder}/{new_rel_path}"
-            audeer.mkdir(new_file_path)
-            new_file_name = f"{new_file_path}/{file_name}"
-            audiofile.write(new_file_name, signal, sampling_rate)
-            new_file_name = os.path.relpath(new_file_name, target_root)
-            files.append(new_file_name)
+            if segments_as_files:
+                signal, sampling_rate = audiofile.read(
+                    file,
+                    offset=start.total_seconds(),
+                    duration=(end - start).total_seconds(),
+                    always_2d=True,
+                )
+                file_name = os.path.basename(file)[:-3]+'_'+start.total_seconds()+'.wav'
+                wav_folder = (
+                    f"{target_root}/{os.path.basename(os.path.normpath(orig_root))}"
+                )
+                audeer.mkdir(wav_folder)
+                new_rel_path = file_dir[file_dir.index(orig_root)+1+len(orig_root):]
+                new_file_path = f"{wav_folder}/{new_rel_path}"
+                audeer.mkdir(new_file_path)
+                new_file_name = f"{new_file_path}/{file_name}"
+                audiofile.write(new_file_name, signal, sampling_rate)
+                new_file_name = os.path.relpath(new_file_name, target_root)
+                files.append(new_file_name)
+            else:
+                file_name = os.path.basename(file)[:-4]+'_'+str(start.total_seconds())+'.wav'
+                wav_folder = (
+                    f"{target_root}/{os.path.basename(os.path.normpath(orig_root))}"
+                )
+                audeer.mkdir(wav_folder)
+                new_rel_path = file_dir[file_dir.index(orig_root)+1+len(orig_root):]
+                new_file_path = f"{wav_folder}/{new_rel_path}"
+                audeer.mkdir(new_file_path)
+                new_file_name = f"{new_file_path}/{file_name}"
+                if not os.path.exists(new_file_name):
+                    shutil.copyfile(file, new_file_name)
+                new_file_name = os.path.relpath(new_file_name, target_root)
+                files.append(new_file_name)
         df = df.set_index(df.index.set_levels(files, level="file"))
         df["split"] = split
         df_all = pd.concat([df_all, df])

{nkululeko-0.65.2 → nkululeko-0.65.5}/nkululeko/feat_extract/feats_wav2vec2.py

@@ -54,7 +54,7 @@ class Wav2vec2(Featureset):
                 signal, sampling_rate = torchaudio.load(file,
                     frame_offset=int(start.total_seconds()*16000),
                     num_frames=int((end - start).total_seconds()*16000))
-                assert sampling_rate == 16000
+                assert sampling_rate == 16000, f"got {sampling_rate} instead of 16000"
                 emb = self.get_embeddings(signal, sampling_rate, file)
                 emb_series[idx] = emb
                 if idx % 10 == 0:

{nkululeko-0.65.2 → nkululeko-0.65.5}/nkululeko/plots.py

@@ -57,6 +57,8 @@ class Plots():
         bin_reals = eval(self.util.config_val('EXPL', 'bin_reals', 'True'))
         for att in attributes:
             if len(att) == 1:
+                if att[0] not in df:
+                    self.util.error(f'unknown feature: {att[0]}')
                 self.util.debug(f'plotting {att[0]}')
                 filename = f'{self.target}-{att[0]}'
                 if self.util.is_categorical(df[att[0]]):
@@ -90,6 +92,10 @@ class Plots():
                 plt.close(fig)
                 # fig.clear()           # avoid error
             elif len(att) == 2:
+                if att[0] not in df:
+                    self.util.error(f'unknown feature: {att[0]}')
+                if att[1] not in df:
+                    self.util.error(f'unknown feature: {att[1]}')
                 self.util.debug(f'plotting {att}')
                 att1 = att[0]
                 att2 = att[1]

{nkululeko-0.65.2 → nkululeko-0.65.5}/nkululeko/utils/stats.py

@@ -3,6 +3,14 @@ import math
 import numpy as np
 import pandas as pd
 
+def check_na(a):
+    if np.isnan(a).any():
+        count = np.count_nonzero(np.isnan(a))
+        print(f'WARNING: got {count} Nans (of {len(a)}), setting to 0')
+        a[np.isnan(a)] = 0
+        return a 
+    else:
+        return a
 
 def cohen_d(d1, d2):
     """
@@ -13,6 +21,9 @@ def cohen_d(d1, d2):
     Returns:
         Cohen's d with precision 3
     """
+    # Checks:
+    d1 = check_na(d1)
+    d2 = check_na(d2)
     # calculate the size of samples
     n1, n2 = len(d1), len(d2)
     # calculate the variance of the samples
@@ -22,8 +33,8 @@ def cohen_d(d1, d2):
     # calculate the means of the samples
     u1, u2 = np.mean(d1), np.mean(d2)
     # calculate the effect size
-    if math.isnan(s):
-        return 0
+    if math.isnan(s) or s == 0:
+        return -1
     return (int(1000 * np.abs((u1 - u2)) / s)) / 1000
 
 

{nkululeko-0.65.2 → nkululeko-0.65.5/nkululeko.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.65.2
+Version: 0.65.5
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -255,6 +255,19 @@ Nkululeko can be used under the [MIT license](https://choosealicense.com/license
 Changelog
 =========
 
+Version 0.65.5
+--------------
+* added fill_na in plot effect size
+
+Version 0.65.4
+--------------
+* added datasets to distribution
+* changes in wav2vec2
+
+Version 0.65.3
+--------------
+* various bugfixes
+
 Version 0.65.2
 --------------
 * fixed bug in dataset.csv that prevented correct paths for relative files

nkululeko-0.65.2/nkululeko/constants.py

@@ -1,2 +0,0 @@
-VERSION="0.65.2"
-SAMPLING_RATE=16000