nkululeko 0.93.6__py3-none-any.whl → 0.93.8__py3-none-any.whl

nkululeko/constants.py

@@ -1,2 +1,2 @@
-VERSION="0.93.6"
+VERSION="0.93.8"
 SAMPLING_RATE = 16000

nkululeko/data/dataset.py

@@ -655,6 +655,8 @@ class Dataset:
         if mappings:
             mapping = ast.literal_eval(mappings)
             df[target] = df[target].map(mapping)
+            # remove any exiting nan values after mapping
+            df = df.dropna()
             self.util.debug(f"{self.name}: mapped {mapping}")
         # remove labels that are not in the labels list
         labels = self.util.config_val("DATA", "labels", False)

nkululeko/feat_extract/feats_import.py

@@ -18,6 +18,10 @@ class ImportSet(Featureset):
     def extract(self):
         """Import the features."""
         self.util.debug(f"importing features for {self.name}")
+        # import_files_append: set this to True if the multiple tables should be combined row-wise, else they are combined column-wise
+        import_files_append = eval(
+            self.util.config_val("FEATS", "import_files_append", "True")
+        )
         try:
             feat_import_files = self.util.config_val("FEATS", "import_file", False)
             feat_import_files = ast.literal_eval(feat_import_files)
@@ -38,7 +42,10 @@ class ImportSet(Featureset):
             df = audformat.utils.read_csv(feat_import_file)
             df = self.util.make_segmented_index(df)
             df = df[df.index.isin(self.data_df.index)]
-            feat_df = pd.concat([feat_df, df])
+            if import_files_append:
+                feat_df = pd.concat([feat_df, df], axis=0)
+            else:
+                feat_df = pd.concat([feat_df, df], axis=1)
         if feat_df.shape[0] == 0:
             self.util.error(f"Imported features for data set {self.name} not found!")
         # and assign to be the "official" feature set

nkululeko/feat_extract/feats_trill.py

@@ -3,15 +3,16 @@ import os
 
 import audiofile as af
 import pandas as pd
-import tensorflow as tf
-import tensorflow_hub as hub
+
+# import tensorflow as tf
+# import tensorflow_hub as hub
 from tqdm import tqdm
 
 import nkululeko.glob_conf as glob_conf
 from nkululeko.feat_extract.featureset import Featureset
 
 # Import TF 2.X and make sure we're running eager.
-assert tf.executing_eagerly()
+# assert tf.executing_eagerly()
 
 
 class TRILLset(Featureset):
@@ -39,7 +40,7 @@ class TRILLset(Featureset):
             "trill.model",
             "https://tfhub.dev/google/nonsemantic-speech-benchmark/trill/3",
         )
-        self.model = hub.load(model_path)
+        # self.model = hub.load(model_path)
         self.feats_type = feats_type
 
     def extract(self):

nkululeko/plots.py

@@ -242,84 +242,100 @@ class Plots:
 
     def _plot2cont_cat(self, df, cont1, cont2, cat, ylab):
         """Plot relation of two continuous distributions with one categorical."""
+        plot_df = df[[cont1, cont2, cat]].copy()
         if cont2 == "class_label":
-            df.rename(columns={cont2: self.target})
+            plot_df = plot_df.rename(columns={cont2: self.target})
             cont2 = self.target
         if cont1 == "class_label":
-            df.rename(columns={cont1: self.target})
+            plot_df = plot_df.rename(columns={cont1: self.target})
             cont1 = self.target
         if cat == "class_label":
-            df.rename(columns={cat: self.target})
+            plot_df = plot_df.rename(columns={cat: self.target})
             cat = self.target
-        pearson = stats.pearsonr(df[cont1], df[cont2])
+        pearson = stats.pearsonr(plot_df[cont1], plot_df[cont2])
         # trunc to three digits
         pearson = int(pearson[0] * 1000) / 1000
         pearson_string = f"PCC: {pearson}"
         ccc_string = ""
         if self.with_ccc:
-            ccc_val = ccc(df[cont1], df[cont2])
+            ccc_val = ccc(plot_df[cont1], plot_df[cont2])
             ccc_val = int(ccc_val * 1000) / 1000
             ccc_string = f"CCC: {ccc_val}"
-        ax = sns.lmplot(data=df, x=cont1, y=cont2, hue=cat)
-        caption = f"{ylab} {df.shape[0]}. {pearson_string} {ccc_string}"
+        ax = sns.lmplot(data=plot_df, x=cont1, y=cont2, hue=cat)
+        caption = f"{ylab} {plot_df.shape[0]}. {pearson_string} {ccc_string}"
         ax.figure.suptitle(caption)
         return ax, caption
 
     def _plot2cont(self, df, col1, col2, ylab):
         """Plot relation of two continuous distributions."""
+        plot_df = df[[col1, col2]].copy()
         # rename "class_label" to the original target
         if col2 == "class_label":
-            df.rename(columns={col2: self.target})
+            plot_df = plot_df.rename(columns={col2: self.target})
             col2 = self.target
         if col1 == "class_label":
-            df.rename(columns={col1: self.target})
+            plot_df = plot_df.rename(columns={col1: self.target})
             col1 = self.target
-        pearson = stats.pearsonr(df[col1], df[col2])
+        pearson = stats.pearsonr(plot_df[col1], plot_df[col2])
         # trunc to three digits
         pearson = int(pearson[0] * 1000) / 1000
         pearson_string = f"PCC: {pearson}"
         ccc_string = ""
         if self.with_ccc:
-            ccc_val = ccc(df[col1], df[col2])
+            ccc_val = ccc(plot_df[col1], plot_df[col2])
             ccc_val = int(ccc_val * 1000) / 1000
             ccc_string = f"CCC: {ccc_val}"
-        ax = sns.lmplot(data=df, x=col1, y=col2)
-        caption = f"{ylab} {df.shape[0]}. {pearson_string} {ccc_string}"
+        ax = sns.lmplot(data=plot_df, x=col1, y=col2)
+        caption = f"{ylab} {plot_df.shape[0]}. {pearson_string} {ccc_string}"
         ax.figure.suptitle(caption)
         return ax, caption
 
     def plotcatcont(self, df, cat_col, cont_col, xlab, ylab):
         """Plot relation of categorical distribution with continuous."""
         # rename "class_label" to the original target
+        plot_df = df[[cat_col, cont_col]].copy()
         if cat_col == "class_label":
-            df.rename(columns={cat_col: self.target})
+            plot_df = plot_df.rename(columns={cat_col: self.target})
             cat_col = self.target
         dist_type = self.util.config_val("EXPL", "dist_type", "kde")
-        cats, cat_str, es = su.get_effect_size(df, cat_col, cont_col)
+        cats, cat_str, es = su.get_effect_size(plot_df, cat_col, cont_col)
         model_type = self.util.get_model_type()
         if dist_type == "hist" and model_type != "tree":
-            ax = sns.histplot(df, x=cont_col, hue=cat_col, kde=True)
-            caption = f"{ylab} {df.shape[0]}. {cat_str} ({cats}):" f" {es}"
+            ax = sns.histplot(plot_df, x=cont_col, hue=cat_col, kde=True)
+            caption = f"{ylab} {plot_df.shape[0]}. {cat_str} ({cats}):" f" {es}"
             ax.set_title(caption)
             ax.set_xlabel(f"{cont_col}")
             ax.set_ylabel(f"number of {ylab}")
         else:
             ax = sns.displot(
-                df, x=cont_col, hue=cat_col, kind="kde", fill=True, warn_singular=False
+                plot_df,
+                x=cont_col,
+                hue=cat_col,
+                kind="kde",
+                fill=True,
+                warn_singular=False,
             )
             ax.set(xlabel=f"{cont_col}")
-            caption = f"{ylab} {df.shape[0]}. {cat_str} ({cats}):" f" {es}"
+            caption = f"{ylab} {plot_df.shape[0]}. {cat_str} ({cats}):" f" {es}"
             ax.figure.suptitle(caption)
         return ax, caption
 
     def _plot2cat(self, df, col1, col2, xlab, ylab):
         """Plot relation of 2 categorical distributions."""
-        crosstab = pd.crosstab(index=df[col1], columns=df[col2])
+        plot_df = df[[col1, col2]].copy()
+        # rename "class_label" to the original target
+        if col2 == "class_label":
+            plot_df = plot_df.rename(columns={col2: self.target})
+            col2 = self.target
+        if col1 == "class_label":
+            plot_df = plot_df.rename(columns={col1: self.target})
+            col1 = self.target
+        crosstab = pd.crosstab(index=plot_df[col1], columns=plot_df[col2])
         res_pval = stats.chi2_contingency(crosstab)
         res_pval = int(res_pval[1] * 1000) / 1000
-        caption = f"{ylab} {df.shape[0]}. P-val chi2: {res_pval}"
+        caption = f"{ylab} {plot_df.shape[0]}. P-val chi2: {res_pval}"
         ax = (
-            df.groupby(col1, observed=False)[col2]
+            plot_df.groupby(col1, observed=False)[col2]
             .value_counts()
             .unstack()
             .plot(kind="bar", stacked=True, title=caption, rot=0)

nkululeko/reporting/reporter.py

@@ -3,32 +3,28 @@ import glob
 import json
 import math
 
+# import os
+from confidence_intervals import evaluate_with_conf_int
 import matplotlib.pyplot as plt
 import numpy as np
+from scipy.special import softmax
+from scipy.stats import entropy
+from scipy.stats import pearsonr
+from sklearn.metrics import ConfusionMatrixDisplay
+from sklearn.metrics import RocCurveDisplay
+from sklearn.metrics import auc
+from sklearn.metrics import classification_report
+from sklearn.metrics import confusion_matrix
+from sklearn.metrics import r2_score
+from sklearn.metrics import roc_auc_score
+from sklearn.metrics import roc_curve
 
 # from torch import is_tensor
-from audmetric import (
-    accuracy,
-    concordance_cc,
-    mean_absolute_error,
-    mean_squared_error,
-    unweighted_average_recall,
-)
-
-# import os
-from confidence_intervals import evaluate_with_conf_int
-from scipy.special import softmax
-from scipy.stats import entropy, pearsonr
-from sklearn.metrics import (
-    ConfusionMatrixDisplay,
-    RocCurveDisplay,
-    auc,
-    classification_report,
-    confusion_matrix,
-    r2_score,
-    roc_auc_score,
-    roc_curve,
-)
+from audmetric import accuracy
+from audmetric import concordance_cc
+from audmetric import mean_absolute_error
+from audmetric import mean_squared_error
+from audmetric import unweighted_average_recall
 
 import nkululeko.glob_conf as glob_conf
 from nkululeko.plots import Plots

{nkululeko-0.93.6.dist-info → nkululeko-0.93.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.93.6
+Version: 0.93.8
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -32,8 +32,6 @@ Requires-Dist: scikit_learn
 Requires-Dist: scipy
 Requires-Dist: seaborn
 Requires-Dist: sounddevice
-Requires-Dist: tensorflow
-Requires-Dist: tensorflow_hub
 Requires-Dist: torch
 Requires-Dist: torchvision
 Requires-Dist: transformers
@@ -355,6 +353,15 @@ If you use it, please mention the Nkululeko paper:
 Changelog
 =========
 
+Version 0.93.8
+--------------
+* fixed bugs in plot
+* added import_files_append=False
+
+Version 0.93.7
+--------------
+* added a safety to remove nan values after mapping
+
 Version 0.93.6
 --------------
 * added error message and hint for data.target_tables_append

{nkululeko-0.93.6.dist-info → nkululeko-0.93.8.dist-info}/RECORD

@@ -2,7 +2,7 @@ nkululeko/__init__.py,sha256=62f8HiEzJ8rG2QlTFJXUCMpvuH3fKI33DoJSj33mscc,63
 nkululeko/aug_train.py,sha256=FoMbBrfyOZd4QAw7oIHl3X6-UpsqAKWVDIolCA7qOWs,3196
 nkululeko/augment.py,sha256=3RzaxB3gRxovgJVjHXi0glprW01J7RaHhUkqotW2T3U,2955
 nkululeko/cacheddataset.py,sha256=XFpWZmbJRg0pvhnIgYf0TkclxllD-Fctu-Ol0PF_00c,969
-nkululeko/constants.py,sha256=CBvqIM1vafeKeZJAFI4K96rOgsKVLyvRhUJoNEsmOIo,39
+nkululeko/constants.py,sha256=CK6zAyfTJ7tNTdh3Js3fEqv5jrLD4c_UfCI4ZOK7bcE,39
 nkululeko/demo-ft.py,sha256=iD9Pzp9QjyAv31q1cDZ75vPez7Ve8A4Cfukv5yfZdrQ,770
 nkululeko/demo.py,sha256=4Yzhg6pCPBYPGJrP7JX2TysVosl_R1llpVDKc2P_gUA,4955
 nkululeko/demo_feats.py,sha256=BvZjeNFTlERIRlq34OHM4Z96jdDQAhB01BGQAUcX9dM,2026
@@ -20,7 +20,7 @@ nkululeko/modelrunner.py,sha256=lJy-xM4QfDDWeL0dLTE_VIb4sYrnd_Z_yJRK3wwohQA,1119
 nkululeko/multidb.py,sha256=sO6OwJn8sn1-C-ig3thsIL8QMWHdV9SnJhDodKjeKrI,6876
 nkululeko/nkuluflag.py,sha256=PGWSmZz-PiiHLgcZJAoGOI_Y-sZDVI1ksB8p5r7riWM,3725
 nkululeko/nkululeko.py,sha256=M7baIq2nAoi6dEoBL4ATEuqAs5U1fvl_hyqAl5DybAQ,2040
-nkululeko/plots.py,sha256=emn2NpZyOGlC8pw0NdAGRbUkPzZzbOKFO50HZ7vJG40,25275
+nkululeko/plots.py,sha256=zHWZ8Ns_0SLOEdbDVulObpRPoXRw_qqPXJv2dM08EeE,26049
 nkululeko/predict.py,sha256=MLnHEyFmSiHLLs-HDczag8Vu3zKF5T1rXLKdZZJ6py8,2083
 nkululeko/resample.py,sha256=rn3-M1A-iwVGibfQNGyeYNa7briD24lIN9Szq_1uTJo,5194
 nkululeko/runmanager.py,sha256=AswmORVUkCIH0gTx6zEyufvFATQBS8C5TXo2erSNdVg,7611
@@ -49,7 +49,7 @@ nkululeko/autopredict/ap_stoi.py,sha256=UEQg1ZV0meAsxgdWB8ieRs9GPXHqArmsaOyCGRwp
 nkululeko/autopredict/ap_valence.py,sha256=WrW4Ltqi_odW49_4QEVKkfnrcztLIVZ4cXIEHu4dBN8,1026
 nkululeko/autopredict/estimate_snr.py,sha256=1k9-XadABudnsNOeFZD_Fg0E64-GUQVS7JEp82MLQS4,4995
 nkululeko/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nkululeko/data/dataset.py,sha256=Sqk5XVArXVjwN-6B9uT0T9OkUPyoQwklJPqgDPPQgNE,29954
+nkululeko/data/dataset.py,sha256=G4jzD2MvzB7d6Oja_pUIdShFr7Qsbs0ogGzuTcyQfLo,30041
 nkululeko/data/dataset_csv.py,sha256=p2b4eS5R2Q5zdOIc56NRRU2PTFXSRt0qrdHGafHkWKo,4830
 nkululeko/feat_extract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/feat_extract/feats_agender.py,sha256=onfAQ6-xx_mFMJXEF1IX8cHBmGtGeX6weJmxbkfh1_o,3184
@@ -60,7 +60,7 @@ nkululeko/feat_extract/feats_auddim.py,sha256=CGLp_aYhudfwoU5522vjrvjPxfZcyw593A
 nkululeko/feat_extract/feats_audmodel.py,sha256=OsZyB1rdcG0Fai2gAwBlbuubmWor1_-P4IDkZLqgPKE,3161
 nkululeko/feat_extract/feats_clap.py,sha256=1tttpfm2SJmQgYm2u8eUVpDiDOpWdKqFChpY3ZZokNs,3395
 nkululeko/feat_extract/feats_hubert.py,sha256=F3vrPCkx8EimJjFWYCZ7Yg9uo1G3NjYt4UKrGIUev8k,5172
-nkululeko/feat_extract/feats_import.py,sha256=skqXq5oLJLyQbY1jlsg3VDoUt93yI1OLwbtcc44AQo4,1627
+nkululeko/feat_extract/feats_import.py,sha256=u9pTRbxZF2AbQDo4bhyqW6tlH3boS3KHpqW5Gh-Uy1U,2006
 nkululeko/feat_extract/feats_mld.py,sha256=5aRoYiGDm5ApoFntxAMQYPjEelXHHRBHZcAJR9dxaeI,1945
 nkululeko/feat_extract/feats_mos.py,sha256=3UXCKe86F49yHpZMQnLfDWXx9XdmlXHOy8efoa3WaOk,4138
 nkululeko/feat_extract/feats_opensmile.py,sha256=BLj5sUaBPz7vLPfNlt9LdQurSypmViqgSpPK-6aXGhQ,4029
@@ -70,7 +70,7 @@ nkululeko/feat_extract/feats_snr.py,sha256=5uEm10d89TQPf0s-CuVpQ3ftc0bLEeuB8aGuu
 nkululeko/feat_extract/feats_spectra.py,sha256=6WhFUpB0WTutg7OFMlAw9lSwVU5OBYCDcPRxaiH-Qn8,3621
 nkululeko/feat_extract/feats_spkrec.py,sha256=o_6bdU4lIkj64S5Kdjf1iyuo1VASeYxE4XdxV94a8gE,4732
 nkululeko/feat_extract/feats_squim.py,sha256=yJifsp9kj9iJjW_UAKr3LlvVhX5rv7el4bepn0wN2a8,4578
-nkululeko/feat_extract/feats_trill.py,sha256=TUCrh5xbfnHD2gzb9mlkMSV4aK6YXazMqsh5xJ5yzUI,3188
+nkululeko/feat_extract/feats_trill.py,sha256=JgyUQ8ihIL5PlUpxjchlbC9547GI0SyUwkFEquya85Q,3197
 nkululeko/feat_extract/feats_wav2vec2.py,sha256=WYB9XlRzgDi8cGSKzhV5jahA0GZ_SiWgaQ25IcEemto,5296
 nkululeko/feat_extract/feats_wavlm.py,sha256=O9cfc39VF5aPJRRATKb37pHT4W11i2cu5O1mY9LOjIA,4755
 nkululeko/feat_extract/feats_whisper.py,sha256=n3ESZtva7wshs8E8diBlQYa9xCH_P0UY1DncSrxz-FY,4508
@@ -102,7 +102,7 @@ nkululeko/reporting/defines.py,sha256=0vh-Tlx4fAPpk1o6mP_4x3EkIoqzYMr38IZnj-JM5z
 nkululeko/reporting/latex_writer.py,sha256=NGwSIfd4nfslDkNUOSZSdqY_VDLA8634thyhe-vj1bY,1824
 nkululeko/reporting/report.py,sha256=bYN8B66gg3IWHAyfd6uIVjpYKy3rOI6aEwgfXU0LSAY,1006
 nkululeko/reporting/report_item.py,sha256=AqKD40AlZpRuHLbggn5PkH6ctGJwh9rGNBNgOvgUODg,534
-nkululeko/reporting/reporter.py,sha256=4OlYZAParkfJKO_aAyxqVpLc21zxZ-jDhtJKIMeUssI,20151
+nkululeko/reporting/reporter.py,sha256=nDQsj0xl3_a1Rn3CAAdbWIenodDIT72LsYFNzCaKtmg,20384
 nkululeko/reporting/result.py,sha256=G63a2tHCwHhM6NBJgYzsWKWJm4Yu3r4hsCHA2Km7eHU,1073
 nkululeko/segmenting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/segmenting/seg_inaspeechsegmenter.py,sha256=b3t0zdpJYofKWMyKRMtMMX91xeR-k8d5pbnNaQHcsOE,1902
@@ -112,9 +112,9 @@ nkululeko/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/utils/files.py,sha256=SrrYaU7AB80MZHiV1jcB0h_zigvYLYgSVNTXV4ao38g,4593
 nkululeko/utils/stats.py,sha256=vCRzhCR0Gx5SiJyAGbj1TIto8ocGz58CM5Pr3LltagA,2948
 nkululeko/utils/util.py,sha256=wFDslqxpCVDwi6LBakIFDDy1kYsxt5G7ykE38CocmtA,16880
-nkululeko-0.93.6.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
-nkululeko-0.93.6.dist-info/METADATA,sha256=Dx6ZisILoZzC7-BXKBskHm-mBPzjY8CEaiRVOo2vlc0,42364
-nkululeko-0.93.6.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-nkululeko-0.93.6.dist-info/entry_points.txt,sha256=lNTkFEdh6Kjo5o95ZAWf_0Lq-4ztGoAoMVSDuPtuyS0,442
-nkululeko-0.93.6.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
-nkululeko-0.93.6.dist-info/RECORD,,
+nkululeko-0.93.8.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
+nkululeko-0.93.8.dist-info/METADATA,sha256=QbeZSf2hWV3iClfliA5r6us-ZE6tHeluJffh5dqyKXA,42477
+nkululeko-0.93.8.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+nkululeko-0.93.8.dist-info/entry_points.txt,sha256=lNTkFEdh6Kjo5o95ZAWf_0Lq-4ztGoAoMVSDuPtuyS0,442
+nkululeko-0.93.8.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
+nkululeko-0.93.8.dist-info/RECORD,,