nkululeko 0.86.4__py3-none-any.whl → 0.86.6__py3-none-any.whl

nkululeko/constants.py

@@ -1,2 +1,2 @@
-VERSION="0.86.4"
+VERSION="0.86.6"
 SAMPLING_RATE = 16000

nkululeko/data/dataset_csv.py

@@ -59,6 +59,20 @@ class Dataset_CSV(Dataset):
                         lambda x: root + "/" + audio_path + "/" + x
                     )
                 )
+        else: # absolute path is True
+            if audformat.index_type(df.index) == "segmented":
+                file_index = (
+                    df.index.levels[0]
+                    .map(lambda x: audio_path + "/" + x)
+                    .values
+                )
+                df = df.set_index(df.index.set_levels(
+                    file_index, level="file"))
+            else:
+                if not isinstance(df, pd.DataFrame):
+                    df = pd.DataFrame(df)
+                df = df.set_index(df.index.to_series().apply(
+                    lambda x: audio_path + "/" + x ))
 
         self.df = df
         self.db = None

nkululeko/experiment.py

@@ -112,11 +112,11 @@ class Experiment:
         auto_labels = list(next(iter(self.datasets.values())).df[self.target].unique())
         if labels:
             self.labels = ast.literal_eval(labels)
-            self.util.debug(f"Target labels (from config): {labels}")
+            self.util.debug(f"Using target labels (from config): {labels}")
         else:
             self.labels = auto_labels
         # print autolabel no matter it is specified or not
-        self.util.debug(f"Target labels (from database): {auto_labels}")
+        self.util.debug(f"Labels (from database): {auto_labels}")
         glob_conf.set_labels(self.labels)
         self.util.debug(f"loaded databases {dbs}")
 

nkululeko/modelrunner.py

@@ -30,6 +30,8 @@ class Modelrunner:
         # intialize a new model
         model_type = glob_conf.config["MODEL"]["type"]
         self._select_model(model_type)
+        self.best_performance = 0
+        self.best_epoch = 0
 
     def do_epochs(self):
         # initialze results
@@ -51,7 +53,8 @@ class Modelrunner:
             # epochs are handled by Huggingface API
             self.model.train()
             report = self.model.predict()
-            # todo: findout the best epoch
+            # todo: findout the best epoch, no need
+            # since oad_best_model_at_end is given in training args
             epoch = epoch_num
             report.set_id(self.run, epoch)
             plot_name = self.util.get_plot_name() + f"_{self.run}_{epoch:03d}_cnf"
@@ -77,10 +80,15 @@ class Modelrunner:
                 report.set_id(self.run, epoch)
                 plot_name = self.util.get_plot_name() + f"_{self.run}_{epoch:03d}_cnf"
                 reports.append(report)
+                test_score_metric = report.get_result().get_test_result()
                 self.util.debug(
-                    f"run: {self.run} epoch: {epoch}: result: "
-                    f"{reports[-1].get_result().get_test_result()}"
+                    f"run: {self.run} epoch: {epoch}: result: {test_score_metric}"
                 )
+                # print(f"performance: {performance.split(' ')[1]}")
+                performance = float(test_score_metric.split(' ')[1])
+                if performance > self.best_performance:
+                    self.best_performance = performance
+                    self.best_epoch = epoch
                 if plot_epochs:
                     self.util.debug(f"plotting conf matrix to {plot_name}")
                     report.plot_confmatrix(plot_name, epoch)
@@ -110,11 +118,14 @@ class Modelrunner:
                             f"reached patience ({str(patience)}): early stopping"
                         )
                         break
+        # After training, report the best performance and epoch
+        best_report = reports[self.best_epoch]
+        # self.util.debug(f"Best score at epoch: {self.best_epoch}, UAR: {self.best_performance}") # move to reporter below
 
         if not plot_epochs:
             # Do at least one confusion matrix plot
             self.util.debug(f"plotting confusion matrix to {plot_name}")
-            reports[-1].plot_confmatrix(plot_name, epoch)
+            best_report.plot_confmatrix(plot_name, self.best_epoch)
         return reports, epoch
 
     def _select_model(self, model_type):

nkululeko/reporting/reporter.py

@@ -7,10 +7,11 @@ from confidence_intervals import evaluate_with_conf_int
 import matplotlib.pyplot as plt
 import numpy as np
 from scipy.stats import pearsonr
-from sklearn.metrics import ConfusionMatrixDisplay
+from sklearn.metrics import ConfusionMatrixDisplay, roc_curve
 from sklearn.metrics import classification_report
 from sklearn.metrics import confusion_matrix
 from sklearn.metrics import r2_score
+from sklearn.metrics import roc_curve, auc, roc_auc_score
 from torch import is_tensor
 
 from audmetric import accuracy
@@ -121,7 +122,7 @@ class Reporter:
         self.truths = np.digitize(self.truths, bins) - 1
         self.preds = np.digitize(self.preds, bins) - 1
 
-    def plot_confmatrix(self, plot_name, epoch):
+    def plot_confmatrix(self, plot_name, epoch=None):
         if not self.util.exp_is_classification():
             self.continuous_to_categorical()
         self._plot_confmat(self.truths, self.preds, plot_name, epoch)
@@ -155,9 +156,11 @@ class Reporter:
             pred = np.digitize(pred, bins) - 1
         self._plot_confmat(truth, pred.astype("int"), plot_name, 0)
 
-    def _plot_confmat(self, truths, preds, plot_name, epoch):
+    def _plot_confmat(self, truths, preds, plot_name, epoch=None):
         # print(truths)
         # print(preds)
+        if epoch is None:
+            epoch = self.epoch
         fig_dir = self.util.get_path("fig_dir")
         labels = glob_conf.labels
         fig = plt.figure()  # figsize=[5, 5]
@@ -224,7 +227,7 @@ class Reporter:
 
         res_dir = self.util.get_path("res_dir")
         rpt = (
-            f"epoch: {epoch}, UAR: {uar_str}"
+            f"Best score at epoch: {epoch}, UAR: {uar_str}"
             + f", (+-{up_str}/{low_str}), ACC: {acc_str}"
         )
         # print(rpt)
@@ -236,7 +239,9 @@ class Reporter:
     def set_filename_add(self, my_string):
         self.filenameadd = f"_{my_string}"
 
-    def print_results(self, epoch):
+    def print_results(self, epoch=None):
+        if epoch is None:
+            epoch = self.epoch
         """Print all evaluation values to text file."""
         res_dir = self.util.get_path("res_dir")
         file_name = f"{res_dir}{self.util.get_exp_name()}_{epoch}{self.filenameadd}.txt"
@@ -261,9 +266,18 @@ class Reporter:
                     c_res = rpt[l]["f1-score"]
                     c_ress[i] = float(f"{c_res:.3f}")
                 self.util.debug(f"labels: {labels}")
-                f1_per_class = f"result per class (F1 score): {c_ress}"
+                f1_per_class = (
+                    f"result per class (F1 score): {c_ress} from epoch: {epoch}"
+                )
+                if len(np.unique(self.truths)) == 2:
+                    fpr, tpr, _ = roc_curve(self.truths, self.preds)
+                    auc_score = auc(fpr, tpr)
+                    pauc_score = roc_auc_score(self.truths, self.preds, max_fpr=0.1)
+                    auc_pauc = f"auc: {auc_score:.3f}, pauc: {pauc_score:.3f} from epoch: {epoch}"
+                    self.util.debug(auc_pauc)
                 self.util.debug(f1_per_class)
                 rpt_str = f"{json.dumps(rpt)}\n{f1_per_class}"
+                # rpt_str += f"\n{auc_auc}"
                 text_file.write(rpt_str)
                 glob_conf.report.add_item(
                     ReportItem(

nkululeko/runmanager.py

@@ -63,8 +63,7 @@ class Runmanager:
             )
             self.reports, last_epoch = self.modelrunner.do_epochs()
             # wrap up the run
-            plot_anim_progression = self.util.config_val(
-                "PLOT", "anim_progression", 0)
+            plot_anim_progression = self.util.config_val("PLOT", "anim_progression", 0)
             if plot_anim_progression:
                 plot_name_suggest = self.util.get_exp_name()
                 plot_name = (
@@ -88,8 +87,7 @@ class Runmanager:
                     + "_epoch_progression"
                 )
                 self.util.debug(f"plotting progression to {plot_name}")
-                self.reports[-1].plot_epoch_progression(
-                    self.reports, plot_name)
+                self.reports[-1].plot_epoch_progression(self.reports, plot_name)
             # remember the best run
             best_report = self.get_best_result(self.reports)
             plot_best_model = self.util.config_val("PLOT", "best_model", False)
@@ -107,9 +105,10 @@ class Runmanager:
                 )
                 self.print_model(best_report, plot_name)
             # finally, print out the numbers for this run
-            self.reports[-1].print_results(
-                int(self.util.config_val("EXP", "epochs", 1))
-            )
+            # self.reports[-1].print_results(
+            #     int(self.util.config_val("EXP", "epochs", 1))
+            # )
+            best_report.print_results(best_report.epoch)
             self.best_results.append(best_report)
             self.last_epochs.append(last_epoch)
 
@@ -145,19 +144,19 @@ class Runmanager:
         )
         self.print_model(report, plot_name)
 
-    def print_model(self, report, plot_name):
+    def print_model(self, reporter, plot_name):
         """Print a confusion matrix for a special report.
 
         Args:
             report: for which report (will be computed newly from model)
             plot_name: name of plot file
         """
-        epoch = report.epoch
+        epoch = reporter.epoch
         # self.load_model(report)
         # report = self.model.predict()
         self.util.debug(f"plotting conf matrix to {plot_name}")
-        report.plot_confmatrix(plot_name, epoch)
-        report.print_results(epoch)
+        reporter.plot_confmatrix(plot_name)
+        reporter.print_results()
 
     def load_model(self, report):
         """Load a model from disk for a specific run and epoch and evaluate it.

{nkululeko-0.86.4.dist-info → nkululeko-0.86.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.86.4
+Version: 0.86.6
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -58,6 +58,8 @@ Requires-Dist: pylatex
   - [Hello World example](#hello-world-example)
   - [Features](#features)
 - [License](#license)
+- [Contributing](#contributing)
+- [Citing](#citing)
 
  
 ## Overview
@@ -65,7 +67,7 @@ A project to detect speaker characteristics by machine learning experiments with
 
 The idea is to have a framework (based on e.g. sklearn and torch) that can be used to rapidly and automatically analyse audio data and explore machine learning models based on that data.
 
-* NEW: Nkululeko now automatically generates PDF reports [sample for EmoDB](meta/images/emodb_report.pdf)
+* NEW with nkululeko: [Finetune transformer-models](http://blog.syntheticspeech.de/2024/05/29/nkululeko-how-to-finetune-a-transformer-model/)
 * The latest features can be seen in [the ini-file](./ini_file.md) options that are used to control Nkululeko
 * Below is a [Hello World example](#helloworld) that should set you up fastly, also on [Google Colab](https://colab.research.google.com/drive/1GYNBd5cdZQ1QC3Jm58qoeMaJg3UuPhjw?usp=sharing#scrollTo=4G_SjuF9xeQf), and [with Kaggle](https://www.kaggle.com/felixburk/nkululeko-hello-world-example)
 * [Here's a blog post on how to set up nkululeko on your computer.](http://blog.syntheticspeech.de/2021/08/30/how-to-set-up-your-first-nkululeko-project/)
@@ -249,7 +251,7 @@ There's my [blog](http://blog.syntheticspeech.de/?s=nkululeko) with tutorials:
 * [Predict new labels for your data from public models and check bias](http://blog.syntheticspeech.de/2023/08/16/nkululeko-how-to-predict-labels-for-your-data-from-existing-models-and-check-them/)
 * [Resample](http://blog.syntheticspeech.de/2023/08/31/how-to-fix-different-sampling-rates-in-a-dataset-with-nkululeko/)
 * [Get some statistics on correlation and effect-size](http://blog.syntheticspeech.de/2023/09/05/nkululeko-get-some-statistics-on-correlation-and-effect-size/)
-* [Generate a latex / pdf report](http://blog.syntheticspeech.de/2023/09/26/nkululeko-generate-a-latex-pdf-report/) 
+* [Automatic generation  of a latex / pdf report](http://blog.syntheticspeech.de/2023/09/26/nkululeko-generate-a-latex-pdf-report/) 
 * [Inspect your data with Spotlight](http://blog.syntheticspeech.de/2023/10/31/nkululeko-inspect-your-data-with-spotlight/)
 * [Automatically stratify your split sets](http://blog.syntheticspeech.de/2023/11/07/nkululeko-automatically-stratify-your-split-sets/)
 * [re-name data column names](http://blog.syntheticspeech.de/2023/11/16/nkululeko-re-name-data-column-names/)
@@ -314,6 +316,12 @@ Here's [an animation that shows the progress of classification done with nkulule
 
 ## License
 Nkululeko can be used under the [MIT license](https://choosealicense.com/licenses/mit/)
+
+
+## Contributing
+Contributions are welcome and encouraged. To learn more about how to contribute to nkululeko please refer to the [Contributing guidelines](./CONTRIBUTING.md)
+
+## Citing
 If you use it, please mention the Nkululeko paper
 
 F. Burkhardt, Johannes Wagner, Hagen Wierstorf, Florian Eyben and Björn Schuller: Nkululeko: A Tool For Rapid Speaker Characteristics Detection, Proc. Proc. LREC, 2022
@@ -335,6 +343,14 @@ F. Burkhardt, Johannes Wagner, Hagen Wierstorf, Florian Eyben and Björn Schulle
 Changelog
 =========
 
+Version 0.86.6
+--------------
+* now best (not last) result is shown at end
+
+Version 0.86.5
+--------------
+* fix audio path detection in data csv import
+
 Version 0.86.4
 --------------
 * add finetuning to the demo module

{nkululeko-0.86.4.dist-info → nkululeko-0.86.6.dist-info}/RECORD

@@ -2,25 +2,25 @@ nkululeko/__init__.py,sha256=62f8HiEzJ8rG2QlTFJXUCMpvuH3fKI33DoJSj33mscc,63
 nkululeko/aug_train.py,sha256=YhuZnS_WVWnun9G-M6g5n6rbRxoVREz6Zh7k6qprFNQ,3194
 nkululeko/augment.py,sha256=4MG0apTAG5RgkuJrYEjGgDdbodZWi_HweSPNI1JJ5QA,3051
 nkululeko/cacheddataset.py,sha256=lIJ6hUo5LoxSrzXtWV8mzwO7wRtUETWnOQ4ws2XfL1E,969
-nkululeko/constants.py,sha256=QtEoU6iCjnUpcJT-FOh4bU4miJ_D0z26OpSub4oEY1c,39
+nkululeko/constants.py,sha256=r-AT0ZtgRKUYQ1VxJ3nCCqbKQzOAQiOexnQFaojN3Z0,39
 nkululeko/demo.py,sha256=WSKr-W5uJ9DQfemK923g7Hd5V3kgAn03Er0JX1Pa45I,5142
 nkululeko/demo_feats.py,sha256=sAeGFojhEj9WEDFtG3SzPBmyYJWLF2rkbpp65m8Ujo4,2025
 nkululeko/demo_predictor.py,sha256=es56xbT8ifkS_vnrlb5NTZT54gNmeUtNlA4zVA_gnN8,4757
-nkululeko/experiment.py,sha256=huhHLQfnzxRJlQi2SY61XMWbC8xEWpe31yq9spBUk-4,31041
+nkululeko/experiment.py,sha256=5nF-eDf8OCp6KRIU7KnryWL5SLJQUtr2BueHhEdcKw0,31040
 nkululeko/explore.py,sha256=lDzRoW_Taa5u4BBABZLD89BcQWnYlrftJR4jgt1yyj0,2609
 nkululeko/export.py,sha256=mHeEAAmtZuxdyebLlbSzPrHSi9OMgJHbk35d3DTxRBc,4632
 nkululeko/feature_extractor.py,sha256=8mssYKmo4LclVI-hiLmJEDZ0ZPyDavFG2YwtXcrGzwM,3976
 nkululeko/file_checker.py,sha256=LoLnL8aHpW-axMQ46qbqrManTs5otG9ShpEZuz9iRSk,3474
 nkululeko/filter_data.py,sha256=w-X2mhKdYr5DxDIz50E5yzO6Jmzk4jjDBoXsgOOVtcA,7222
 nkululeko/glob_conf.py,sha256=KL9YJQTHvTztxo1vr25qRRgaPnx4NTg0XrdbovKGMmw,525
-nkululeko/modelrunner.py,sha256=iCmfJxsS2UafcikjRdUqPQuqQMOYA-Ctr3et3HeNR3c,10452
+nkululeko/modelrunner.py,sha256=OU35qwP94GxW_EtL4I2-RhqB-wxbjNvp8CIHNbtnt7Q,11155
 nkululeko/multidb.py,sha256=fG3VukEWP1vreVN4gB1IRXxwwg4jLftsSEYtu0o1f78,5634
 nkululeko/nkuluflag.py,sha256=PGWSmZz-PiiHLgcZJAoGOI_Y-sZDVI1ksB8p5r7riWM,3725
 nkululeko/nkululeko.py,sha256=Kn3s2E3yyH8cJ7z6lkMxrnqtCxTu7-qfe9Zr_ONTD5g,1968
 nkululeko/plots.py,sha256=C2mwQFK0Vxfl5ZM7CO87tULDoEf7G16ek0nU77bhOc4,23070
 nkululeko/predict.py,sha256=sF091sSSLnEWcISx9ZcULLie3tY5XeFsQJd6b3vrxFg,2409
 nkululeko/resample.py,sha256=2d9eao_0sLrGZ_KSl8OVKsPor3BkFrlmMhrpB9WelIs,4267
-nkululeko/runmanager.py,sha256=eTM1DNQKt1lxYhzt4vZyZluPXW9sWlIJHNQzex4lkJU,7624
+nkululeko/runmanager.py,sha256=ul1fIxJln3LUd9N8QZOMyWdC3alcsvaygdJ-7B63FDQ,7645
 nkululeko/scaler.py,sha256=4nkIqoajkIkuTPK0Z02ifMN_awl6fP_i-GBYdoGYgGM,4101
 nkululeko/segment.py,sha256=YLKckX44tbvTb3LrdgYw9X4guzuF27sutl92z9DkpZU,4835
 nkululeko/syllable_nuclei.py,sha256=Sky-C__MeUDaxqHnDl2TGLLYOYvsahD35TUjWGeG31k,10047
@@ -46,7 +46,7 @@ nkululeko/autopredict/ap_valence.py,sha256=n-hctRKySzhmJtowuMOTUu0T_ld3uK5pnfOzW
 nkululeko/autopredict/estimate_snr.py,sha256=S-bpS0xFkwWc4Ch75UrjbS8y538lQ0U3g_iLRFXureY,5048
 nkululeko/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/data/dataset.py,sha256=JGzMD6HIvkFkYBekmbmslIKc5ADaCj06T-8gpqH_kFo,27650
-nkululeko/data/dataset_csv.py,sha256=vTnjIc2UdSJT7foL-ltE9MWrZTCg0nplwKdEtMPxt2o,3933
+nkululeko/data/dataset_csv.py,sha256=dzOrbKB8t0UATAIYaKAOqHTogmYPBqskt6Hak7VjbSM,4537
 nkululeko/feat_extract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/feat_extract/feats_agender.py,sha256=Qm69G4kqAyTVVk7wwRgrXlNwGaDMGRYyKGpuf0vOEgM,3113
 nkululeko/feat_extract/feats_agender_agender.py,sha256=tgH2BnwcxpvuLmOkrMbVdBSX0Onfz2MG12FsddalRKI,3424
@@ -96,7 +96,7 @@ nkululeko/reporting/defines.py,sha256=IsY1YgKRMaABpylVKjBJgJ5bNCEbGCVA_E6pivraqS
 nkululeko/reporting/latex_writer.py,sha256=qiCRSmB4KOD_za4oHu5x-PhwjZohzfo8wecMOwlXZwc,1886
 nkululeko/reporting/report.py,sha256=W0rcigDdjBvxZQ3pZja_gvToILYvaZ1BFtnN2qFRfYI,1060
 nkululeko/reporting/report_item.py,sha256=siWeGNgo4bAE46YBMNcsdf3jTMTy76BO9Fi6DTvDig4,533
-nkululeko/reporting/reporter.py,sha256=4dXRwJ-CZ49NlF_kv9hfDjZT3bbWNNMyNpKEBLKs3Ew,13447
+nkululeko/reporting/reporter.py,sha256=S9A62AxdMTEV-9XDUQNxdoevGLXBP52WiDmZ694QMV4,14161
 nkululeko/reporting/result.py,sha256=nSN5or-Py2GPRWHkWpGRh7UCi1W0er7WLEHz8fYLk-A,742
 nkululeko/segmenting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/segmenting/seg_inaspeechsegmenter.py,sha256=pmLHuXsaqvcdYxB4PSW9l1mbQWZZBJFhi_CGabqydas,1947
@@ -105,8 +105,8 @@ nkululeko/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/utils/files.py,sha256=UiGAtZRWYjHSvlmPaTMtzyNNGE6qaLaxQkybctS7iRM,4021
 nkululeko/utils/stats.py,sha256=1yUq0FTOyqkU8TwUocJRYdJaqMU5SlOBBRUun9STo2M,2829
 nkululeko/utils/util.py,sha256=ILpfNuaeq-hy1bUkRhVrzO2wG9z9Upaozs9EBoIaMG0,14123
-nkululeko-0.86.4.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
-nkululeko-0.86.4.dist-info/METADATA,sha256=D1y8wrwDr0gLVdafV4E_GcER5yrt3IaKUdqJ8huMCwA,37480
-nkululeko-0.86.4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-nkululeko-0.86.4.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
-nkululeko-0.86.4.dist-info/RECORD,,
+nkululeko-0.86.6.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
+nkululeko-0.86.6.dist-info/METADATA,sha256=2aKLKe_0C1iMKL1tZc5aLiMxp8GocGdQpnAQL4Z9agA,37924
+nkululeko-0.86.6.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+nkululeko-0.86.6.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
+nkululeko-0.86.6.dist-info/RECORD,,