nkululeko 0.86.2__py3-none-any.whl → 0.86.3__py3-none-any.whl

nkululeko/constants.py

@@ -1,2 +1,2 @@
-VERSION="0.86.2"
+VERSION="0.86.3"
 SAMPLING_RATE = 16000

nkululeko/feat_extract/feats_import.py

@@ -35,6 +35,7 @@ class ImportSet(Featureset):
             if not os.path.isfile(feat_import_file):
                 self.util.error(f"no import file: {feat_import_file}")
             df = audformat.utils.read_csv(feat_import_file)
+            df = self.util.make_segmented_index(df)
             df = df[df.index.isin(self.data_df.index)]
             feat_df = pd.concat([feat_df, df])
         if feat_df.shape[0] == 0:

nkululeko/models/model_tuned.py

@@ -62,11 +62,13 @@ class TunedModel(BaseModel):
         if drop:
             self.drop = float(drop)
         self.util.debug(f"init: training with dropout: {self.drop}")
+        self.push = eval(self.util.config_val("MODEL", "push_to_hub", "False"))
         self._init_model()
 
     def _init_model(self):
         model_path = "facebook/wav2vec2-large-robust-ft-swbd-300h"
-        pretrained_model = self.util.config_val("MODEL", "pretrained_model", model_path)
+        pretrained_model = self.util.config_val(
+            "MODEL", "pretrained_model", model_path)
         self.num_layers = None
         self.sampling_rate = 16000
         self.max_duration_sec = self.max_duration
@@ -131,6 +133,11 @@ class TunedModel(BaseModel):
         tokenizer = transformers.Wav2Vec2CTCTokenizer("./vocab.json")
         tokenizer.save_pretrained(".")
 
+        # uoload tokenizer to hub if true
+        if self.push:
+            tokenizer.push_to_hub(self.util.get_name())
+
+
         feature_extractor = transformers.Wav2Vec2FeatureExtractor(
             feature_size=1,
             sampling_rate=16000,
@@ -273,7 +280,7 @@ class TunedModel(BaseModel):
                 self.util.error(f"criterion {criterion} not supported for regressor")
 
         # set push_to_hub value, default false
-        push = eval(self.util.config_val("MODEL", "push_to_hub", "False"))
+        # push = eval(self.util.config_val("MODEL", "push_to_hub", "False"))
 
         class Trainer(transformers.Trainer):
             def compute_loss(
@@ -312,7 +319,7 @@ class TunedModel(BaseModel):
             self.util.error(f"unknown metric/measure: {metrics_for_best_model}")
 
         training_args = transformers.TrainingArguments(
-            output_dir=model_root,
+            output_dir=self.torch_root,
             logging_dir=self.log_root,
             per_device_train_batch_size=self.batch_size,
             per_device_eval_batch_size=self.batch_size,
@@ -331,7 +338,7 @@ class TunedModel(BaseModel):
             load_best_model_at_end=True,
             remove_unused_columns=False,
             report_to="none",
-            push_to_hub=push,
+            push_to_hub=self.push,
             hub_model_id=f"{self.util.get_name()}",
         )
 
@@ -347,7 +354,7 @@ class TunedModel(BaseModel):
         )
 
         trainer.train()
-        trainer.save_model(self.torch_root)
+        # trainer.save_model(self.torch_root)
         log_file = os.path.join(
             self.log_root,
             "log.txt",
@@ -517,7 +524,10 @@ class Model(Wav2Vec2PreTrainedModel):
             )
             outputs = torch.sum(hidden_states, dim=1)
             attention_sum = torch.sum(attention_mask, dim=1)
-            outputs = outputs / torch.reshape(attention_sum, (-1, 1))
+            
+            epsilon = 1e-6 # to avoid division by zero and numerical instability
+            outputs = outputs / (torch.reshape(attention_sum, (-1, 1)) + 
+                                 epsilon)
 
         return outputs
 

nkululeko/utils/util.py

@@ -35,9 +35,9 @@ class Util:
         if has_config:
             try:
                 import nkululeko.glob_conf as glob_conf
+
                 self.config = glob_conf.config
-                self.got_data_roots = self.config_val(
-                    "DATA", "root_folders", False)
+                self.got_data_roots = self.config_val("DATA", "root_folders", False)
                 if self.got_data_roots:
                     # if there is a global data rootfolder file, read from there
                     if not os.path.isfile(self.got_data_roots):
@@ -116,8 +116,7 @@ class Util:
                         )
                     return default
             if not default in self.stopvals:
-                self.debug(
-                    f"value for {key} not found, using default: {default}")
+                self.debug(f"value for {key} not found, using default: {default}")
             return default
 
     def set_config(self, config):
@@ -160,8 +159,8 @@ class Util:
         if len(df) == 0:
             return df
         if not isinstance(df.index, pd.MultiIndex):
-            df.index = audformat.utils.to_segmented_index(
-                df.index, allow_nat=False)
+            self.debug("converting to segmented index, this might take a while...")
+            df.index = audformat.utils.to_segmented_index(df.index, allow_nat=False)
         return df
 
     def _get_value_descript(self, section, name):
@@ -272,8 +271,7 @@ class Util:
             return self.config[section][key]
         except KeyError:
             if default not in self.stopvals:
-                self.debug(
-                    f"value for {key} not found, using default: {default}")
+                self.debug(f"value for {key} not found, using default: {default}")
             return default
 
     def config_val_list(self, section, key, default):
@@ -281,8 +279,7 @@ class Util:
             return ast.literal_eval(self.config[section][key])
         except KeyError:
             if not default in self.stopvals:
-                self.debug(
-                    f"value for {key} not found, using default: {default}")
+                self.debug(f"value for {key} not found, using default: {default}")
             return default
 
     def continuous_to_categorical(self, series):

{nkululeko-0.86.2.dist-info → nkululeko-0.86.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.86.2
+Version: 0.86.3
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -334,6 +334,11 @@ F. Burkhardt, Johannes Wagner, Hagen Wierstorf, Florian Eyben and Björn Schulle
 Changelog
 =========
 
+Version 0.86.3
+--------------
+* bugfixed: nan in finetuned model and double saving
+* import features now get multiindex automatically
+
 Version 0.86.2
 --------------
 * plots epoch progression for finetuned models now

{nkululeko-0.86.2.dist-info → nkululeko-0.86.3.dist-info}/RECORD

@@ -2,7 +2,7 @@ nkululeko/__init__.py,sha256=62f8HiEzJ8rG2QlTFJXUCMpvuH3fKI33DoJSj33mscc,63
 nkululeko/aug_train.py,sha256=YhuZnS_WVWnun9G-M6g5n6rbRxoVREz6Zh7k6qprFNQ,3194
 nkululeko/augment.py,sha256=4MG0apTAG5RgkuJrYEjGgDdbodZWi_HweSPNI1JJ5QA,3051
 nkululeko/cacheddataset.py,sha256=lIJ6hUo5LoxSrzXtWV8mzwO7wRtUETWnOQ4ws2XfL1E,969
-nkululeko/constants.py,sha256=E9mXpvAI5IDamRRXgBlBH8XGTw1xEjEBzNibjhFPEFc,39
+nkululeko/constants.py,sha256=2ysebEFzu3zwO0-FXWf2pBOs8XRLPmy718GFrZ2O9pU,39
 nkululeko/demo.py,sha256=8bl15Kitoesnz8oa8yrs52T6YCSOhWbbq9PnZ8Hj6D0,3232
 nkululeko/demo_feats.py,sha256=sAeGFojhEj9WEDFtG3SzPBmyYJWLF2rkbpp65m8Ujo4,2025
 nkululeko/demo_predictor.py,sha256=es56xbT8ifkS_vnrlb5NTZT54gNmeUtNlA4zVA_gnN8,4757
@@ -55,7 +55,7 @@ nkululeko/feat_extract/feats_auddim.py,sha256=VlzKKXTXa5kjLgQBWyEFy-daIyU1SkOwCC
 nkululeko/feat_extract/feats_audmodel.py,sha256=VjBNgAoxsHJhwr6Kwt9CxX6SaCM4RK_OV-GU2W5-bhU,3187
 nkululeko/feat_extract/feats_clap.py,sha256=nR6eEIRdsMHcfmD1bNtt5WfDvkxKjvEbukSSrXHm-HU,3489
 nkululeko/feat_extract/feats_hubert.py,sha256=cLoUzSLjSYBkQnftjacSL7ES3O7Ysh_KrPYvZtLX_TU,5196
-nkululeko/feat_extract/feats_import.py,sha256=rj1p8lz19tCAC8hLzzZAwZ0M6gzwH3BzfabFUgal0yw,1622
+nkululeko/feat_extract/feats_import.py,sha256=WiU5lCkJsmFNTDyPV0qIh8mJssa6bpgP7AYw_ClKfWM,1674
 nkululeko/feat_extract/feats_mld.py,sha256=Vvu7GZOkn7Vda8eIOXqHjg78zegkFe3vTUaCXyVM0eA,2021
 nkululeko/feat_extract/feats_mos.py,sha256=KXNt7QYEfxkvr6UyVhig2aWQBaIvovlrR4gPuP03gmo,4174
 nkululeko/feat_extract/feats_opensmile.py,sha256=g6ZsAxjjGGvGfrr5fngWC-NJ8E7CP1kYZwrlodZJzzU,4028
@@ -88,7 +88,7 @@ nkululeko/models/model_svm.py,sha256=rsME3KvKvNG7bdE5lbvYUu85WZhaASZxxmdNDIVJRZ4
 nkululeko/models/model_svr.py,sha256=_YZeksqB3eBENGlg3g9RwYFlk9rQQ-XCeNBKLlGGVoE,725
 nkululeko/models/model_tree.py,sha256=rf16faUm4o2LJgkoYpeY998b8DQIvXZ73_m1IS3TnnE,417
 nkululeko/models/model_tree_reg.py,sha256=IgQcPTE-304HQLYSKPF8Z4ot_Ur9dH01fZjS0nXke_M,428
-nkululeko/models/model_tuned.py,sha256=xOoY5TROzzTVu3sDtlmEle3V1MAgpf8S3WxO9o4MzV4,20777
+nkululeko/models/model_tuned.py,sha256=RDcvcejBQNGY_uW00r22i7EDT6oKchS5uqFFnj0Gtzg,21146
 nkululeko/models/model_xgb.py,sha256=Thgx5ESdIok4v72mKh4plxpo4smGcKALWNCJTDScY0M,447
 nkululeko/models/model_xgr.py,sha256=aGBtNGLWjOE_2rICGYGFxmT8DtnHYsIl1lIpMtghHsY,418
 nkululeko/reporting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -104,9 +104,9 @@ nkululeko/segmenting/seg_silero.py,sha256=lLytS38KzARS17omwv8VBw-zz60RVSXGSvZ5Ev
 nkululeko/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/utils/files.py,sha256=UiGAtZRWYjHSvlmPaTMtzyNNGE6qaLaxQkybctS7iRM,4021
 nkululeko/utils/stats.py,sha256=1yUq0FTOyqkU8TwUocJRYdJaqMU5SlOBBRUun9STo2M,2829
-nkululeko/utils/util.py,sha256=mK1MgO14NinrPhavJw72eR_2WN_kBKjVKiEJnzvdO1Q,13946
-nkululeko-0.86.2.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
-nkululeko-0.86.2.dist-info/METADATA,sha256=DmmpMrftBptpWqx7h9US7_4mvMIQbZ5ugzv_4kyBjkM,37170
-nkululeko-0.86.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-nkululeko-0.86.2.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
-nkululeko-0.86.2.dist-info/RECORD,,
+nkululeko/utils/util.py,sha256=PcyAuCGgGuxjlv-e4JrVbpewiRTiAXWk47w5X0dVgx8,13930
+nkululeko-0.86.3.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
+nkululeko-0.86.3.dist-info/METADATA,sha256=Nnb3gRWEI1DSqf8KpaD8CDqdkHyiKdv-j9HpN4jjeks,37305
+nkululeko-0.86.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+nkululeko-0.86.3.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
+nkululeko-0.86.3.dist-info/RECORD,,