britekit 0.0.7__py3-none-any.whl → 0.0.8__py3-none-any.whl

britekit/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2025-present Jan Huus <jhuus1@gmail.com>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.0.7"
+__version__ = "0.0.8"

britekit/commands/_reports.py

@@ -276,14 +276,14 @@ def rpt_epochs(
             tester.initialize()
 
             pr_stats = tester.get_pr_auc_stats()
-            pr_score = pr_stats["macro_pr_auc"]
+            pr_score = pr_stats["micro_pr_auc"]
             pr_scores.append(pr_score)
             if pr_score > max_pr_score:
                 max_pr_score = pr_score
                 max_pr_epoch = epoch_num
 
             roc_stats = tester.get_roc_auc_stats()
-            roc_score = roc_stats["macro_roc_auc"]
+            roc_score = roc_stats["micro_roc_auc"]
             roc_scores.append(roc_score)
             if roc_score > max_roc_score:
                 max_roc_score = roc_score
@@ -323,8 +323,8 @@ def rpt_epochs(
     plot_path = str(Path(output_path) / "training_scores.jpeg")
     plt.savefig(plot_path, dpi=300, bbox_inches="tight")
 
-    logging.info(f"Maximum PR-AUC score = {max_pr_score:.3f} at epoch {max_pr_epoch}")
-    logging.info(f"Maximum ROC-AUC score = {max_roc_score:.3f} at epoch {max_roc_epoch}")
+    logging.info(f"Maximum micro-averaged PR-AUC score = {max_pr_score:.3f} at epoch {max_pr_epoch}")
+    logging.info(f"Maximum micro-averaged ROC-AUC score = {max_roc_score:.3f} at epoch {max_roc_epoch}")
     logging.info(f"See plot at {plot_path}")
 
 

britekit/core/data_module.py

@@ -52,6 +52,7 @@ class DataModule(LightningDataModule):
             )
         except Exception as e:
             logging.error(f"Failed to load training data: {e}")
+            raise
 
         if (
             self.cfg.train.noise_class_name

britekit/core/trainer.py

@@ -1,4 +1,5 @@
 # Defer some imports to improve initialization performance.
+import logging
 from pathlib import Path
 
 from britekit.core.config_loader import get_config
@@ -43,6 +44,7 @@ class Trainer:
         # load all the data once for performance, then split as needed in each fold
         dm = DataModule()
 
+        val_rocs = []
         for k in range(self.cfg.train.num_folds):
             logger = TensorBoardLogger(
                 save_dir="logs", name=f"fold-{k}", default_hp_metric=False
@@ -116,6 +118,26 @@ class Trainer:
             if self.cfg.train.test_pickle is not None:
                 trainer.test(model, dm)
 
+            # save stats from k-fold cross-validation
+            if self.cfg.train.num_folds > 1 and "val_roc" in trainer.callback_metrics:
+                val_rocs.append(float(trainer.callback_metrics["val_roc"]))
+
+        if val_rocs:
+            import math
+            import numpy as np
+            mean = float(np.mean(val_rocs))
+            std  = float(np.std(val_rocs, ddof=1)) if len(val_rocs) > 1 else 0.0
+            n = len(val_rocs)
+            se = std / math.sqrt(n) if n > 1 else 0.0
+            ci95 = 1.96 * se # 95% CI using normal approximation
+
+            logging.info("Using micro-averaged ROC AUC")
+            scores_str = ", ".join(f"{v:.4f}" for v in val_rocs)
+            logging.info(f"folds: {scores_str}")
+            logging.info(f"mean: {mean:.4f}")
+            logging.info(f"standard deviation: {std:.4f}")
+            logging.info(f"95% confidence interval: {mean-ci95:.4f} to {mean+ci95:.4f}")
+
     def find_lr(self, num_batches: int = 100):
         """
         Suggest a learning rate and produce a plot.

britekit/core/tuner.py

@@ -288,7 +288,7 @@ class Tuner:
         self.cfg.misc.ckpt_folder = str(
             Path(self.train_log_dir) / train_dir / "checkpoints"
         )
-        print(f"{self.cfg.misc.ckpt_folder=}")
+        logging.info(f"Using checkpoints in {self.cfg.misc.ckpt_folder}")
         self.cfg.infer.min_score = 0
 
         # suppress console output during inference and test analysis

{britekit-0.0.7.dist-info → britekit-0.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: britekit
-Version: 0.0.7
+Version: 0.0.8
 Summary: Core functions for bioacoustic recognizers.
 Project-URL: Documentation, https://github.com/jhuus/BriteKit#readme
 Project-URL: Issues, https://github.com/jhuus/BriteKit/issues

{britekit-0.0.7.dist-info → britekit-0.0.8.dist-info}/RECORD

@@ -4,15 +4,15 @@ britekit/core/audio.py,sha256=8QLbNDAiQyViEhrVC8jU0n32we4C22W_jPfc_KcOlmQ,15853
 britekit/core/augmentation.py,sha256=5_wyB-6gt7uM68Zl-rO_fPu1D6tlsd2m5oWhA6l0W9Q,5721
 britekit/core/base_config.py,sha256=wbCJI9cEH9mktUTSfLSCnU5AhJT6xhxhwZS4QWRYTpM,8744
 britekit/core/config_loader.py,sha256=epHNlH7yi_sCJX01FwgOsM6vPFk25rgUwkkcoGvVsYg,1341
-britekit/core/data_module.py,sha256=x3c_sIq9Xh_ThW_v-NBU7BCk49xJKDOMU_tpmskJE-E,9090
+britekit/core/data_module.py,sha256=f6BL-ngqkklX06Q7xZ9PMKEkvfeCzQTLYdmXRg3RmCo,9108
 britekit/core/dataset.py,sha256=Xu-lTz3TsHMuW10lHg4NN_r1baS9OQEhAD7EVz1a3A4,5804
 britekit/core/exceptions.py,sha256=ti_ve7ZdhDmzgTuspXXqyw__SUt5NoAXGEwoe3agPU8,443
 britekit/core/pickler.py,sha256=Vj-_DdFQUQj2bIVoyWe5puI8g8dTP9x7ZavbvM1iQZo,5788
 britekit/core/plot.py,sha256=hLuLB1VdtdFyaSHVDGl5tjjFCRgOJJ1ucTVJHM_3D_0,5332
 britekit/core/predictor.py,sha256=u4H8horTTvcg4Oqfpy5PG44eiiMeR5RU3aPZnMiXRCw,22914
 britekit/core/reextractor.py,sha256=gazhIZN8V1K4T_Q_kc-ihxUYbkNnc_hoAS6bpYQc95I,8396
-britekit/core/trainer.py,sha256=uKSXsrehb7-IPSCaw0oPpGBoDOcjDtaQovTt022kvzw,5467
-britekit/core/tuner.py,sha256=m0z4nEqNfCckX3GJWiAAYvsD-ND6gJWkMZupcVvq5is,16408
+britekit/core/trainer.py,sha256=N5EsbCzxw3wXxs2PTJJ0OfYFkIi49HCRM0ylT5zSSZk,6439
+britekit/core/tuner.py,sha256=FMmy4p3_j2Tojs4ONPzuUeRpCPWGlttr4rUJac7Hkyk,16435
 britekit/core/util.py,sha256=0JsEEN09hFPQzuttCKaejWofXAjCGSvWEewjkiLAh3E,19172
 britekit/models/base_model.py,sha256=9T7TwHx3K8fl10Vb-qUuypK3NDDZM-ktB8ZLHzqQhdc,16883
 britekit/models/dla.py,sha256=ALMY997AbERN7-sHqQuE5e43llRjpUDPZSFGL-Flv4M,3137
@@ -32,7 +32,7 @@ britekit/testing/per_segment_tester.py,sha256=FnaozQ8VmH99aYc1ibmDFfOk_ADgsXQGU_
 britekit/training_db/extractor.py,sha256=pT7lAUsNzYs3RXDzpMv7q0MKg6TktiFLKrRtKTWv6ho,8409
 britekit/training_db/training_data_provider.py,sha256=V5aBjsCvrWViZ0Jv05hgcKRizcAXmqoj4q3hAHedoD8,5651
 britekit/training_db/training_db.py,sha256=OOfD1pcbq5HVJbzhmuI-D-gkPHWSoz0cCO4zIUGFvoY,65011
-britekit/__about__.py,sha256=wVVqKeDr1kbfywz7cERdcfhJ0MkM0r5qQboD-dAkPOg,122
+britekit/__about__.py,sha256=-uGInVbPaVLti1Rr4PYUteRetwYfxeLtIuqiLmEcRjA,122
 britekit/__init__.py,sha256=RpruzdjbvTcFNf21zJYY8HrAhJei91FtNNLjIBmw-kw,1857
 britekit/install/data/classes.csv,sha256=OdTZ8oQdx7N-HKyhftxZStGZYsjhCy4UbanwtQJ2wBM,54
 britekit/install/data/ignore.txt,sha256=RbKvEHtUCbgRYolwR1IucClwyD3q7l2s6QuRjph-Us4,68
@@ -79,7 +79,7 @@ britekit/commands/_init.py,sha256=FmaQRY-7SYSHCLXL__47LEPecWir7X6zEB05KpradFw,28
 britekit/commands/_pickle.py,sha256=p990FsJGfSXcgjtBzH7nPGPh023b8cH0D7RZywQQ5Aw,3488
 britekit/commands/_plot.py,sha256=7vZXsYP9dv4PbHb8K3YbJFZc65YoPIBjEMBolyh6Has,13084
 britekit/commands/_reextract.py,sha256=kCmSjeghg6mhrJ46ibRTmBkGVytU7flFvTbqsnYhBvY,3770
-britekit/commands/_reports.py,sha256=v1eXJzMoSE-Qlwwyj_d0df9NWpG2yWLMzvvxu5axz7k,21993
+britekit/commands/_reports.py,sha256=KVYtpeFQpUC4jAIm2k2xV7aiNq826DL6sUrYEJD38X0,22023
 britekit/commands/_search.py,sha256=HIUXwfPvh3rxpgaFSR3bAAI38OtGVPyMo5GMfLtLX-8,9991
 britekit/commands/_train.py,sha256=vGFKlfcv35cOelArQNbVbTRbDWogT_IMg0wZt5virHY,4158
 britekit/commands/_tune.py,sha256=8dEZZURE769C0JZwhNpzB6pQxVklzl2w2cyXyWyhWXs,7331
@@ -87,8 +87,8 @@ britekit/commands/_wav2mp3.py,sha256=2Q4cjT6OhJmBPTNzGRMrDd6dSdBBufuQdjhH1V8ghLo
 britekit/commands/_xeno.py,sha256=_6YxQ7xFdaSy5DNUaigkbYp3E8EhtOhTC9b6OFS0MFA,6026
 britekit/commands/_youtube.py,sha256=_u1LrwY_2GxllKd505N_2ArFMbACQ_PtVxuqUCYxFe0,2214
 britekit/core/__init__.py,sha256=QcjcFyvO5KqJLF_HBeqiCk925uU5jTUjIV5lJix9XY4,556
-britekit-0.0.7.dist-info/METADATA,sha256=y8JY_v_wLMCsG6-zaSY9O9qgGj5S6Ry2sXQkPRS6VdU,18555
-britekit-0.0.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-britekit-0.0.7.dist-info/entry_points.txt,sha256=ycnPy5DLX14RTf7lKfkQAVyIf1B1zTL1gMsHm455wmg,46
-britekit-0.0.7.dist-info/licenses/LICENSE.txt,sha256=kPoHm6iop8-CUa_720Tt8gqyvLD6D_7218u1hCCpErk,1092
-britekit-0.0.7.dist-info/RECORD,,
+britekit-0.0.8.dist-info/METADATA,sha256=Qtzlff9X_WI1Cz8zpTyntAwFemS8hNbS0ClWJV9KVXk,18555
+britekit-0.0.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+britekit-0.0.8.dist-info/entry_points.txt,sha256=ycnPy5DLX14RTf7lKfkQAVyIf1B1zTL1gMsHm455wmg,46
+britekit-0.0.8.dist-info/licenses/LICENSE.txt,sha256=kPoHm6iop8-CUa_720Tt8gqyvLD6D_7218u1hCCpErk,1092
+britekit-0.0.8.dist-info/RECORD,,