britekit 0.0.7__tar.gz → 0.0.8__tar.gz

{britekit-0.0.7 → britekit-0.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: britekit
-Version: 0.0.7
+Version: 0.0.8
 Summary: Core functions for bioacoustic recognizers.
 Project-URL: Documentation, https://github.com/jhuus/BriteKit#readme
 Project-URL: Issues, https://github.com/jhuus/BriteKit/issues

{britekit-0.0.7 → britekit-0.0.8}/britekit/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2025-present Jan Huus <jhuus1@gmail.com>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.0.7"
+__version__ = "0.0.8"

{britekit-0.0.7 → britekit-0.0.8}/britekit/commands/_reports.py

@@ -276,14 +276,14 @@ def rpt_epochs(
             tester.initialize()
 
             pr_stats = tester.get_pr_auc_stats()
-            pr_score = pr_stats["macro_pr_auc"]
+            pr_score = pr_stats["micro_pr_auc"]
             pr_scores.append(pr_score)
             if pr_score > max_pr_score:
                 max_pr_score = pr_score
                 max_pr_epoch = epoch_num
 
             roc_stats = tester.get_roc_auc_stats()
-            roc_score = roc_stats["macro_roc_auc"]
+            roc_score = roc_stats["micro_roc_auc"]
             roc_scores.append(roc_score)
             if roc_score > max_roc_score:
                 max_roc_score = roc_score
@@ -323,8 +323,8 @@ def rpt_epochs(
     plot_path = str(Path(output_path) / "training_scores.jpeg")
     plt.savefig(plot_path, dpi=300, bbox_inches="tight")
 
-    logging.info(f"Maximum PR-AUC score = {max_pr_score:.3f} at epoch {max_pr_epoch}")
-    logging.info(f"Maximum ROC-AUC score = {max_roc_score:.3f} at epoch {max_roc_epoch}")
+    logging.info(f"Maximum micro-averaged PR-AUC score = {max_pr_score:.3f} at epoch {max_pr_epoch}")
+    logging.info(f"Maximum micro-averaged ROC-AUC score = {max_roc_score:.3f} at epoch {max_roc_epoch}")
     logging.info(f"See plot at {plot_path}")
 
 

{britekit-0.0.7 → britekit-0.0.8}/src/britekit/core/data_module.py

@@ -52,6 +52,7 @@ class DataModule(LightningDataModule):
             )
         except Exception as e:
             logging.error(f"Failed to load training data: {e}")
+            raise
 
         if (
             self.cfg.train.noise_class_name

{britekit-0.0.7 → britekit-0.0.8}/src/britekit/core/trainer.py

@@ -1,4 +1,5 @@
 # Defer some imports to improve initialization performance.
+import logging
 from pathlib import Path
 
 from britekit.core.config_loader import get_config
@@ -43,6 +44,7 @@ class Trainer:
         # load all the data once for performance, then split as needed in each fold
         dm = DataModule()
 
+        val_rocs = []
         for k in range(self.cfg.train.num_folds):
             logger = TensorBoardLogger(
                 save_dir="logs", name=f"fold-{k}", default_hp_metric=False
@@ -116,6 +118,26 @@ class Trainer:
             if self.cfg.train.test_pickle is not None:
                 trainer.test(model, dm)
 
+            # save stats from k-fold cross-validation
+            if self.cfg.train.num_folds > 1 and "val_roc" in trainer.callback_metrics:
+                val_rocs.append(float(trainer.callback_metrics["val_roc"]))
+
+        if val_rocs:
+            import math
+            import numpy as np
+            mean = float(np.mean(val_rocs))
+            std  = float(np.std(val_rocs, ddof=1)) if len(val_rocs) > 1 else 0.0
+            n = len(val_rocs)
+            se = std / math.sqrt(n) if n > 1 else 0.0
+            ci95 = 1.96 * se # 95% CI using normal approximation
+
+            logging.info("Using micro-averaged ROC AUC")
+            scores_str = ", ".join(f"{v:.4f}" for v in val_rocs)
+            logging.info(f"folds: {scores_str}")
+            logging.info(f"mean: {mean:.4f}")
+            logging.info(f"standard deviation: {std:.4f}")
+            logging.info(f"95% confidence interval: {mean-ci95:.4f} to {mean+ci95:.4f}")
+
     def find_lr(self, num_batches: int = 100):
         """
         Suggest a learning rate and produce a plot.

{britekit-0.0.7 → britekit-0.0.8}/src/britekit/core/tuner.py

@@ -288,7 +288,7 @@ class Tuner:
         self.cfg.misc.ckpt_folder = str(
             Path(self.train_log_dir) / train_dir / "checkpoints"
         )
-        print(f"{self.cfg.misc.ckpt_folder=}")
+        logging.info(f"Using checkpoints in {self.cfg.misc.ckpt_folder}")
         self.cfg.infer.min_score = 0
 
         # suppress console output during inference and test analysis