britekit 0.0.6__py3-none-any.whl → 0.0.8__py3-none-any.whl

britekit/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2025-present Jan Huus <jhuus1@gmail.com>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.0.6"
+__version__ = "0.0.8"

britekit/commands/_reports.py

@@ -276,14 +276,14 @@ def rpt_epochs(
             tester.initialize()
 
             pr_stats = tester.get_pr_auc_stats()
-            pr_score = pr_stats["macro_pr_auc"]
+            pr_score = pr_stats["micro_pr_auc"]
             pr_scores.append(pr_score)
             if pr_score > max_pr_score:
                 max_pr_score = pr_score
                 max_pr_epoch = epoch_num
 
             roc_stats = tester.get_roc_auc_stats()
-            roc_score = roc_stats["macro_roc_auc"]
+            roc_score = roc_stats["micro_roc_auc"]
             roc_scores.append(roc_score)
             if roc_score > max_roc_score:
                 max_roc_score = roc_score
@@ -323,8 +323,8 @@ def rpt_epochs(
     plot_path = str(Path(output_path) / "training_scores.jpeg")
     plt.savefig(plot_path, dpi=300, bbox_inches="tight")
 
-    logging.info(f"Maximum PR-AUC score = {max_pr_score:.3f} at epoch {max_pr_epoch}")
-    logging.info(f"Maximum ROC-AUC score = {max_roc_score:.3f} at epoch {max_roc_epoch}")
+    logging.info(f"Maximum micro-averaged PR-AUC score = {max_pr_score:.3f} at epoch {max_pr_epoch}")
+    logging.info(f"Maximum micro-averaged ROC-AUC score = {max_roc_score:.3f} at epoch {max_roc_epoch}")
     logging.info(f"See plot at {plot_path}")
 
 

britekit/commands/_tune.py

@@ -9,6 +9,7 @@ from typing import Optional
 
 import click
 
+from britekit.core.config_loader import get_config
 from britekit.core import util
 
 
@@ -57,6 +58,7 @@ def tune(
     from britekit.core.tuner import Tuner
 
     try:
+        cfg, _ = get_config(cfg_path)
         if extract and skip_training:
             logging.error(
                 "Performing spectrogram extract is incompatible with skipping training."

britekit/core/data_module.py

@@ -52,6 +52,7 @@ class DataModule(LightningDataModule):
             )
         except Exception as e:
             logging.error(f"Failed to load training data: {e}")
+            raise
 
         if (
             self.cfg.train.noise_class_name

britekit/core/trainer.py

@@ -1,4 +1,5 @@
 # Defer some imports to improve initialization performance.
+import logging
 from pathlib import Path
 
 from britekit.core.config_loader import get_config
@@ -43,6 +44,7 @@ class Trainer:
         # load all the data once for performance, then split as needed in each fold
         dm = DataModule()
 
+        val_rocs = []
         for k in range(self.cfg.train.num_folds):
             logger = TensorBoardLogger(
                 save_dir="logs", name=f"fold-{k}", default_hp_metric=False
@@ -116,6 +118,26 @@ class Trainer:
             if self.cfg.train.test_pickle is not None:
                 trainer.test(model, dm)
 
+            # save stats from k-fold cross-validation
+            if self.cfg.train.num_folds > 1 and "val_roc" in trainer.callback_metrics:
+                val_rocs.append(float(trainer.callback_metrics["val_roc"]))
+
+        if val_rocs:
+            import math
+            import numpy as np
+            mean = float(np.mean(val_rocs))
+            std  = float(np.std(val_rocs, ddof=1)) if len(val_rocs) > 1 else 0.0
+            n = len(val_rocs)
+            se = std / math.sqrt(n) if n > 1 else 0.0
+            ci95 = 1.96 * se # 95% CI using normal approximation
+
+            logging.info("Using micro-averaged ROC AUC")
+            scores_str = ", ".join(f"{v:.4f}" for v in val_rocs)
+            logging.info(f"folds: {scores_str}")
+            logging.info(f"mean: {mean:.4f}")
+            logging.info(f"standard deviation: {std:.4f}")
+            logging.info(f"95% confidence interval: {mean-ci95:.4f} to {mean+ci95:.4f}")
+
     def find_lr(self, num_batches: int = 100):
         """
         Suggest a learning rate and produce a plot.

britekit/core/tuner.py

@@ -1,7 +1,6 @@
 # Defer some imports to improve initialization performance.
 import copy
 import logging
-import os
 from pathlib import Path
 import random
 import re
@@ -263,6 +262,21 @@ class Tuner:
             self.trial_num += 1
             self.trial_metrics[self.trial_num] = {}
 
+    @staticmethod
+    def _find_latest_version_dir(root):
+        root = Path(root)
+        version_dirs = []
+        for d in root.iterdir():
+            if d.is_dir() and d.name.startswith("version"):
+                m = re.search(r"\d+", d.name)
+                if m:
+                    version_dirs.append((int(m.group()), d))
+
+        assert version_dirs, "Failed to find training log directory"
+
+        # Sort numerically by the extracted version number
+        return max(version_dirs, key=lambda x: x[0])[1].name
+
     def _run_test(self):
         """
         Run inference with the generated checkpoints and return the selected metric.
@@ -270,10 +284,11 @@ class Tuner:
         from britekit.core.analyzer import Analyzer
         from britekit.testing.per_segment_tester import PerSegmentTester
 
-        train_dir = sorted(os.listdir(self.train_log_dir), key=natural_key)[-1]
+        train_dir = self._find_latest_version_dir(self.train_log_dir)
         self.cfg.misc.ckpt_folder = str(
             Path(self.train_log_dir) / train_dir / "checkpoints"
         )
+        logging.info(f"Using checkpoints in {self.cfg.misc.ckpt_folder}")
         self.cfg.infer.min_score = 0
 
         # suppress console output during inference and test analysis

britekit/models/base_model.py

@@ -11,7 +11,6 @@ import torch
 from torch import nn
 import torch.nn.functional as F
 from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR, SequentialLR
-from torchmetrics.functional import accuracy
 
 from britekit.core.config_loader import get_config
 from britekit.core import util
@@ -195,7 +194,9 @@ class BaseModel(pl.LightningModule):
         if self.multi_label:
             preds = torch.sigmoid(seg_logits)
             roc_auc = metrics.roc_auc_score(y.cpu(), preds.cpu(), average="micro")
-            self.log("test_roc_auc", roc_auc, on_step=False, on_epoch=True, prog_bar=True)
+            self.log(
+                "test_roc_auc", roc_auc, on_step=False, on_epoch=True, prog_bar=True
+            )
 
         return loss
 

{britekit-0.0.6.dist-info → britekit-0.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: britekit
-Version: 0.0.6
+Version: 0.0.8
 Summary: Core functions for bioacoustic recognizers.
 Project-URL: Documentation, https://github.com/jhuus/BriteKit#readme
 Project-URL: Issues, https://github.com/jhuus/BriteKit/issues

{britekit-0.0.6.dist-info → britekit-0.0.8.dist-info}/RECORD

@@ -4,17 +4,17 @@ britekit/core/audio.py,sha256=8QLbNDAiQyViEhrVC8jU0n32we4C22W_jPfc_KcOlmQ,15853
 britekit/core/augmentation.py,sha256=5_wyB-6gt7uM68Zl-rO_fPu1D6tlsd2m5oWhA6l0W9Q,5721
 britekit/core/base_config.py,sha256=wbCJI9cEH9mktUTSfLSCnU5AhJT6xhxhwZS4QWRYTpM,8744
 britekit/core/config_loader.py,sha256=epHNlH7yi_sCJX01FwgOsM6vPFk25rgUwkkcoGvVsYg,1341
-britekit/core/data_module.py,sha256=x3c_sIq9Xh_ThW_v-NBU7BCk49xJKDOMU_tpmskJE-E,9090
+britekit/core/data_module.py,sha256=f6BL-ngqkklX06Q7xZ9PMKEkvfeCzQTLYdmXRg3RmCo,9108
 britekit/core/dataset.py,sha256=Xu-lTz3TsHMuW10lHg4NN_r1baS9OQEhAD7EVz1a3A4,5804
 britekit/core/exceptions.py,sha256=ti_ve7ZdhDmzgTuspXXqyw__SUt5NoAXGEwoe3agPU8,443
 britekit/core/pickler.py,sha256=Vj-_DdFQUQj2bIVoyWe5puI8g8dTP9x7ZavbvM1iQZo,5788
 britekit/core/plot.py,sha256=hLuLB1VdtdFyaSHVDGl5tjjFCRgOJJ1ucTVJHM_3D_0,5332
 britekit/core/predictor.py,sha256=u4H8horTTvcg4Oqfpy5PG44eiiMeR5RU3aPZnMiXRCw,22914
 britekit/core/reextractor.py,sha256=gazhIZN8V1K4T_Q_kc-ihxUYbkNnc_hoAS6bpYQc95I,8396
-britekit/core/trainer.py,sha256=uKSXsrehb7-IPSCaw0oPpGBoDOcjDtaQovTt022kvzw,5467
-britekit/core/tuner.py,sha256=FG81jbcT4H3l9ugtQnLOeB0pY621nZ3iG54LLlyxeBI,15857
+britekit/core/trainer.py,sha256=N5EsbCzxw3wXxs2PTJJ0OfYFkIi49HCRM0ylT5zSSZk,6439
+britekit/core/tuner.py,sha256=FMmy4p3_j2Tojs4ONPzuUeRpCPWGlttr4rUJac7Hkyk,16435
 britekit/core/util.py,sha256=0JsEEN09hFPQzuttCKaejWofXAjCGSvWEewjkiLAh3E,19172
-britekit/models/base_model.py,sha256=nuJe01HI8V5PUULr8PGzvFDNDOAV6aw2b8uBQSor57o,16898
+britekit/models/base_model.py,sha256=9T7TwHx3K8fl10Vb-qUuypK3NDDZM-ktB8ZLHzqQhdc,16883
 britekit/models/dla.py,sha256=ALMY997AbERN7-sHqQuE5e43llRjpUDPZSFGL-Flv4M,3137
 britekit/models/effnet.py,sha256=e7WdZMsLPXe8jcWChk6n97c8DMV0YyGV6lDP_Jv6Wz4,3129
 britekit/models/gernet.py,sha256=7MEUZaDTfr-6oa8eE8dyDQb2LgahGBOEp1pTZSu1KOE,7022
@@ -32,7 +32,7 @@ britekit/testing/per_segment_tester.py,sha256=FnaozQ8VmH99aYc1ibmDFfOk_ADgsXQGU_
 britekit/training_db/extractor.py,sha256=pT7lAUsNzYs3RXDzpMv7q0MKg6TktiFLKrRtKTWv6ho,8409
 britekit/training_db/training_data_provider.py,sha256=V5aBjsCvrWViZ0Jv05hgcKRizcAXmqoj4q3hAHedoD8,5651
 britekit/training_db/training_db.py,sha256=OOfD1pcbq5HVJbzhmuI-D-gkPHWSoz0cCO4zIUGFvoY,65011
-britekit/__about__.py,sha256=YpAqd8Dr5d0riiqmNf8S5h4r5b9EreukhdhH71k7J00,122
+britekit/__about__.py,sha256=-uGInVbPaVLti1Rr4PYUteRetwYfxeLtIuqiLmEcRjA,122
 britekit/__init__.py,sha256=RpruzdjbvTcFNf21zJYY8HrAhJei91FtNNLjIBmw-kw,1857
 britekit/install/data/classes.csv,sha256=OdTZ8oQdx7N-HKyhftxZStGZYsjhCy4UbanwtQJ2wBM,54
 britekit/install/data/ignore.txt,sha256=RbKvEHtUCbgRYolwR1IucClwyD3q7l2s6QuRjph-Us4,68
@@ -79,16 +79,16 @@ britekit/commands/_init.py,sha256=FmaQRY-7SYSHCLXL__47LEPecWir7X6zEB05KpradFw,28
 britekit/commands/_pickle.py,sha256=p990FsJGfSXcgjtBzH7nPGPh023b8cH0D7RZywQQ5Aw,3488
 britekit/commands/_plot.py,sha256=7vZXsYP9dv4PbHb8K3YbJFZc65YoPIBjEMBolyh6Has,13084
 britekit/commands/_reextract.py,sha256=kCmSjeghg6mhrJ46ibRTmBkGVytU7flFvTbqsnYhBvY,3770
-britekit/commands/_reports.py,sha256=v1eXJzMoSE-Qlwwyj_d0df9NWpG2yWLMzvvxu5axz7k,21993
+britekit/commands/_reports.py,sha256=KVYtpeFQpUC4jAIm2k2xV7aiNq826DL6sUrYEJD38X0,22023
 britekit/commands/_search.py,sha256=HIUXwfPvh3rxpgaFSR3bAAI38OtGVPyMo5GMfLtLX-8,9991
 britekit/commands/_train.py,sha256=vGFKlfcv35cOelArQNbVbTRbDWogT_IMg0wZt5virHY,4158
-britekit/commands/_tune.py,sha256=lENPOuPNdY50PwqbFd3n10gzD5ujY1vuM7PByoMzLd0,7242
+britekit/commands/_tune.py,sha256=8dEZZURE769C0JZwhNpzB6pQxVklzl2w2cyXyWyhWXs,7331
 britekit/commands/_wav2mp3.py,sha256=2Q4cjT6OhJmBPTNzGRMrDd6dSdBBufuQdjhH1V8ghLo,2167
 britekit/commands/_xeno.py,sha256=_6YxQ7xFdaSy5DNUaigkbYp3E8EhtOhTC9b6OFS0MFA,6026
 britekit/commands/_youtube.py,sha256=_u1LrwY_2GxllKd505N_2ArFMbACQ_PtVxuqUCYxFe0,2214
 britekit/core/__init__.py,sha256=QcjcFyvO5KqJLF_HBeqiCk925uU5jTUjIV5lJix9XY4,556
-britekit-0.0.6.dist-info/METADATA,sha256=FAvpkc7b7s-EOI6fk7NPavvyLQelKRsAOxImtBUYI8A,18555
-britekit-0.0.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-britekit-0.0.6.dist-info/entry_points.txt,sha256=ycnPy5DLX14RTf7lKfkQAVyIf1B1zTL1gMsHm455wmg,46
-britekit-0.0.6.dist-info/licenses/LICENSE.txt,sha256=kPoHm6iop8-CUa_720Tt8gqyvLD6D_7218u1hCCpErk,1092
-britekit-0.0.6.dist-info/RECORD,,
+britekit-0.0.8.dist-info/METADATA,sha256=Qtzlff9X_WI1Cz8zpTyntAwFemS8hNbS0ClWJV9KVXk,18555
+britekit-0.0.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+britekit-0.0.8.dist-info/entry_points.txt,sha256=ycnPy5DLX14RTf7lKfkQAVyIf1B1zTL1gMsHm455wmg,46
+britekit-0.0.8.dist-info/licenses/LICENSE.txt,sha256=kPoHm6iop8-CUa_720Tt8gqyvLD6D_7218u1hCCpErk,1092
+britekit-0.0.8.dist-info/RECORD,,