britekit 0.0.7__tar.gz → 0.0.9__tar.gz

{britekit-0.0.7 → britekit-0.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: britekit
-Version: 0.0.7
+Version: 0.0.9
 Summary: Core functions for bioacoustic recognizers.
 Project-URL: Documentation, https://github.com/jhuus/BriteKit#readme
 Project-URL: Issues, https://github.com/jhuus/BriteKit/issues

{britekit-0.0.7 → britekit-0.0.9}/britekit/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2025-present Jan Huus <jhuus1@gmail.com>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.0.7"
+__version__ = "0.0.9"

{britekit-0.0.7 → britekit-0.0.9}/britekit/commands/__init__.py

@@ -13,6 +13,7 @@ from ._db_delete import (
     del_stype,
 )
 from ._embed import embed
+from ._ensemble import ensemble
 from ._extract import extract_all, extract_by_image
 from ._find_dup import find_dup
 from ._inat import inat
@@ -54,6 +55,7 @@ __all__ = [
     "del_src",
     "del_stype",
     "embed",
+    "ensemble",
     "extract_all",
     "extract_by_image",
     "find_dup",

britekit-0.0.9/britekit/commands/_ensemble.py

@@ -0,0 +1,237 @@
+# File name starts with _ to keep it out of typeahead for API users.
+# Defer some imports to improve --help performance.
+import logging
+import os
+from pathlib import Path
+import tempfile
+from typing import Optional
+
+import click
+
+from britekit.core.config_loader import get_config
+from britekit.core import util
+
+def _eval_ensemble(ensemble, temp_dir, annotations_path, recording_dir):
+    import shutil
+
+    from britekit.core.analyzer import Analyzer
+    from britekit.testing.per_segment_tester import PerSegmentTester
+
+    # delete any checkpoints in the temp dir
+    for filename in os.listdir(temp_dir):
+        file_path = os.path.join(temp_dir, filename)
+        os.remove(file_path)
+
+    # copy checkpoints to the temp dir
+    for file_path in ensemble:
+        file_name = Path(file_path).name
+        dest_path = os.path.join(temp_dir, file_name)
+        shutil.copyfile(file_path, dest_path)
+
+    # run inference on the given test
+    util.set_logging(level=logging.ERROR) # suppress logging during inference and analysis
+    label_dir = "ensemble_evaluation_labels"
+    inference_output_dir = str(Path(recording_dir) / label_dir)
+    Analyzer().run(recording_dir, inference_output_dir)
+
+    min_score = 0.8 # irrelevant really
+    with tempfile.TemporaryDirectory() as output_dir:
+        tester = PerSegmentTester(
+            annotations_path,
+            recording_dir,
+            inference_output_dir,
+            output_dir,
+            min_score,
+        )
+        tester.initialize()
+
+        pr_stats = tester.get_pr_auc_stats()
+        roc_stats = tester.get_roc_auc_stats()
+
+        scores = {
+            "macro_pr": pr_stats["macro_pr_auc"],
+            "micro_pr": pr_stats["micro_pr_auc_trained"],
+            "macro_roc": roc_stats["macro_roc_auc"],
+            "micro_roc": roc_stats["micro_roc_auc_trained"]
+        }
+
+        shutil.rmtree(inference_output_dir)
+        util.set_logging() # restore logging
+
+    return scores
+
+def ensemble(
+    cfg_path: Optional[str]=None,
+    ckpt_path: str="",
+    ensemble_size: int=3,
+    num_tries: int=100,
+    metric: str = "micro_roc",
+    annotations_path: str = "",
+    recordings_path: Optional[str] = None,
+    output_path: str = "",
+) -> None:
+    """
+    Find the best ensemble of a given size from a group of checkpoints.
+
+    Given a directory containing checkpoints, and an ensemble size (default=3), select random
+    ensembles of the given size and test each one to identify the best ensemble.
+
+    Args:
+        cfg_path (str, optional): Path to YAML file defining configuration overrides.
+        ckpt_path (str): Path to directory containing checkpoints.
+        ensemble_size (int): Number of checkpoints in ensemble (default=3).
+        num_tries (int): Maximum number of ensembles to try (default=100).
+        metric (str): Metric to use to compare ensembles (default=micro_roc).
+        annotations_path (str): Path to CSV file containing ground truth annotations.
+        recordings_path (str, optional): Directory containing audio recordings. Defaults to annotations directory.
+        output_path (str): Directory where reports will be saved.
+    """
+    import glob
+    import itertools
+    import math
+    import random
+
+    if metric not in ["macro_pr", "micro_pr", "macro_roc", "micro_roc"]:
+        logging.error(f"Error: invalid metric ({metric})")
+        return
+
+    cfg, _ = get_config(cfg_path)
+    ckpt_paths = sorted(glob.glob(os.path.join(ckpt_path, "*.ckpt")))
+    num_ckpts = len(ckpt_paths)
+    if num_ckpts == 0:
+        logging.error(f"Error: no checkpoints found in {ckpt_path}")
+        return
+    elif num_ckpts < ensemble_size:
+        logging.error(f"Error: number of checkpoints ({num_ckpts}) is less than requested ensemble size ({ensemble_size})")
+        return
+
+    if not recordings_path:
+        recordings_path = str(Path(annotations_path).parent)
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        cfg.misc.ckpt_folder = temp_dir
+        cfg.infer.min_score = 0
+
+        best_score = 0
+        best_ensemble = None
+        count = 1
+        total_combinations = math.comb(len(ckpt_paths), ensemble_size)
+        if total_combinations <= num_tries:
+            # Exhaustive search
+            logging.info("Doing exhaustive search")
+            for ensemble in itertools.combinations(ckpt_paths, ensemble_size):
+                scores = _eval_ensemble(ensemble, temp_dir, annotations_path, recordings_path)
+                logging.info(f"For ensemble {count} of {total_combinations}, score = {scores[metric]:.4f}")
+                if scores[metric] > best_score:
+                    best_score = scores[metric]
+                    best_ensemble = ensemble
+
+                count += 1
+        else:
+            # Random sampling without replacement
+            logging.info("Doing random sampling")
+            seen: set = set()
+            while len(seen) < num_tries:
+                ensemble = tuple(sorted(random.sample(ckpt_paths, ensemble_size)))
+                if ensemble not in seen:
+                    seen.add(ensemble)
+                    scores = _eval_ensemble(ensemble, temp_dir, annotations_path, recordings_path)
+                    logging.info(f"For ensemble {count} of {num_tries}, score = {scores[metric]:.4f}")
+                    if scores[metric] > best_score:
+                        best_score = scores[metric]
+                        best_ensemble = ensemble
+
+                count += 1
+
+    logging.info(f"Best score = {best_score:.4f}")
+
+    best_names = [Path(ckpt_path).name for ckpt_path in best_ensemble]
+    logging.info(f"Best ensemble = {best_names}")
+
+@click.command(
+    name="ensemble",
+    short_help="Find the best ensemble of a given size from a group of checkpoints.",
+    help=util.cli_help_from_doc(ensemble.__doc__),
+)
+@click.option(
+    "-c",
+    "--cfg",
+    "cfg_path",
+    type=click.Path(exists=True),
+    required=False,
+    help="Path to YAML file defining config overrides.",
+)
+@click.option(
+    "--ckpt_path",
+    "ckpt_path",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
+    required=True,
+    help="Directory containing checkpoints."
+)
+@click.option(
+    "-e",
+    "--ensemble_size",
+    "ensemble_size",
+    type=int,
+    default=3,
+    help="Number of checkpoints in ensemble (default=3)."
+)
+@click.option(
+    "-n",
+    "--num_tries",
+    "num_tries",
+    type=int,
+    default=100,
+    help="Maximum number of ensembles to try (default=100)."
+)
+@click.option(
+    "-m",
+    "--metric",
+    "metric",
+    type=click.Choice(
+        [
+            "macro_pr",
+            "micro_pr",
+            "macro_roc",
+            "micro_roc",
+        ]
+    ),
+    default="micro_roc",
+    help="Metric used to compare ensembles (default=micro_roc). Macro-averaging uses annotated classes only, but micro-averaging uses all classes.",
+)
+@click.option(
+    "-a",
+    "--annotations",
+    "annotations_path",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False),
+    required=True,
+    help="Path to CSV file containing annotations or ground truth).",
+)
+@click.option(
+    "-r",
+    "--recordings",
+    "recordings_path",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
+    required=False,
+    help="Recordings directory. Default is directory containing annotations file.",
+)
+@click.option(
+    "-o",
+    "--output",
+    "output_path",
+    type=click.Path(file_okay=False, dir_okay=True),
+    required=True,
+    help="Path to output directory.",
+)
+def _ensemble_cmd(
+    cfg_path: Optional[str],
+    ckpt_path: str,
+    ensemble_size: int,
+    num_tries: int,
+    metric: str,
+    annotations_path: str,
+    recordings_path: Optional[str],
+    output_path: str,
+) -> None:
+    util.set_logging()
+    ensemble(cfg_path, ckpt_path, ensemble_size, num_tries, metric, annotations_path, recordings_path, output_path)

{britekit-0.0.7 → britekit-0.0.9}/britekit/commands/_reports.py

@@ -276,14 +276,14 @@ def rpt_epochs(
             tester.initialize()
 
             pr_stats = tester.get_pr_auc_stats()
-            pr_score = pr_stats["macro_pr_auc"]
+            pr_score = pr_stats["micro_pr_auc_trained"]
             pr_scores.append(pr_score)
             if pr_score > max_pr_score:
                 max_pr_score = pr_score
                 max_pr_epoch = epoch_num
 
             roc_stats = tester.get_roc_auc_stats()
-            roc_score = roc_stats["macro_roc_auc"]
+            roc_score = roc_stats["micro_roc_auc_trained"]
             roc_scores.append(roc_score)
             if roc_score > max_roc_score:
                 max_roc_score = roc_score
@@ -323,8 +323,8 @@ def rpt_epochs(
     plot_path = str(Path(output_path) / "training_scores.jpeg")
     plt.savefig(plot_path, dpi=300, bbox_inches="tight")
 
-    logging.info(f"Maximum PR-AUC score = {max_pr_score:.3f} at epoch {max_pr_epoch}")
-    logging.info(f"Maximum ROC-AUC score = {max_roc_score:.3f} at epoch {max_roc_epoch}")
+    logging.info(f"Maximum micro-averaged PR-AUC score = {max_pr_score:.3f} at epoch {max_pr_epoch}")
+    logging.info(f"Maximum micro-averaged ROC-AUC score = {max_roc_score:.3f} at epoch {max_roc_epoch}")
     logging.info(f"See plot at {plot_path}")
 
 

{britekit-0.0.7 → britekit-0.0.9}/britekit/commands/_tune.py

@@ -18,7 +18,7 @@ def tune(
     param_path: Optional[str] = None,
     output_path: str = "",
     annotations_path: str = "",
-    metric: str = "macro_roc",
+    metric: str = "micro_roc",
     recordings_path: str = "",
     train_log_path: str = "",
     num_trials: int = 0,
@@ -159,7 +159,7 @@ def tune(
             "micro_roc",
         ]
     ),
-    default="macro_roc",
+    default="micro_roc",
     help="Metric used to compare runs. Macro-averaging uses annotated classes only, but micro-averaging uses all classes.",
 )
 @click.option(

{britekit-0.0.7 → britekit-0.0.9}/pyproject.toml

@@ -86,6 +86,7 @@ packages = ["src/britekit"]
 "src/britekit/commands/_db_add.py" = "britekit/commands/_db_add.py"
 "src/britekit/commands/_db_delete.py" = "britekit/commands/_db_delete.py"
 "src/britekit/commands/_embed.py" = "britekit/commands/_embed.py"
+"src/britekit/commands/_ensemble.py" = "britekit/commands/_ensemble.py"
 "src/britekit/commands/_extract.py" = "britekit/commands/_extract.py"
 "src/britekit/commands/_find_dup.py" = "britekit/commands/_find_dup.py"
 "src/britekit/commands/_inat.py" = "britekit/commands/_inat.py"
@@ -120,6 +121,7 @@ only-include = [
 "src/britekit/commands/_db_add.py" = "britekit/commands/_db_add.py"
 "src/britekit/commands/_db_delete.py" = "britekit/commands/_db_delete.py"
 "src/britekit/commands/_embed.py" = "britekit/commands/_embed.py"
+"src/britekit/commands/_ensemble.py" = "britekit/commands/_ensemble.py"
 "src/britekit/commands/_extract.py" = "britekit/commands/_extract.py"
 "src/britekit/commands/_find_dup.py" = "britekit/commands/_find_dup.py"
 "src/britekit/commands/_inat.py" = "britekit/commands/_inat.py"

{britekit-0.0.7 → britekit-0.0.9}/src/britekit/cli.py

@@ -30,6 +30,7 @@ from .commands._db_delete import (
     _del_stype_cmd,
 )
 from .commands._embed import _embed_cmd
+from .commands._ensemble import _ensemble_cmd
 from .commands._extract import _extract_all_cmd, _extract_by_image_cmd
 from .commands._find_dup import _find_dup_cmd
 from .commands._inat import _inat_cmd
@@ -80,6 +81,7 @@ cli.add_command(_del_src_cmd)
 cli.add_command(_del_stype_cmd)
 
 cli.add_command(_embed_cmd)
+cli.add_command(_ensemble_cmd)
 cli.add_command(_extract_all_cmd)
 cli.add_command(_extract_by_image_cmd)
 

{britekit-0.0.7 → britekit-0.0.9}/src/britekit/core/data_module.py

@@ -52,6 +52,7 @@ class DataModule(LightningDataModule):
             )
         except Exception as e:
             logging.error(f"Failed to load training data: {e}")
+            raise
 
         if (
             self.cfg.train.noise_class_name

{britekit-0.0.7 → britekit-0.0.9}/src/britekit/core/trainer.py

@@ -1,4 +1,5 @@
 # Defer some imports to improve initialization performance.
+import logging
 from pathlib import Path
 
 from britekit.core.config_loader import get_config
@@ -43,6 +44,7 @@ class Trainer:
         # load all the data once for performance, then split as needed in each fold
         dm = DataModule()
 
+        val_rocs = []
         for k in range(self.cfg.train.num_folds):
             logger = TensorBoardLogger(
                 save_dir="logs", name=f"fold-{k}", default_hp_metric=False
@@ -116,6 +118,27 @@ class Trainer:
             if self.cfg.train.test_pickle is not None:
                 trainer.test(model, dm)
 
+            # save stats from k-fold cross-validation
+            if self.cfg.train.num_folds > 1 and "val_roc" in trainer.callback_metrics:
+                val_rocs.append(float(trainer.callback_metrics["val_roc"]))
+
+        if val_rocs:
+            import math
+            import numpy as np
+
+            mean = float(np.mean(val_rocs))
+            std = float(np.std(val_rocs, ddof=1)) if len(val_rocs) > 1 else 0.0
+            n = len(val_rocs)
+            se = std / math.sqrt(n) if n > 1 else 0.0
+            ci95 = 1.96 * se  # 95% CI using normal approximation
+
+            logging.info("Using micro-averaged ROC AUC")
+            scores_str = ", ".join(f"{v:.4f}" for v in val_rocs)
+            logging.info(f"folds: {scores_str}")
+            logging.info(f"mean: {mean:.4f}")
+            logging.info(f"standard deviation: {std:.4f}")
+            logging.info(f"95% confidence interval: {mean-ci95:.4f} to {mean+ci95:.4f}")
+
     def find_lr(self, num_batches: int = 100):
         """
         Suggest a learning rate and produce a plot.

{britekit-0.0.7 → britekit-0.0.9}/src/britekit/core/tuner.py

@@ -288,7 +288,7 @@ class Tuner:
         self.cfg.misc.ckpt_folder = str(
             Path(self.train_log_dir) / train_dir / "checkpoints"
         )
-        print(f"{self.cfg.misc.ckpt_folder=}")
+        logging.info(f"Using checkpoints in {self.cfg.misc.ckpt_folder}")
         self.cfg.infer.min_score = 0
 
         # suppress console output during inference and test analysis