britekit 0.1.1__py3-none-any.whl → 0.1.2__py3-none-any.whl

britekit/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2025-present Jan Huus <jhuus1@gmail.com>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.1.1"
+__version__ = "0.1.2"

britekit/commands/_ensemble.py

@@ -11,34 +11,24 @@ import click
 from britekit.core.config_loader import get_config
 from britekit.core import util
 
-def _eval_ensemble(ensemble, temp_dir, annotations_path, recording_dir):
-    import shutil
-
-    from britekit.core.analyzer import Analyzer
+def _eval_ensemble(ensemble, dataframe_dict, annotations_path, recordings_path, inference_output_dir):
+    import pandas as pd
     from britekit.testing.per_segment_tester import PerSegmentTester
 
-    # delete any checkpoints in the temp dir
-    for filename in os.listdir(temp_dir):
-        file_path = os.path.join(temp_dir, filename)
-        os.remove(file_path)
-
-    # copy checkpoints to the temp dir
-    for file_path in ensemble:
-        file_name = Path(file_path).name
-        dest_path = os.path.join(temp_dir, file_name)
-        shutil.copyfile(file_path, dest_path)
+    # create a dataframe with the average scores for the ensemble
+    avg_df: pd.DataFrame = dataframe_dict[ensemble[0]].copy()
+    avg_df["score"] = sum(dataframe_dict[ckpt_path]["score"] for ckpt_path in ensemble) / len(ensemble)
 
-    # run inference on the given test
-    util.set_logging(level=logging.ERROR) # suppress logging during inference and analysis
-    label_dir = "ensemble_evaluation_labels"
-    inference_output_dir = str(Path(recording_dir) / label_dir)
-    Analyzer().run(recording_dir, inference_output_dir)
+    # save the dataframe to the usual inference output location
+    scores_csv_path = str(Path(inference_output_dir) / 'scores.csv')
+    avg_df.to_csv(scores_csv_path, index=False)
 
-    min_score = 0.8 # irrelevant really
     with tempfile.TemporaryDirectory() as output_dir:
+        util.set_logging(level=logging.ERROR) # suppress logging during test reporting
+        min_score = 0.8 # arbitrary threshold
         tester = PerSegmentTester(
             annotations_path,
-            recording_dir,
+            recordings_path,
             inference_output_dir,
             output_dir,
             min_score,
@@ -47,6 +37,7 @@ def _eval_ensemble(ensemble, temp_dir, annotations_path, recording_dir):
 
         pr_stats = tester.get_pr_auc_stats()
         roc_stats = tester.get_roc_auc_stats()
+        util.set_logging() # restore logging
 
         scores = {
             "macro_pr": pr_stats["macro_pr_auc"],
@@ -55,11 +46,9 @@ def _eval_ensemble(ensemble, temp_dir, annotations_path, recording_dir):
             "micro_roc": roc_stats["micro_roc_auc_trained"]
         }
 
-        shutil.rmtree(inference_output_dir)
-        util.set_logging() # restore logging
-
     return scores
 
+
 def ensemble(
     cfg_path: Optional[str]=None,
     ckpt_path: str="",
@@ -87,7 +76,13 @@ def ensemble(
     import glob
     import itertools
     import math
+    import os
     import random
+    import shutil
+
+    import pandas as pd
+
+    from britekit.core.analyzer import Analyzer
 
     if metric not in ["macro_pr", "micro_pr", "macro_roc", "micro_roc"]:
         logging.error(f"Error: invalid metric ({metric})")
@@ -106,10 +101,29 @@ def ensemble(
     if not recordings_path:
         recordings_path = str(Path(annotations_path).parent)
 
-    with tempfile.TemporaryDirectory() as temp_dir:
-        cfg.misc.ckpt_folder = temp_dir
+    with tempfile.TemporaryDirectory() as ensemble_dir:
+        cfg.misc.ckpt_folder = ensemble_dir
         cfg.infer.min_score = 0
 
+        # get a dataframe of predictions per checkpoint
+        label_dir = "ensemble_evaluation_labels"
+        inference_output_dir = str(Path(recordings_path) / label_dir)
+        scores_csv_path = str(Path(inference_output_dir) / 'scores.csv')
+        dataframe_dict = {}
+        for ckpt_path in ckpt_paths:
+            ckpt_name = Path(ckpt_path).name
+            logging.info(f"Running inference with {ckpt_name}")
+            dest_path = str(Path(ensemble_dir) / ckpt_name)
+            shutil.copyfile(ckpt_path, dest_path)
+
+            util.set_logging(level=logging.ERROR) # suppress logging during inference
+            Analyzer().run(recordings_path, inference_output_dir, rtype='csv')
+            util.set_logging()
+
+            df = pd.read_csv(scores_csv_path)
+            dataframe_dict[ckpt_path] = df
+            os.remove(dest_path)
+
         best_score = 0
         best_ensemble = None
         count = 1
@@ -118,7 +132,7 @@ def ensemble(
             # Exhaustive search
             logging.info("Doing exhaustive search")
             for ensemble in itertools.combinations(ckpt_paths, ensemble_size):
-                scores = _eval_ensemble(ensemble, temp_dir, annotations_path, recordings_path)
+                scores = _eval_ensemble(ensemble, dataframe_dict, annotations_path, recordings_path, inference_output_dir)
                 logging.info(f"For ensemble {count} of {total_combinations}, score = {scores[metric]:.4f}")
                 if scores[metric] > best_score:
                     best_score = scores[metric]
@@ -133,7 +147,7 @@ def ensemble(
                 ensemble = tuple(sorted(random.sample(ckpt_paths, ensemble_size)))
                 if ensemble not in seen:
                     seen.add(ensemble)
-                    scores = _eval_ensemble(ensemble, temp_dir, annotations_path, recordings_path)
+                    scores = _eval_ensemble(ensemble, dataframe_dict, annotations_path, recordings_path, inference_output_dir)
                     logging.info(f"For ensemble {count} of {num_tries}, score = {scores[metric]:.4f}")
                     if scores[metric] > best_score:
                         best_score = scores[metric]
@@ -141,6 +155,8 @@ def ensemble(
 
                 count += 1
 
+        shutil.rmtree(inference_output_dir)
+
     logging.info(f"Best score = {best_score:.4f}")
 
     assert best_ensemble is not None

{britekit-0.1.1.dist-info → britekit-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: britekit
-Version: 0.1.1
+Version: 0.1.2
 Summary: Core functions for bioacoustic recognizers.
 Project-URL: Documentation, https://github.com/jhuus/BriteKit#readme
 Project-URL: Issues, https://github.com/jhuus/BriteKit/issues

{britekit-0.1.1.dist-info → britekit-0.1.2.dist-info}/RECORD

@@ -32,7 +32,7 @@ britekit/testing/per_segment_tester.py,sha256=5zFpe8wVEEtVjthKKFRb0dwE9T0v4JIll9
 britekit/training_db/extractor.py,sha256=yblYTco9b-bZhBTOkGkNANOabsNo8IfQGJxPvh_eJyo,8406
 britekit/training_db/training_data_provider.py,sha256=V5aBjsCvrWViZ0Jv05hgcKRizcAXmqoj4q3hAHedoD8,5651
 britekit/training_db/training_db.py,sha256=xZqN1sMC2yFMEDm9rOrCigN3CUfUzTaTUkX3IZ_zHRc,64922
-britekit/__about__.py,sha256=iTA-8NubPgFzs2Mkc2etUzBbWweMfnEmfcp3_GiYcqw,122
+britekit/__about__.py,sha256=p7Eggv-ttiNa77u9u08fyOSsZF30h6Wirtx06CqlBII,122
 britekit/__init__.py,sha256=RpruzdjbvTcFNf21zJYY8HrAhJei91FtNNLjIBmw-kw,1857
 britekit/install/data/classes.csv,sha256=OdTZ8oQdx7N-HKyhftxZStGZYsjhCy4UbanwtQJ2wBM,54
 britekit/install/data/ignore.txt,sha256=RbKvEHtUCbgRYolwR1IucClwyD3q7l2s6QuRjph-Us4,68
@@ -72,7 +72,7 @@ britekit/commands/_ckpt_ops.py,sha256=2l-eJuxGBTHtQZ2Nked82KeSbP7WIwZ-yAYuM4v4Hl
 britekit/commands/_db_add.py,sha256=brUCb7LZVJ7XezlSmpaKilz9hYoII_DvfjhS1v64cr0,7249
 britekit/commands/_db_delete.py,sha256=ziqxnQhBOjHgqlu0uk6GA8A7I9FOMYcPEscmPxThAVY,14520
 britekit/commands/_embed.py,sha256=gTQK4YOilwsZCY2r8HhaWUZBpMkA-OEZsR5RkNV1euM,4388
-britekit/commands/_ensemble.py,sha256=O6_BDgHWXdlCn_JdbZdI8X5NdGlnjJiz6nuJfVYkE0g,7583
+britekit/commands/_ensemble.py,sha256=jssoAKvKgy2KNAQ2-tIapQgo_YfywwxSnPRnIwnmC38,8427
 britekit/commands/_extract.py,sha256=iz9VG1KnV-d7cFliQpmWiSi6Ezt9hCm5Iur9r-XBb20,8859
 britekit/commands/_find_dup.py,sha256=Zig-s04BUQzeY4s7DjEQM53-e6KgCXxSw05R6BfSkNw,6350
 britekit/commands/_inat.py,sha256=ojTre5BCj_jmEh6x2kzNhcminLN6h5bzsYpxyrxGRdQ,4164
@@ -88,8 +88,8 @@ britekit/commands/_wav2mp3.py,sha256=2Q4cjT6OhJmBPTNzGRMrDd6dSdBBufuQdjhH1V8ghLo
 britekit/commands/_xeno.py,sha256=4qS2iXeGMl0jYvTAvhSvX0VvIlp7STlX27o273WwBHk,6165
 britekit/commands/_youtube.py,sha256=_u1LrwY_2GxllKd505N_2ArFMbACQ_PtVxuqUCYxFe0,2214
 britekit/core/__init__.py,sha256=QcjcFyvO5KqJLF_HBeqiCk925uU5jTUjIV5lJix9XY4,556
-britekit-0.1.1.dist-info/METADATA,sha256=Ni0op2mIqZvbDJQw8DiekHMMkMTSU1fkUPjSgadbLWE,18552
-britekit-0.1.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-britekit-0.1.1.dist-info/entry_points.txt,sha256=ycnPy5DLX14RTf7lKfkQAVyIf1B1zTL1gMsHm455wmg,46
-britekit-0.1.1.dist-info/licenses/LICENSE.txt,sha256=kPoHm6iop8-CUa_720Tt8gqyvLD6D_7218u1hCCpErk,1092
-britekit-0.1.1.dist-info/RECORD,,
+britekit-0.1.2.dist-info/METADATA,sha256=MLgsBLSqBDBxK3yjFtsB5I50SbD2HSVsmrLmYNnVgSk,18552
+britekit-0.1.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+britekit-0.1.2.dist-info/entry_points.txt,sha256=ycnPy5DLX14RTf7lKfkQAVyIf1B1zTL1gMsHm455wmg,46
+britekit-0.1.2.dist-info/licenses/LICENSE.txt,sha256=kPoHm6iop8-CUa_720Tt8gqyvLD6D_7218u1hCCpErk,1092
+britekit-0.1.2.dist-info/RECORD,,