megadetector 5.0.14__py3-none-any.whl → 5.0.16__py3-none-any.whl

megadetector/postprocessing/postprocess_batch_results.py

@@ -949,6 +949,13 @@ def process_batch_results(options):
               f'negative, {n_positive} positive, {n_unknown} unknown, '
               f'{n_ambiguous} ambiguous')
 
+        if n_positive == 0:
+            print('\n*** Warning: no positives found in ground truth, analysis won\'t be very meaningful ***\n')
+        if n_negative == 0:
+            print('\n*** Warning: no negatives found in ground truth, analysis won\'t be very meaningful ***\n')
+        if n_ambiguous > 0:
+            print('\n*** Warning: {} images with ambiguous positive/negative status found in ground truth ***\n'.format(
+                n_ambiguous))
 
     ##%% Load detection (and possibly classification) results
 
@@ -1095,25 +1102,34 @@ def process_batch_results(options):
 
         ##%% Detection evaluation: compute precision/recall
 
-        # numpy array of detection probabilities
+        # numpy array of maximum confidence values
         p_detection = detections_df['max_detection_conf'].values
-        n_detections = len(p_detection)
+        n_detection_values = len(p_detection)
 
         # numpy array of bools (0.0/1.0), and -1 as null value
-        gt_detections = np.zeros(n_detections, dtype=float)
+        gt_detections = np.zeros(n_detection_values, dtype=float)
 
+        n_positive = 0
+        n_negative = 0
+        
         for i_detection, fn in enumerate(detector_files):
+            
             image_id = ground_truth_indexed_db.filename_to_id[fn]
             image = ground_truth_indexed_db.image_id_to_image[image_id]
             detection_status = image['_detection_status']
 
             if detection_status == DetectionStatus.DS_NEGATIVE:
                 gt_detections[i_detection] = 0.0
+                n_negative += 1
             elif detection_status == DetectionStatus.DS_POSITIVE:
                 gt_detections[i_detection] = 1.0
+                n_positive += 1
             else:
                 gt_detections[i_detection] = -1.0
 
+        print('Of {} ground truth values, found {} positives and {} negatives'.format(
+            len(detections_df),n_positive,n_negative))
+        
         # Don't include ambiguous/unknown ground truth in precision/recall analysis
         b_valid_ground_truth = gt_detections >= 0.0
 

megadetector/postprocessing/subset_json_detector_output.py

@@ -124,7 +124,7 @@ class SubsetJsonDetectorOutputOptions:
         self.remove_failed_images = False
         
         #: Either a list of category IDs (as string-ints) (not names), or a dictionary mapping category *IDs* 
-        #: (as string-ints) (not names) to thresholds.  Removes  non-matching detections, does not 
+        #: (as string-ints) (not names) to thresholds.  Removes non-matching detections, does not 
         #: remove images.  Not technically mutually exclusize with category_names_to_keep, but it's an esoteric 
         #: scenario indeed where you would want to specify both.
         self.categories_to_keep = None
@@ -517,7 +517,7 @@ def subset_json_detector_output(input_filename, output_filename, options, data=N
     else:
         
         # Map images to unique folders
-        print('Finding unique folders')    
+        print('Finding unique folders')
         
         folders_to_images = {}
         
@@ -670,16 +670,26 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('input_file', type=str, help='Input .json filename')
     parser.add_argument('output_file', type=str, help='Output .json filename')
-    parser.add_argument('--query', type=str, default=None, help='Query string to search for (omitting this matches all)')
-    parser.add_argument('--replacement', type=str, default=None, help='Replace [query] with this')
-    parser.add_argument('--confidence_threshold', type=float, default=None, help='Remove detections below this confidence level')
-    parser.add_argument('--split_folders', action='store_true', help='Split .json files by leaf-node folder')
-    parser.add_argument('--split_folder_param', type=int, help='Directory level count for n_from_bottom and n_from_top splitting')
-    parser.add_argument('--split_folder_mode', type=str, help='Folder level to use for splitting ("top" or "bottom")')
-    parser.add_argument('--make_folder_relative', action='store_true', help='Make image paths relative to their containing folder (only meaningful with split_folders)')
-    parser.add_argument('--overwrite_json_files', action='store_true', help='Overwrite output files')
-    parser.add_argument('--copy_jsons_to_folders', action='store_true', help='When using split_folders and make_folder_relative, copy jsons to their corresponding folders (relative to output_file)')
-    parser.add_argument('--create_folders', action='store_true', help='When using copy_jsons_to_folders, create folders that don''t exist')    
+    parser.add_argument('--query', type=str, default=None, 
+                        help='Query string to search for (omitting this matches all)')
+    parser.add_argument('--replacement', type=str, default=None, 
+                        help='Replace [query] with this')
+    parser.add_argument('--confidence_threshold', type=float, default=None, 
+                        help='Remove detections below this confidence level')
+    parser.add_argument('--split_folders', action='store_true', 
+                        help='Split .json files by leaf-node folder')
+    parser.add_argument('--split_folder_param', type=int,
+                        help='Directory level count for n_from_bottom and n_from_top splitting')
+    parser.add_argument('--split_folder_mode', type=str,
+                        help='Folder level to use for splitting ("top" or "bottom")')
+    parser.add_argument('--make_folder_relative', action='store_true', 
+                        help='Make image paths relative to their containing folder (only meaningful with split_folders)')
+    parser.add_argument('--overwrite_json_files', action='store_true', 
+                        help='Overwrite output files')
+    parser.add_argument('--copy_jsons_to_folders', action='store_true', 
+                        help='When using split_folders and make_folder_relative, copy jsons to their corresponding folders (relative to output_file)')
+    parser.add_argument('--create_folders', action='store_true',
+                        help='When using copy_jsons_to_folders, create folders that don''t exist')    
     
     if len(sys.argv[1:]) == 0:
         parser.print_help()

megadetector/taxonomy_mapping/map_new_lila_datasets.py

@@ -15,10 +15,10 @@ import json
 # Created by get_lila_category_list.py
 input_lila_category_list_file = os.path.expanduser('~/lila/lila_categories_list/lila_dataset_to_categories.json')
 
-output_file = os.path.expanduser('~/lila/lila_additions_2023.12.29.csv')
+output_file = os.path.expanduser('~/lila/lila_additions_2024.07.16.csv')
 
 datasets_to_map = [
-    'Trail Camera Images of New Zealand Animals'
+    'Desert Lion Conservation Camera Traps'
     ]
 
 
@@ -133,7 +133,7 @@ if False:
     # q = 'white-throated monkey'
     # q = 'cingulata'
     # q = 'notamacropus'
-    q = 'porzana'
+    q = 'aves'
     taxonomy_preference = 'inat'
     m = get_preferred_taxonomic_match(q,taxonomy_preference)
     # print(m.scientific_name); import clipboard; clipboard.copy(m.scientific_name)

megadetector/taxonomy_mapping/prepare_lila_taxonomy_release.py

@@ -24,7 +24,7 @@ if False:
     release_taxonomy_file = os.path.expanduser('~/lila/lila-taxonomy-mapping_release.csv')
     # import clipboard; clipboard.copy(release_taxonomy_file)
 
-    # Created by get_lila_category_list.py... contains counts for each category
+    # Created by get_lila_annotation_counts.py... contains counts for each category
     lila_dataset_to_categories_file = os.path.expanduser('~/lila/lila_categories_list/lila_dataset_to_categories.json')
 
     assert os.path.isfile(lila_dataset_to_categories_file)
@@ -140,3 +140,4 @@ if False:
 
     print('Wrote final output to {}'.format(release_taxonomy_file))
 
+# ...if False

megadetector/taxonomy_mapping/preview_lila_taxonomy.py

@@ -16,7 +16,7 @@ import os
 import pandas as pd
 
 # lila_taxonomy_file = r"c:\git\agentmorrisprivate\lila-taxonomy\lila-taxonomy-mapping.csv"
-lila_taxonomy_file = os.path.expanduser('~/lila/lila_additions_2023.12.29.csv')
+lila_taxonomy_file = os.path.expanduser('~/lila/lila_additions_2024.07.16.csv')
 
 preview_base = os.path.expanduser('~/lila/lila_taxonomy_preview')
 os.makedirs(preview_base,exist_ok=True)

megadetector/taxonomy_mapping/simple_image_download.py

@@ -8,6 +8,11 @@ Slightly modified from:
 
 https://github.com/RiddlerQ/simple_image_download
 
+pip install python-magic
+
+# On Windows, also run:
+pip install python-magic-bin
+
 """
 
 #%% Imports

megadetector/taxonomy_mapping/species_lookup.py

@@ -208,7 +208,7 @@ def initialize_taxonomy_lookup(force_init=False) -> None:
     # Load GBIF taxonomy
     gbif_taxonomy_file = os.path.join(taxonomy_download_dir, 'GBIF', 'Taxon.tsv')
     print('Loading GBIF taxonomy from {}'.format(gbif_taxonomy_file))
-    gbif_taxonomy = pd.read_csv(gbif_taxonomy_file, sep='\t')
+    gbif_taxonomy = pd.read_csv(gbif_taxonomy_file, sep='\t', encoding='utf-8',on_bad_lines='warn')
     gbif_taxonomy['scientificName'] = gbif_taxonomy['scientificName'].fillna('').str.strip()
     gbif_taxonomy['canonicalName'] = gbif_taxonomy['canonicalName'].fillna('').str.strip()