megadetector 10.0.9__py3-none-any.whl → 10.0.11__py3-none-any.whl

megadetector/detection/video_utils.py

@@ -221,11 +221,13 @@ def frames_to_video(images, fs, output_file_name, codec_spec=default_fourcc):
         print('Warning: no frames to render')
         return
 
-    os.makedirs(os.path.dirname(output_file_name),exist_ok=True)
+    output_dir = os.path.dirname(output_file_name)
+    if len(output_dir) > 0:
+        os.makedirs(output_dir, exist_ok=True)
 
     # Determine the width and height from the first image
     frame = cv2.imread(images[0])
-    cv2.imshow('video',frame)
+    # cv2.imshow('video',frame)
     height, width, channels = frame.shape
 
     # Define the codec and create VideoWriter object
@@ -297,7 +299,7 @@ def _filename_to_frame_number(filename):
     try:
         frame_number = int(frame_number)
     except Exception:
-        raise ValueError('Filename {} does contain a valid frame number'.format(filename))
+        raise ValueError('Filename {} does not contain a valid frame number'.format(filename))
 
     return frame_number
 
@@ -1059,9 +1061,10 @@ def video_folder_to_frames(input_folder,
 
         finally:
 
-            pool.close()
-            pool.join()
-            print('Pool closed and joined for video processing')
+            if pool is not None:
+                pool.close()
+                pool.join()
+                print('Pool closed and joined for video processing')
 
         # ...try/finally
 

megadetector/postprocessing/add_max_conf.py

@@ -18,7 +18,8 @@ import json
 import sys
 import argparse
 
-from megadetector.utils import ct_utils
+from megadetector.utils.ct_utils import get_max_conf
+from megadetector.utils.ct_utils import write_json
 
 
 #%% Main function
@@ -39,15 +40,14 @@ def add_max_conf(input_file,output_file):
 
     for im in d['images']:
 
-        max_conf = ct_utils.get_max_conf(im)
+        max_conf = get_max_conf(im)
 
         if 'max_detection_conf' in im:
             assert abs(max_conf - im['max_detection_conf']) < 0.00001
         else:
             im['max_detection_conf'] = max_conf
 
-    with open(output_file,'w') as f:
-        json.dump(d,f,indent=1)
+    write_json(output_file,d)
 
 
 #%% Driver

megadetector/postprocessing/categorize_detections_by_size.py

@@ -14,6 +14,8 @@ import json
 from collections import defaultdict
 from tqdm import tqdm
 
+from megadetector.utils.ct_utils import write_json
+
 
 #%% Support classes
 
@@ -157,8 +159,7 @@ def categorize_detections_by_size(input_file,output_file=None,options=None):
         print('Found {} detections in category {}'.format(category_count,category_name))
 
     if output_file is not None:
-        with open(output_file,'w') as f:
-            json.dump(data,f,indent=1)
+        write_json(output_file,data)
 
     return data
 

megadetector/postprocessing/classification_postprocessing.py

@@ -25,6 +25,7 @@ from megadetector.utils.ct_utils import is_empty
 from megadetector.utils.ct_utils import sort_dictionary_by_value
 from megadetector.utils.ct_utils import sort_dictionary_by_key
 from megadetector.utils.ct_utils import invert_dictionary
+from megadetector.utils.ct_utils import write_json
 
 from megadetector.utils.wi_taxonomy_utils import clean_taxonomy_string
 from megadetector.utils.wi_taxonomy_utils import taxonomy_level_index
@@ -420,7 +421,7 @@ def _smooth_classifications_for_list_of_detections(detections,
 
     if verbose_debug_enabled:
         _print_counts_with_names(category_to_count,classification_descriptions)
-        from IPython import embed; embed()
+        # from IPython import embed; embed()
 
 
     ## Possibly change "other" classifications to the most common category
@@ -448,7 +449,7 @@ def _smooth_classifications_for_list_of_detections(detections,
                 if verbose_debug_enabled:
                     print('Replacing {} with {}'.format(
                         classification_descriptions[c[0]],
-                        classification_descriptions[c[1]]))
+                        most_common_category))
 
                 n_other_classifications_changed_this_image += 1
                 c[0] = most_common_category
@@ -918,8 +919,7 @@ def smooth_classification_results_image_level(input_file,output_file=None,option
 
     if output_file is not None:
         print('Writing results after image-level smoothing to:\n{}'.format(output_file))
-        with open(output_file,'w') as f:
-            json.dump(d,f,indent=1)
+        write_json(output_file,d)
 
     return d
 
@@ -1092,8 +1092,7 @@ def smooth_classification_results_sequence_level(input_file,
     if output_file is not None:
         print('Writing sequence-smoothed classification results to {}'.format(
             output_file))
-        with open(output_file,'w') as f:
-            json.dump(d,f,indent=1)
+        write_json(output_file,d)
 
     return d
 
@@ -1168,7 +1167,7 @@ def restrict_to_taxa_list(taxa_list,
     # Convert all NaN values in the "common" column to empty strings
     taxa_list_df['common'] = taxa_list_df['common'].fillna('')
 
-    # Create a dictionary mapping latin names to common names
+    # Create a dictionary mapping source Latin names to target common names
     target_latin_to_common = {}
 
     for i_row,row in taxa_list_df.iterrows():
@@ -1332,7 +1331,7 @@ def restrict_to_taxa_list(taxa_list,
         _insert_taxonomy_string(new_taxon_string)
 
 
-    ##%% Make sure all species on the allow-list are in the taxonomy
+    ##%% Make sure all taxa on the allow-list are in the taxonomy
 
     n_failed_mappings = 0
 
@@ -1498,7 +1497,8 @@ def restrict_to_taxa_list(taxa_list,
         if (protected_common_names is not None) and \
             (common_name in protected_common_names):
             if verbose:
-                print('Not messing with protected category {}'.format(common_name))
+                print('Not messing with protected category {}:\n{}'.format(
+                    common_name,input_taxon_string))
             input_category_id_to_output_taxon_string[input_category_id] = \
                 input_taxon_string
             continue
@@ -1578,12 +1578,13 @@ def restrict_to_taxa_list(taxa_list,
             output_taxon_string = speciesnet_latin_name_to_taxon_string[target_taxon]
         input_category_id_to_output_taxon_string[input_category_id] = output_taxon_string
 
-    # ...for each category
+    # ...for each category (mapping input category IDs to output taxon strings)
 
 
-    ##%% Build the new tables
+    ##%% Map input category IDs to output category IDs
 
-    speciesnet_taxon_string_to_latin_name = invert_dictionary(speciesnet_latin_name_to_taxon_string)
+    speciesnet_taxon_string_to_latin_name = \
+        invert_dictionary(speciesnet_latin_name_to_taxon_string)
 
     input_category_id_to_output_category_id = {}
     output_taxon_string_to_category_id = {}
@@ -1604,7 +1605,8 @@ def restrict_to_taxa_list(taxa_list,
             if speciesnet_latin_name in speciesnet_latin_name_to_output_common_name:
                 custom_common_name = speciesnet_latin_name_to_output_common_name[speciesnet_latin_name]
                 if custom_common_name != output_common_name:
-                    print('Substituting common name {} for {}'.format(custom_common_name,output_common_name))
+                    if verbose:
+                        print('Substituting common name {} for {}'.format(custom_common_name,output_common_name))
                     output_common_name = custom_common_name
 
         # Do we need to create a new output category?
@@ -1625,20 +1627,16 @@ def restrict_to_taxa_list(taxa_list,
         if False:
             original_common_name = \
               input_category_id_to_common_name[input_category_id]
-
             original_taxon_string = \
                 input_category_id_to_taxonomy_string[input_category_id]
-
             print('Mapping {} ({}) to:\n{} ({})\n'.format(
                 original_common_name,original_taxon_string,
                 output_common_name,output_taxon_string))
-            print('Mapping {} to {}'.format(
-                original_common_name,output_common_name,))
 
-    # ...for each category
+    # ...for each category (mapping input category IDs to output category IDs)
 
 
-    #%% Remap all category labels
+    ##%% Remap all category labels
 
     assert len(set(output_taxon_string_to_category_id.keys())) == \
            len(set(output_taxon_string_to_category_id.values())), \
@@ -1682,7 +1680,7 @@ def restrict_to_taxa_list(taxa_list,
 
     ##%% Write output
 
-    with open(output_file,'w') as f:
-        json.dump(output_data,f,indent=1)
+    write_json(output_file,output_data)
+
 
 # ...def restrict_to_taxa_list(...)

megadetector/postprocessing/combine_batch_outputs.py

@@ -203,7 +203,8 @@ def combine_api_shard_files(input_files, output_file=None):
     input_lists = []
     print('Loading input files')
     for fn in input_files:
-        input_lists.append(json.load(open(fn)))
+        with open(fn,'r') as f:
+            input_lists.append(json.load(f))
 
     detections = []
     # detection_list = input_lists[0]
@@ -214,7 +215,7 @@ def combine_api_shard_files(input_files, output_file=None):
             assert 'file' in d
             assert 'max_detection_conf' in d
             assert 'detections' in d
-            detections.extend([d])
+            detections.append(d)
 
     print('Writing output')
     if output_file is not None:

megadetector/postprocessing/compare_batch_results.py

@@ -353,10 +353,11 @@ def _render_image_pair(fn,image_pairs,category_folder,options,pairwise_options):
         im_gt = image_pair['im_gt']
         annotations_gt = image_pair['annotations_gt']
         gt_boxes = []
+        gt_categories = []
         for ann in annotations_gt:
             if 'bbox' in ann:
                 gt_boxes.append(ann['bbox'])
-        gt_categories = [ann['category_id'] for ann in annotations_gt]
+                gt_categories.append(ann['category_id'])
 
         if len(gt_boxes) > 0:
 
@@ -474,7 +475,7 @@ def _result_types_to_comparison_category(result_types_present_a,
                 ('tp' not in result_types_present_b):
                 return 'clean_tp_a_only'
         # Otherwise, TPs are cases where one model has only TPs, and the other model
-        # has any mistakse
+        # has any mistakes
         if ('fn' in result_types_present_b) or ('fp' in result_types_present_b):
             return 'tp_a_only'
 
@@ -486,7 +487,7 @@ def _result_types_to_comparison_category(result_types_present_a,
                 ('tp' not in result_types_present_a):
                 return 'clean_tp_b_only'
         # Otherwise, TPs are cases where one model has only TPs, and the other model
-        # has any mistakse
+        # has any mistakes
         if ('fn' in result_types_present_a) or ('fp' in result_types_present_a):
             return 'tp_b_only'
 
@@ -674,11 +675,17 @@ def _pairwise_compare_batch_results(options,output_index,pairwise_options):
     category_ids_to_include_a = []
     category_ids_to_include_b = []
 
-    for category_name in options.category_names_to_include:
-        if category_name in category_name_to_id_a:
-            category_ids_to_include_a.append(category_name_to_id_a[category_name])
-        if category_name in category_name_to_id_b:
-            category_ids_to_include_b.append(category_name_to_id_b[category_name])
+    # If we're supposed to be including all categories, we don't actually need to
+    # populate category_ids_to_include_a/b, but we're doing this for future-proofing.
+    if options.category_names_to_include is None:
+        category_ids_to_include_a = sorted(list(category_name_to_id_a.values()))
+        category_ids_to_include_b = sorted(list(category_name_to_id_b.values()))
+    else:
+        for category_name in options.category_names_to_include:
+            if category_name in category_name_to_id_a:
+                category_ids_to_include_a.append(category_name_to_id_a[category_name])
+            if category_name in category_name_to_id_b:
+                category_ids_to_include_b.append(category_name_to_id_b[category_name])
 
     if pairwise_options.results_description_a is None:
         if 'detector' not in results_a['info']:
@@ -814,7 +821,7 @@ def _pairwise_compare_batch_results(options,output_index,pairwise_options):
             print('Warning: {} files are only available in the ground truth (not in MD results)'.format(
                 len(filenames_only_in_gt)))
 
-        filenames_only_in_results = gt_filenames_set.difference(gt_filenames)
+        filenames_only_in_results = filenames_to_compare_set.difference(gt_filenames_set)
         if len(filenames_only_in_results) > 0:
             print('Warning: {} files are only available in the MD results (not in ground truth)'.format(
                 len(filenames_only_in_results)))
@@ -1185,13 +1192,6 @@ def _pairwise_compare_batch_results(options,output_index,pairwise_options):
             if ('tp' in result_types_present_b) or ('fn' in result_types_present_b):
                 assert 'tp' in result_types_present_a or 'fn' in result_types_present_a
 
-            # If either model has a TP or FN, the other has to have a TP or FN, since
-            # there was something in the GT
-            if ('tp' in result_types_present_a) or ('fn' in result_types_present_a):
-                assert 'tp' in result_types_present_b or 'fn' in result_types_present_b
-            if ('tp' in result_types_present_b) or ('fn' in result_types_present_b):
-                assert 'tp' in result_types_present_a or 'fn' in result_types_present_a
-
 
             ## Choose a comparison category based on result types
 
@@ -1677,8 +1677,8 @@ def n_way_comparison(filenames,
         '[detection_thresholds] should be the same length as [filenames]'
 
     if rendering_thresholds is not None:
-        assert len(rendering_thresholds) == len(filenames)
-        '[rendering_thresholds] should be the same length as [filenames]'
+        assert len(rendering_thresholds) == len(filenames), \
+            '[rendering_thresholds] should be the same length as [filenames]'
     else:
         rendering_thresholds = [(x*0.6666) for x in detection_thresholds]
 
@@ -1932,32 +1932,54 @@ def find_equivalent_threshold(results_a,
 
 if False:
 
+    #%% Prepare test files
+
+    from megadetector.utils.path_utils import insert_before_extension
+
+    model_names = ['mdv5a','mdv5b']
+    image_folder = 'g:/temp/md-test-images'
+    output_filename_base = os.path.join(image_folder,'comparison_test.json')
+
+    output_filenames = []
+
+    commands = []
+
+    for model_name in model_names:
+        output_filename = insert_before_extension(output_filename_base,model_name)
+        output_filenames.append(output_filename)
+        cmd = 'python -m megadetector.detection.run_detector_batch'
+        cmd += ' {} {} {} --recursive --output_relative_filenames'.format(
+            model_name, image_folder,output_filename)
+        commands.append(cmd)
+
+    cmd = '\n\n'.join(commands)
+    print(cmd)
+    import clipboard
+    clipboard.copy(cmd)
+
+
     #%% Test two-way comparison
 
     options = BatchComparisonOptions()
 
     options.parallelize_rendering_with_threads = True
 
-    options.job_name = 'BCT'
+    options.job_name = 'md-test-images'
     options.output_folder = r'g:\temp\comparisons'
-    options.image_folder = r'g:\camera_traps\camera_trap_images'
+    options.image_folder = image_folder
     options.max_images_per_category = 100
     options.sort_by_confidence = True
 
     options.pairwise_options = []
 
     results_base = os.path.expanduser('~/postprocessing/bellevue-camera-traps')
-    filenames = [
-        os.path.join(results_base,r'bellevue-camera-traps-2023-12-05-v5a.0.0\combined_api_outputs\bellevue-camera-traps-2023-12-05-v5a.0.0_detections.json'),
-        os.path.join(results_base,r'bellevue-camera-traps-2023-12-05-aug-v5a.0.0\combined_api_outputs\bellevue-camera-traps-2023-12-05-aug-v5a.0.0_detections.json')
-        ]
 
     detection_thresholds = [0.15,0.15]
     rendering_thresholds = None
 
-    results = n_way_comparison(filenames,
-                               options,
-                               detection_thresholds,
+    results = n_way_comparison(filenames=output_filenames,
+                               options=options,
+                               detection_thresholds=detection_thresholds,
                                rendering_thresholds=rendering_thresholds)
 
     from megadetector.utils.path_utils import open_file

megadetector/postprocessing/convert_output_format.py

@@ -22,7 +22,8 @@ import pandas as pd
 from megadetector.postprocessing.load_api_results import load_api_results_csv
 from megadetector.utils.wi_taxonomy_utils import load_md_or_speciesnet_file
 from megadetector.data_management.annotations import annotation_constants
-from megadetector.utils import ct_utils
+from megadetector.utils.ct_utils import get_max_conf
+from megadetector.utils.ct_utils import write_json
 
 CONF_DIGITS = 3
 
@@ -138,7 +139,7 @@ def convert_json_to_csv(input_path,
             # print('Skipping failed image {} ({})'.format(im['file'],im['failure']))
             continue
 
-        max_conf = ct_utils.get_max_conf(im)
+        max_conf = get_max_conf(im)
         detection_category_id_to_max_conf = defaultdict(float)
         classification_category_id_to_max_conf = defaultdict(float)
         detections = []
@@ -177,7 +178,8 @@ def convert_json_to_csv(input_path,
                     classification_category_max = \
                         classification_category_id_to_max_conf[classification_category_id]
                     if classification_conf > classification_category_max:
-                        classification_category_id_to_max_conf[classification_category_id] = d['conf']
+                        classification_category_id_to_max_conf[classification_category_id] = \
+                            classification_conf
 
                 # ...for each classification
 
@@ -210,7 +212,7 @@ def convert_json_to_csv(input_path,
 
     if omit_bounding_boxes:
         df = df.drop('detections',axis=1)
-    df.to_csv(output_path,index=False,header=True)
+    df.to_csv(output_path,index=False,header=True,encoding=output_encoding)
 
 # ...def convert_json_to_csv(...)
 
@@ -295,7 +297,7 @@ def convert_csv_to_json(input_path,output_path=None,overwrite=True):
     json_out['classification_categories'] = classification_categories
     json_out['images'] = images
 
-    json.dump(json_out,open(output_path,'w'),indent=1)
+    write_json(output_path,json_out)
 
 # ...def convert_csv_to_json(...)
 
@@ -372,7 +374,7 @@ def main():
                         help='Output filename ending in .json or .csv (defaults to ' + \
                              'input file, with .json/.csv replaced by .csv/.json)')
     parser.add_argument('--omit_bounding_boxes',action='store_true',
-                        help='Output bounding box text from .csv output (large and usually not useful)')
+                        help='Omit bounding box text from .csv output (large and usually not useful)')
 
     if len(sys.argv[1:]) == 0:
         parser.print_help()

megadetector/postprocessing/create_crop_folder.py

@@ -169,7 +169,9 @@ def crop_results_to_image_results(image_results_file_with_crop_ids,
         'Could not find image-level input file {}'.format(image_results_file_with_crop_ids)
     assert os.path.isfile(crop_results_file), \
         'Could not find crop results file {}'.format(crop_results_file)
-    os.makedirs(os.path.dirname(output_file),exist_ok=True)
+    output_dir = os.path.dirname(output_file)
+    if len(output_dir) > 0:
+        os.makedirs(output_dir,exist_ok=True)
 
 
     ##%% Read input files
@@ -259,7 +261,11 @@ def crop_results_to_image_results(image_results_file_with_crop_ids,
                                 detections_without_classification_handling
                         ))
 
-                if not skip_detection:
+                if skip_detection:
+
+                    n_skipped_detections += 1
+
+                else:
 
                     crop_results_this_detection = crop_filename_to_results[crop_filename_relative]
 
@@ -340,8 +346,11 @@ def create_crop_folder(input_file,
     assert os.path.isfile(input_file), 'Input file {} not found'.format(input_file)
     assert os.path.isdir(input_folder), 'Input folder {} not found'.format(input_folder)
     os.makedirs(output_folder,exist_ok=True)
+
     if output_file is not None:
-        os.makedirs(os.path.dirname(output_file),exist_ok=True)
+        output_dir = os.path.dirname(output_file)
+        if len(output_dir) > 0:
+            os.makedirs(output_dir,exist_ok=True)
 
 
     ##%% Read input
@@ -599,7 +608,7 @@ def main():
 
     print('Starting crop folder creation...')
     print('Input MD results: {}'.format(args.input_file))
-    print('Input image folder {}'.format(args.input_folder))
+    print('Input image folder: {}'.format(args.input_folder))
     print('Output crop folder: {}'.format(args.output_folder))
 
     if args.output_file:

megadetector/postprocessing/generate_csv_report.py

@@ -126,6 +126,7 @@ def generate_csv_report(md_results_file,
                                                      recursive=True)
 
         else:
+
             assert os.path.isfile(datetime_source), \
                 'datetime source {} is neither a folder nor a file'.format(datetime_source)
 
@@ -153,11 +154,14 @@ def generate_csv_report(md_results_file,
                     print('Warning: a MD results file was supplied as the datetime source, but it does not appear '
                           'to contain datetime information.')
 
+        # ...if datetime_source is a folder/file
+
         assert all_exif_results is not None
 
         filename_to_datetime_string = {}
 
         for exif_result in all_exif_results:
+
             datetime_string = unknown_datetime_tag
             if ('exif_tags' in exif_result) and \
                (exif_result['exif_tags'] is not None) and \
@@ -169,6 +173,8 @@ def generate_csv_report(md_results_file,
                     assert isinstance(datetime_string,str), 'Unrecognized datetime format'
             filename_to_datetime_string[exif_result['file_name']] = datetime_string
 
+        # ...for each exif result
+
         image_files = [im['file'] for im in results['images']]
         image_files_set = set(image_files)
 
@@ -250,11 +256,10 @@ def generate_csv_report(md_results_file,
         base_record['filename'] = im['file'].replace('\\','/')
 
         # Datetime (if necessary)
+        datetime_string = ''
         if filename_to_datetime_string is not None:
             if im['file'] in filename_to_datetime_string:
                 datetime_string = filename_to_datetime_string[im['file']]
-            else:
-                datetime_string = ''
         base_record['datetime'] = datetime_string
 
         for s in ['detection_category','max_detection_confidence',
@@ -383,13 +388,22 @@ def generate_csv_report(md_results_file,
     # ...for each image
 
     # Make sure every record has the same columns
-    column_names = output_records[0].keys()
-    for record in output_records:
-        assert record.keys() == column_names
 
-    # Write to .csv
-    df = pd.DataFrame(output_records)
-    df.to_csv(output_file,header=True,index=False)
+    if len(output_records) == 0:
+        print('Warning: no output records generated')
+    else:
+        column_names = output_records[0].keys()
+        for record in output_records:
+            assert record.keys() == column_names
+
+        # Create folder for output file if necessary
+        output_dir = os.path.dirname(output_file)
+        if len(output_dir) > 0:
+            os.makedirs(output_dir, exist_ok=True)
+
+        # Write to .csv
+        df = pd.DataFrame(output_records)
+        df.to_csv(output_file,header=True,index=False)
 
     # from megadetector.utils.path_utils import open_file; open_file(output_file)
 

megadetector/postprocessing/load_api_results.py

@@ -23,7 +23,8 @@ from collections.abc import Mapping
 
 import pandas as pd
 
-from megadetector.utils import ct_utils
+from megadetector.utils.ct_utils import get_max_conf
+from megadetector.utils.ct_utils import write_json
 from megadetector.utils.wi_taxonomy_utils import load_md_or_speciesnet_file
 
 
@@ -85,7 +86,7 @@ def load_api_results(api_output_path: str, normalize_paths: bool = True,
     # add them, because our unofficial internal dataframe format includes this.
     for im in detection_results['images']:
         if 'max_detection_conf' not in im:
-            im['max_detection_conf'] = ct_utils.get_max_conf(im)
+            im['max_detection_conf'] = get_max_conf(im)
 
     # Pack the json output into a Pandas DataFrame
     detection_results = pd.DataFrame(detection_results['images'])
@@ -139,8 +140,7 @@ def write_api_results(detection_results_table, other_fields, out_path):
         print('Warning: error removing max_detection_conf from output')
         pass
 
-    with open(out_path, 'w') as f:
-        json.dump(fields, f, indent=1)
+    write_json(out_path,fields)
 
     print('Finished writing detection results to {}'.format(out_path))
 
@@ -214,6 +214,10 @@ def write_api_results_csv(detection_results, filename):
 
     print('Writing detection results to {}'.format(filename))
 
+    output_dir = os.path.dirname(filename)
+    if len(output_dir) > 0:
+        os.makedirs(output_dir, exist_ok=True)
+
     detection_results.to_csv(filename, index=False)
 
     print('Finished writing detection results to {}'.format(filename))

megadetector/postprocessing/md_to_coco.py

@@ -22,6 +22,7 @@ from tqdm import tqdm
 
 from megadetector.visualization import visualization_utils as vis_utils
 from megadetector.utils.path_utils import insert_before_extension
+from megadetector.utils.ct_utils import write_json
 
 default_confidence_threshold = 0.15
 
@@ -296,9 +297,7 @@ def md_to_coco(md_results_file,
     if verbose:
         print('Writing COCO output file...')
 
-    if coco_output_file is not None:
-        with open(coco_output_file,'w') as f:
-            json.dump(output_dict,f,indent=1)
+    write_json(coco_output_file,output_dict)
 
     return output_dict