megadetector 5.0.6__py3-none-any.whl → 5.0.8__py3-none-any.whl

api/batch_processing/postprocessing/subset_json_detector_output.py

@@ -17,6 +17,11 @@
 #    rather than an output path.  All images in the folder blah\foo\bar will end up 
 #    in a .json file called blah_foo_bar.json.
 #
+# Can also apply a confidence threshold.
+#
+# Can also subset by categories above a threshold (programmatic invocation only, this is
+# not supported at the command line yet).
+#
 ###
 #
 # Sample invocations (splitting into multiple json's):
@@ -65,6 +70,7 @@ from tqdm import tqdm
 
 from md_utils.ct_utils import args_to_object
 from md_utils.ct_utils import get_max_conf
+from md_utils.ct_utils import invert_dictionary
 
 
 #%% Helper classes
@@ -118,6 +124,17 @@ class SubsetJsonDetectorOutputOptions:
     # Should we remove failed images?
     remove_failed_images = False
     
+    # Either a list of category IDs (as string-ints) (not names), or a dictionary mapping category *IDs* 
+    # (as string-ints) (not names) to thresholds.  Removes  non-matching detections, does not 
+    # remove images.  Not technically mutually exclusize with category_names_to_keep, but it's an esoteric 
+    # scenario indeed where you would want to specify both.
+    categories_to_keep = None
+    
+    # Either a list of category names (not IDs), or a dictionary mapping category *names* (not IDs) to thresholds.  
+    # Removes non-matching detections, does not remove images.  Not technically mutually exclusize with 
+    # category_ids_to_keep, but it's an esoteric scenario indeed where you would want to specify both.
+    category_names_to_keep = None
+    
     debug_max_images = -1
     
     
@@ -151,7 +168,7 @@ def subset_json_detector_output_by_confidence(data, options):
     Remove all detections below options.confidence_threshold, update max confidences accordingly.
     """
     
-    if not options.confidence_threshold:
+    if options.confidence_threshold is None:
         return data
     
     images_in = data['images']
@@ -161,10 +178,13 @@ def subset_json_detector_output_by_confidence(data, options):
     
     n_max_changes = 0
     
-    # iImage = 0; im = images_in[0]
-    for iImage, im in tqdm(enumerate(images_in), total=len(images_in)):
+    # im = images_in[0]
+    for i_image, im in tqdm(enumerate(images_in), total=len(images_in)):
         
+        # Always keep failed images; if the caller wants to remove these, they
+        # will use remove_failed_images
         if ('detections' not in im) or (im['detections'] is None):
+            images_out.append(im)
             continue
         
         p_orig = get_max_conf(im)
@@ -182,7 +202,7 @@ def subset_json_detector_output_by_confidence(data, options):
 
         # Otherwise find the max confidence
         else:
-            p = max(d['conf'] for d in detections)
+            p = max([d['conf'] for d in detections])
         
         im['detections'] = detections
 
@@ -190,11 +210,13 @@ def subset_json_detector_output_by_confidence(data, options):
         if abs(p_orig - p) > 0.00001:
 
             # We should only be *lowering* max confidence values (i.e., making them negative)
-            assert (p_orig <= 0) or (p < p_orig), 'Confidence changed from {} to {}'.format(p_orig, p)
+            assert (p_orig <= 0) or (p < p_orig), \
+                'Confidence changed from {} to {}'.format(p_orig, p)
             n_max_changes += 1
         
         if 'max_detection_conf' in im:
             im['max_detection_conf'] = p
+            
         images_out.append(im)
         
     # ...for each image        
@@ -208,6 +230,107 @@ def subset_json_detector_output_by_confidence(data, options):
 # ...subset_json_detector_output_by_confidence()
 
 
+def subset_json_detector_output_by_categories(data, options):
+    """
+    Remove all detections without detections above a threshold for specific categories.
+    """
+    
+    # If categories_to_keep is supplied as a list, convert to a dict
+    if options.categories_to_keep is not None:
+        if not isinstance(options.categories_to_keep, dict):
+            dict_categories_to_keep = {}
+            for category_id in options.categories_to_keep:
+                # Set unspecified thresholds to a silly negative value
+                dict_categories_to_keep[category_id] = -100000.0
+            options.categories_to_keep = dict_categories_to_keep
+    
+    # If category_names_to_keep is supplied as a list, convert to a dict
+    if options.category_names_to_keep is not None:
+        if not isinstance(options.category_names_to_keep, dict):
+            dict_category_names_to_keep = {}
+            for category_name in options.category_names_to_keep:
+                # Set unspecified thresholds to a silly negative value
+                dict_category_names_to_keep[category_name] = -100000.0
+            options.category_names_to_keep = dict_category_names_to_keep
+            
+    category_name_to_category_id = invert_dictionary(data['detection_categories'])
+    
+    # If some categories are supplied as names, convert all to IDs and add to "categories_to_keep"
+    if options.category_names_to_keep is not None:
+        if options.categories_to_keep is None:
+            options.categories_to_keep = {}
+        for category_name in options.category_names_to_keep:
+            assert category_name in category_name_to_category_id, \
+                'Category {} not in detection categories'.format(category_name)
+            category_id = category_name_to_category_id[category_name]
+            assert category_id not in options.categories_to_keep, \
+                'Category {} ({}) specified as both a name and an ID'.format(
+                    category_name,category_id)
+            options.categories_to_keep[category_id] = options.category_names_to_keep[category_name]
+    
+    if options.categories_to_keep is None:
+        return data
+    
+    images_in = data['images']
+    images_out = []    
+    
+    print('Subsetting by categories (keeping {} categories):'.format(
+        len(options.categories_to_keep)))
+    
+    for category_id in sorted(list(options.categories_to_keep.keys())):
+        if category_id not in data['detection_categories']:
+            print('Warning: category ID {} not in category map in this file'.format(category_id))
+        else:
+            print('{} ({}) (threshold {})'.format(
+                category_id,
+                data['detection_categories'][category_id],
+                options.categories_to_keep[category_id]))
+    
+    n_detections_in = 0
+    n_detections_kept = 0
+    
+    # im = images_in[0]
+    for i_image, im in tqdm(enumerate(images_in), total=len(images_in)):
+        
+        # Always keep failed images; if the caller wants to remove these, they
+        # will use remove_failed_images        
+        if ('detections' not in im) or (im['detections'] is None):
+            images_out.append(im)
+            continue
+        
+        n_detections_in += len(im['detections'])
+                                  
+        # Find all matching detections for this image
+        detections = []
+        for d in im['detections']:
+            if (d['category'] in options.categories_to_keep) and \
+               (d['conf'] > options.categories_to_keep[d['category']]):
+               detections.append(d)
+                       
+        im['detections'] = detections
+
+        if 'max_detection_conf' in im:
+            if len(detections) == 0:
+                p = 0
+            else:
+                p = max([d['conf'] for d in detections])
+            im['max_detection_conf'] = p
+        
+        n_detections_kept += len(im['detections'])
+            
+        images_out.append(im)
+        
+    # ...for each image        
+    
+    data['images'] = images_out    
+    print('done, kept {} detections (of {})'.format(
+        n_detections_kept,n_detections_in))
+    
+    return data
+
+# ...subset_json_detector_output_by_categories()
+
+
 def remove_failed_images(data,options):
     """
     Removed failed images from [data]
@@ -399,6 +522,10 @@ def subset_json_detector_output(input_filename, output_filename, options, data=N
         
         data = subset_json_detector_output_by_confidence(data, options)
         
+    if (options.categories_to_keep is not None) or (options.category_names_to_keep is not None):
+        
+        data = subset_json_detector_output_by_categories(data, options)
+        
     if not options.split_folders:
         
         write_detection_results(data, output_filename, options)

api/batch_processing/postprocessing/top_folders_to_bottom.py

@@ -186,4 +186,4 @@ if __name__ == '__main__':
     main()
     
 
-    
+    

classification/prepare_classification_script.py

@@ -1,191 +1,191 @@
-########
-#
-# prepare_classification_script.py
-#
-# Notebook-y script used to prepare a series of shell commands to run a classifier
-# (other than MegaClassifier) on a MegaDetector result set.
-#
-# Differs from prepare_classification_script_mc.py only in the final class mapping step.
-#
-########
-
-#%% Job options
-
-import os
-
-organization_name = 'idfg'
-job_name = 'idfg-2022-01-27-EOE2021S_Group6'
-input_filename = 'idfg-2022-01-27-EOE2021S_Group6_detections.filtered_rde_0.60_0.85_30_0.20.json'
-image_base = '/datadrive/idfg/EOE2021S_Group6'
-crop_path = os.path.join(os.path.expanduser('~/crops'),job_name + '_crops')
-device_id = 1
-
-working_dir_base = os.path.join(os.path.expanduser('~/postprocessing'),
-                                                   organization_name,
-                                                   job_name)
-
-output_base = os.path.join(working_dir_base,'combined_api_outputs')
-
-assert os.path.isdir(working_dir_base)
-assert os.path.isdir(output_base)
-
-output_file = os.path.join(working_dir_base,'run_idfgclassifier_' + job_name +  '.sh')
-
-input_files = [
-    os.path.join(
-        os.path.expanduser('~/postprocessing'),
-                           organization_name,
-                           job_name,
-                           'combined_api_outputs',
-                           input_filename
-        )
-    ]
-
-for fn in input_files:
-    assert os.path.isfile(fn)
-    
-
-#%% Constants
-
-include_cropping = False
-
-classifier_base = os.path.expanduser('~/models/camera_traps/idfg_classifier/idfg_classifier_20200905_042558')
-assert os.path.isdir(classifier_base)
-
-checkpoint_path = os.path.join(classifier_base,'idfg_classifier_ckpt_14_compiled.pt')
-assert os.path.isfile(checkpoint_path)
-
-classifier_categories_path = os.path.join(classifier_base,'label_index.json')
-assert os.path.isfile(classifier_categories_path)
-
-classifier_output_suffix = '_idfg_classifier_output.csv.gz'
-final_output_suffix = '_idfgclassifier.json'
-
-threshold_str = '0.65'
-n_threads_str = '50'
-image_size_str = '300'
-batch_size_str = '64'
-num_workers_str = '8'
-logdir = working_dir_base
-
-classification_threshold_str = '0.05'
-
-# This is just passed along to the metadata in the output file, it has no impact
-# on how the classification scripts run.
-typical_classification_threshold_str = '0.75'
-
-classifier_name = 'idfg4'
-        
-
-#%% Set up environment
-
-commands = []
-# commands.append('cd MegaDetector/classification\n')
-# commands.append('conda activate cameratraps-classifier\n')
-
-
-#%% Crop images
-
-if include_cropping:
-    
-    commands.append('\n### Cropping ###\n')
-    
-    # fn = input_files[0]
-    for fn in input_files:
-    
-        input_file_path = fn
-        crop_cmd = ''
-        
-        crop_comment = '\n# Cropping {}\n'.format(fn)
-        crop_cmd += crop_comment
-        
-        crop_cmd += "python crop_detections.py \\\n" + \
-        	 input_file_path + ' \\\n' + \
-             crop_path + ' \\\n' + \
-             '--images-dir "' + image_base + '"' + ' \\\n' + \
-             '--threshold "' + threshold_str + '"' + ' \\\n' + \
-             '--square-crops ' + ' \\\n' + \
-             '--threads "' + n_threads_str + '"' + ' \\\n' + \
-             '--logdir "' + logdir + '"' + ' \\\n' + \
-             '\n'
-        crop_cmd = '{}'.format(crop_cmd)
-        commands.append(crop_cmd)
-
-
-#%% Run classifier
-
-commands.append('\n### Classifying ###\n')
-
-# fn = input_files[0]
-for fn in input_files:
-
-    input_file_path = fn
-    classifier_output_path = crop_path + classifier_output_suffix
-    
-    classify_cmd = ''
-    
-    classify_comment = '\n# Classifying {}\n'.format(fn)
-    classify_cmd += classify_comment
-    
-    classify_cmd += "python run_classifier.py \\\n" + \
-    	 checkpoint_path + ' \\\n' + \
-         crop_path + ' \\\n' + \
-         classifier_output_path + ' \\\n' + \
-         '--detections-json "' + input_file_path + '"' + ' \\\n' + \
-         '--classifier-categories "' + classifier_categories_path + '"' + ' \\\n' + \
-         '--image-size "' + image_size_str + '"' + ' \\\n' + \
-         '--batch-size "' + batch_size_str + '"' + ' \\\n' + \
-         '--num-workers "' + num_workers_str + '"' + ' \\\n'
-    
-    if device_id is not None:
-        classify_cmd += '--device {}'.format(device_id)
-        
-    classify_cmd += '\n\n'    
-    classify_cmd = '{}'.format(classify_cmd)
-    commands.append(classify_cmd)
-		
-
-#%% Merge classification and detection outputs
-
-commands.append('\n### Merging ###\n')
-
-# fn = input_files[0]
-for fn in input_files:
-
-    input_file_path = fn
-    classifier_output_path = crop_path + classifier_output_suffix
-    final_output_path = os.path.join(output_base,
-                                     os.path.basename(classifier_output_path)).\
-                                     replace(classifier_output_suffix,
-                                     final_output_suffix)
-    final_output_path = final_output_path.replace('_detections','')
-    final_output_path = final_output_path.replace('_crops','')
-    
-    merge_cmd = ''
-    
-    merge_comment = '\n# Merging {}\n'.format(fn)
-    merge_cmd += merge_comment
-    
-    merge_cmd += "python merge_classification_detection_output.py \\\n" + \
-    	 classifier_output_path + ' \\\n' + \
-         classifier_categories_path + ' \\\n' + \
-         '--output-json "' + final_output_path + '"' + ' \\\n' + \
-         '--detection-json "' + input_file_path + '"' + ' \\\n' + \
-         '--classifier-name "' + classifier_name + '"' + ' \\\n' + \
-         '--threshold "' + classification_threshold_str + '"' + ' \\\n' + \
-         '--typical-confidence-threshold "' + typical_classification_threshold_str + '"' + ' \\\n' + \
-         '\n'
-    merge_cmd = '{}'.format(merge_cmd)
-    commands.append(merge_cmd)
-
-
-#%% Write everything out
-
-with open(output_file,'w') as f:
-    for s in commands:
-        f.write('{}'.format(s))
-
-import stat
-st = os.stat(output_file)
-os.chmod(output_file, st.st_mode | stat.S_IEXEC)
-        
+########
+#
+# prepare_classification_script.py
+#
+# Notebook-y script used to prepare a series of shell commands to run a classifier
+# (other than MegaClassifier) on a MegaDetector result set.
+#
+# Differs from prepare_classification_script_mc.py only in the final class mapping step.
+#
+########
+
+#%% Job options
+
+import os
+
+organization_name = 'idfg'
+job_name = 'idfg-2022-01-27-EOE2021S_Group6'
+input_filename = 'idfg-2022-01-27-EOE2021S_Group6_detections.filtered_rde_0.60_0.85_30_0.20.json'
+image_base = '/datadrive/idfg/EOE2021S_Group6'
+crop_path = os.path.join(os.path.expanduser('~/crops'),job_name + '_crops')
+device_id = 1
+
+working_dir_base = os.path.join(os.path.expanduser('~/postprocessing'),
+                                                   organization_name,
+                                                   job_name)
+
+output_base = os.path.join(working_dir_base,'combined_api_outputs')
+
+assert os.path.isdir(working_dir_base)
+assert os.path.isdir(output_base)
+
+output_file = os.path.join(working_dir_base,'run_idfgclassifier_' + job_name +  '.sh')
+
+input_files = [
+    os.path.join(
+        os.path.expanduser('~/postprocessing'),
+                           organization_name,
+                           job_name,
+                           'combined_api_outputs',
+                           input_filename
+        )
+    ]
+
+for fn in input_files:
+    assert os.path.isfile(fn)
+    
+
+#%% Constants
+
+include_cropping = False
+
+classifier_base = os.path.expanduser('~/models/camera_traps/idfg_classifier/idfg_classifier_20200905_042558')
+assert os.path.isdir(classifier_base)
+
+checkpoint_path = os.path.join(classifier_base,'idfg_classifier_ckpt_14_compiled.pt')
+assert os.path.isfile(checkpoint_path)
+
+classifier_categories_path = os.path.join(classifier_base,'label_index.json')
+assert os.path.isfile(classifier_categories_path)
+
+classifier_output_suffix = '_idfg_classifier_output.csv.gz'
+final_output_suffix = '_idfgclassifier.json'
+
+threshold_str = '0.65'
+n_threads_str = '50'
+image_size_str = '300'
+batch_size_str = '64'
+num_workers_str = '8'
+logdir = working_dir_base
+
+classification_threshold_str = '0.05'
+
+# This is just passed along to the metadata in the output file, it has no impact
+# on how the classification scripts run.
+typical_classification_threshold_str = '0.75'
+
+classifier_name = 'idfg4'
+        
+
+#%% Set up environment
+
+commands = []
+# commands.append('cd MegaDetector/classification\n')
+# commands.append('conda activate cameratraps-classifier\n')
+
+
+#%% Crop images
+
+if include_cropping:
+    
+    commands.append('\n### Cropping ###\n')
+    
+    # fn = input_files[0]
+    for fn in input_files:
+    
+        input_file_path = fn
+        crop_cmd = ''
+        
+        crop_comment = '\n# Cropping {}\n'.format(fn)
+        crop_cmd += crop_comment
+        
+        crop_cmd += "python crop_detections.py \\\n" + \
+        	 input_file_path + ' \\\n' + \
+             crop_path + ' \\\n' + \
+             '--images-dir "' + image_base + '"' + ' \\\n' + \
+             '--threshold "' + threshold_str + '"' + ' \\\n' + \
+             '--square-crops ' + ' \\\n' + \
+             '--threads "' + n_threads_str + '"' + ' \\\n' + \
+             '--logdir "' + logdir + '"' + ' \\\n' + \
+             '\n'
+        crop_cmd = '{}'.format(crop_cmd)
+        commands.append(crop_cmd)
+
+
+#%% Run classifier
+
+commands.append('\n### Classifying ###\n')
+
+# fn = input_files[0]
+for fn in input_files:
+
+    input_file_path = fn
+    classifier_output_path = crop_path + classifier_output_suffix
+    
+    classify_cmd = ''
+    
+    classify_comment = '\n# Classifying {}\n'.format(fn)
+    classify_cmd += classify_comment
+    
+    classify_cmd += "python run_classifier.py \\\n" + \
+    	 checkpoint_path + ' \\\n' + \
+         crop_path + ' \\\n' + \
+         classifier_output_path + ' \\\n' + \
+         '--detections-json "' + input_file_path + '"' + ' \\\n' + \
+         '--classifier-categories "' + classifier_categories_path + '"' + ' \\\n' + \
+         '--image-size "' + image_size_str + '"' + ' \\\n' + \
+         '--batch-size "' + batch_size_str + '"' + ' \\\n' + \
+         '--num-workers "' + num_workers_str + '"' + ' \\\n'
+    
+    if device_id is not None:
+        classify_cmd += '--device {}'.format(device_id)
+        
+    classify_cmd += '\n\n'    
+    classify_cmd = '{}'.format(classify_cmd)
+    commands.append(classify_cmd)
+		
+
+#%% Merge classification and detection outputs
+
+commands.append('\n### Merging ###\n')
+
+# fn = input_files[0]
+for fn in input_files:
+
+    input_file_path = fn
+    classifier_output_path = crop_path + classifier_output_suffix
+    final_output_path = os.path.join(output_base,
+                                     os.path.basename(classifier_output_path)).\
+                                     replace(classifier_output_suffix,
+                                     final_output_suffix)
+    final_output_path = final_output_path.replace('_detections','')
+    final_output_path = final_output_path.replace('_crops','')
+    
+    merge_cmd = ''
+    
+    merge_comment = '\n# Merging {}\n'.format(fn)
+    merge_cmd += merge_comment
+    
+    merge_cmd += "python merge_classification_detection_output.py \\\n" + \
+    	 classifier_output_path + ' \\\n' + \
+         classifier_categories_path + ' \\\n' + \
+         '--output-json "' + final_output_path + '"' + ' \\\n' + \
+         '--detection-json "' + input_file_path + '"' + ' \\\n' + \
+         '--classifier-name "' + classifier_name + '"' + ' \\\n' + \
+         '--threshold "' + classification_threshold_str + '"' + ' \\\n' + \
+         '--typical-confidence-threshold "' + typical_classification_threshold_str + '"' + ' \\\n' + \
+         '\n'
+    merge_cmd = '{}'.format(merge_cmd)
+    commands.append(merge_cmd)
+
+
+#%% Write everything out
+
+with open(output_file,'w') as f:
+    for s in commands:
+        f.write('{}'.format(s))
+
+import stat
+st = os.stat(output_file)
+os.chmod(output_file, st.st_mode | stat.S_IEXEC)
+        

data_management/cct_json_utils.py

@@ -142,7 +142,8 @@ class IndexedJsonDb:
     def __init__(self, json_filename: Union[str, JSONObject],
                  b_normalize_paths: bool = False,
                  filename_replacements: Optional[Mapping[str, str]] = None,
-                 b_convert_classes_to_lower: bool = True):
+                 b_convert_classes_to_lower: bool = True,
+                 b_force_forward_slashes: bool = True):
         """
         json_filename can also be an existing json db
         """
@@ -162,11 +163,15 @@ class IndexedJsonDb:
             for c in self.db['categories']:
                 c['name'] = c['name'].lower()
 
+        # Normalize paths to simplify comparisons later
         if b_normalize_paths:
-            # Normalize paths to simplify comparisons later
             for im in self.db['images']:
                 im['file_name'] = os.path.normpath(im['file_name'])
 
+        if b_force_forward_slashes:
+            for im in self.db['images']:
+                im['file_name'] = im['file_name'].replace('\\','/')
+
         if filename_replacements is not None:
             for s in filename_replacements:
                 # Make custom replacements in filenames, typically used to