megadetector 5.0.8__py3-none-any.whl → 5.0.10__py3-none-any.whl

api/batch_processing/postprocessing/remap_detection_categories.py

@@ -1,13 +1,13 @@
-########
-#
-# remap_detection_categories.py
-#
-# Given a MegaDetector results file, remap the category IDs according to a specified 
-# dictionary, writing the results to a new file.
-#
-# Currently only supports remapping detection categories, not classification categories.
-#
-########
+"""
+
+remap_detection_categories.py
+
+Given a MegaDetector results file, remap the category IDs according to a specified 
+dictionary, writing the results to a new file.
+
+Currently only supports remapping detection categories, not classification categories.
+
+"""
 
 #%% Constants and imports
 
@@ -27,27 +27,35 @@ def remap_detection_categories(input_file,
                                extra_category_handling='error',
                                overwrite=False):
     """
-    Given a MD results file [input_file], remap the category IDs according to the dictionary
+    Given a MegaDetector results file [input_file], remap the category IDs according to the dictionary
     [target_category_map], writing the results to [output_file].  The remapped dictionary needs to have 
     the same category names as the input file's detection_categories dictionary.
     
-    Currently only supports remapping detection categories, not classification categories.
-    
-    target_category_map can also be a MD results file, in which case we'll use that file's
-    detection_categories dictionary.
+    Typically used to map, e.g., a variety of species to the class "mammal" or the class "animal".
     
-    [extra_category_handling] specifies what we should do if categories are present in the source file 
-    that are not present in the target mapping.
-    
-    'error' == Error in this case.
-    'drop_if_unused' == Don't include these in the output file's category mappings if they are unused,
-       error if they are.
-    'remap' == Remap to unused category IDs.  This is reserved for future use, not currently implemented.
+    Currently only supports remapping detection categories, not classification categories.
     
+    Args:
+        input_file (str): the MD .json results file to remap
+        output_file (str): the remapped .json file to write
+        target_category_map (dict): the category mapping that should be used in the output file.
+            This can also be a MD results file, in which case we'll use that file's
+            detection_categories dictionary.
+        extra_category_handling (str, optional): specifies what we should do if categories are present
+            in the source file that are not present in the target mapping:
+            
+            * 'error' == Error in this case.
+            * 'drop_if_unused' == Don't include these in the output file's category mappings if they are 
+              unused, error if they are.
+            * 'remap' == Remap to unused category IDs.  This is reserved for future use, not currently
+              implemented.
+        overwrite (bool, optional): whether to overwrite [output_file] if it exists; if this is True and
+            [output_file] exists, this function is a no-op
+               
     """
     
     if os.path.exists(output_file) and (not overwrite):
-        print('File {} exists, bypassing remapping'.format(output_file))
+        print('File {} exists, bypassing remapping'.format(output_file))        
         return
     
     assert os.path.isfile(input_file), \
@@ -130,7 +138,7 @@ def remap_detection_categories(input_file,
     
     
     print('Saved remapped results to {}'.format(output_file))
-
+    
 
 #%% Interactive driver
 
@@ -160,4 +168,3 @@ if False:
 #%% Command-line driver
 
 # TODO
-

api/batch_processing/postprocessing/render_detection_confusion_matrix.py

@@ -1,12 +1,12 @@
-########
-#
-# render_detection_confusion_matrix.py
-#
-# Given a CCT-formatted ground truth file and a MegaDetector-formatted results file,
-# render an HTML confusion matrix.  Typically used for multi-class detectors.  Currently
-# assumes a single class per image.
-#
-########
+"""
+
+render_detection_confusion_matrix.py
+
+Given a CCT-formatted ground truth file and a MegaDetector-formatted results file,
+render an HTML confusion matrix.  Typically used for multi-class detectors.  Currently
+assumes a single class per image.
+
+"""
 
 #%% Imports and constants
 
@@ -33,7 +33,10 @@ from multiprocessing.pool import Pool
 
 #%% Support functions
 
-def image_to_output_file(im,preview_images_folder):
+def _image_to_output_file(im,preview_images_folder):
+    """
+    Produces a clean filename from im (if [im] is a str) or im['file'] (if [im] is a dict).
+    """
     
     if isinstance(im,str):
         filename_relative = im
@@ -44,7 +47,10 @@ def image_to_output_file(im,preview_images_folder):
     return os.path.join(preview_images_folder,fn_clean)
 
 
-def render_image(im,render_image_constants):
+def _render_image(im,render_image_constants):
+    """
+    Internal function for rendering a single image to the confusion matrix preview folder.
+    """
     
     filename_to_ground_truth_im = render_image_constants['filename_to_ground_truth_im']
     image_folder = render_image_constants['image_folder']
@@ -56,7 +62,7 @@ def render_image(im,render_image_constants):
     
     assert im['file'] in filename_to_ground_truth_im
     
-    output_file = image_to_output_file(im,preview_images_folder)
+    output_file = _image_to_output_file(im,preview_images_folder)
     if os.path.isfile(output_file) and not force_render_images:
         return output_file
     
@@ -105,9 +111,36 @@ def render_detection_confusion_matrix(ground_truth_file,
     confidence_thresholds and rendering_confidence_thresholds are dictionaries mapping
     class names to thresholds.  "default" is a special token that will be used for all
     classes not otherwise assigned thresholds.
+    
+    Args:
+        ground_truth_file (str): the CCT-formatted .json file with ground truth information
+        results_file (str): the MegaDetector results .json file
+        image_folder (str): the folder where images live; filenames in [ground_truth_file] and
+            [results_file] should be relative to this folder.
+        preview_folder (str): the output folder, i.e. the folder in which we'll create our nifty
+            HTML stuff.
+        force_rendering_images (bool, optional): if False, skips images that already exist
+        confidence_thresholds (dict, optional): a dictionary mapping class names to thresholds;
+            all classes not explicitly named here will use the threshold for the "default" category.
+        rendering_thresholds (dict, optional): a dictionary mapping class names to thresholds;
+            all classes not explicitly named here will use the threshold for the "default" category.
+        target_image_size (tuple, optional): output image size, as a pair of ints (width,height).  If one 
+            value is -1 and the other is not, aspect ratio is preserved.  If both are -1, the original image
+            sizes are preserved.
+        parallelize_rendering (bool, optional): enable (default) or disable parallelization when rendering
+        parallelize_rendering_n_core (int, optional): number of threads or processes to use for rendering, only
+            used if parallelize_rendering is True
+        parallelize_rendering_with_threads: whether to use threads (True) or processes (False) when rendering,
+            only used if parallelize_rendering is True
+        job_name (str, optional): job name to include in big letters in the output file
+        model_file (str, optional) model filename to include in HTML output
+        empty_category_name (str, optional): special category name that we should treat as empty, typically
+            "empty"
+        html_image_list_options (dict, optional): options listed passed along to write_html_image_list; 
+            see write_html_image_list for documentation.            
     """
     
-    #%% Argument and path handling
+    ##%% Argument and path handling
     
     preview_images_folder = os.path.join(preview_folder,'images')
     os.makedirs(preview_images_folder,exist_ok=True)
@@ -118,7 +151,7 @@ def render_detection_confusion_matrix(ground_truth_file,
         rendering_confidence_thresholds = {'default':0.4}
     
 
-    #%% Load ground truth 
+    ##%% Load ground truth 
         
     with open(ground_truth_file,'r') as f:
         ground_truth_data_cct = json.load(f)
@@ -129,14 +162,14 @@ def render_detection_confusion_matrix(ground_truth_file,
         filename_to_ground_truth_im[im['file_name']] = im
     
     
-    #%% Confirm that the ground truth images are present in the image folder
+    ##%% Confirm that the ground truth images are present in the image folder
     
     ground_truth_images = find_images(image_folder,return_relative_paths=True,recursive=True)
     assert len(ground_truth_images) == len(ground_truth_data_cct['images'])
     del ground_truth_images
     
     
-    #%% Map images to categories
+    ##%% Map images to categories
     
     # gt_image_id_to_image = {im['id']:im for im in ground_truth_data_cct['images']}
     gt_image_id_to_annotations = defaultdict(list)
@@ -175,7 +208,7 @@ def render_detection_confusion_matrix(ground_truth_file,
             'No ground truth category assigned to {}'.format(filename)
     
         
-    #%% Load results
+    ##%% Load results
     
     with open(results_file,'r') as f:
         md_formatted_results = json.load(f)
@@ -183,7 +216,7 @@ def render_detection_confusion_matrix(ground_truth_file,
     results_category_id_to_name = md_formatted_results['detection_categories']
     
     
-    #%% Render images with detections    
+    ##%% Render images with detections    
     
     render_image_constants = {}
     render_image_constants['filename_to_ground_truth_im'] = filename_to_ground_truth_im
@@ -211,7 +244,7 @@ def render_detection_confusion_matrix(ground_truth_file,
             print('Rendering images with {} {}'.format(parallelize_rendering_n_cores,
                                                        worker_string))
             
-        _ = list(tqdm(pool.imap(partial(render_image,render_image_constants=render_image_constants),
+        _ = list(tqdm(pool.imap(partial(_render_image,render_image_constants=render_image_constants),
                                 md_formatted_results['images']),
                                 total=len(md_formatted_results['images'])))        
     
@@ -219,10 +252,10 @@ def render_detection_confusion_matrix(ground_truth_file,
         
         # im = md_formatted_results['images'][0]
         for im in tqdm(md_formatted_results['images']):    
-            render_image(im,render_image_constants)
+            _render_image(im,render_image_constants)
     
     
-    #%% Map images to predicted categories, and vice-versa
+    ##%% Map images to predicted categories, and vice-versa
     
     filename_to_predicted_categories = defaultdict(set)
     predicted_category_name_to_filenames = defaultdict(set)
@@ -247,7 +280,7 @@ def render_detection_confusion_matrix(ground_truth_file,
     # ...for each image
     
     
-    #%% Create TP/TN/FP/FN lists
+    ##%% Create TP/TN/FP/FN lists
     
     category_name_to_image_lists = {}
     
@@ -301,7 +334,7 @@ def render_detection_confusion_matrix(ground_truth_file,
     # ...for each filename
     
     
-    #%% Create confusion matrix
+    ##%% Create confusion matrix
     
     gt_category_name_to_category_index = {}
     
@@ -383,7 +416,7 @@ def render_detection_confusion_matrix(ground_truth_file,
     # open_file(cm_figure_fn_abs)
     
     
-    #%% Create HTML confusion matrix
+    ##%% Create HTML confusion matrix
     
     html_confusion_matrix = '<table class="result-table">\n'
     html_confusion_matrix += '<tr>\n'
@@ -423,7 +456,7 @@ def render_detection_confusion_matrix(ground_truth_file,
                     title = '<b>Image</b>: {}, <b>Max conf</b>: {:0.3f}'.format(
                         image_filename_relative, max_conf)
                     image_link = 'images/' + os.path.basename(
-                        image_to_output_file(image_filename_relative,preview_images_folder))
+                        _image_to_output_file(image_filename_relative,preview_images_folder))
                     html_image_info = {
                         'filename': image_link,
                         'title': title,
@@ -527,7 +560,7 @@ def render_detection_confusion_matrix(ground_truth_file,
                     title = '<b>Image</b>: {}, <b>Max conf</b>: {:0.3f}'.format(
                         image_filename_relative, max_conf)
                     image_link = 'images/' + os.path.basename(
-                        image_to_output_file(image_filename_relative,preview_images_folder))
+                        _image_to_output_file(image_filename_relative,preview_images_folder))
                     html_image_info = {
                         'filename': image_link,
                         'title': title,
@@ -618,7 +651,7 @@ def render_detection_confusion_matrix(ground_truth_file,
         f.write(html)
     
     
-    #%% Prepare return data
+    ##%% Prepare return data
     
     confusion_matrix_info = {}
     confusion_matrix_info['html_file'] = target_html_file

api/batch_processing/postprocessing/repeat_detection_elimination/find_repeat_detections.py

@@ -1,24 +1,26 @@
-########
-#
-# find_repeat_detections.py
-#
-# If you want to use this script, we recommend that you read the user's guide:
-#
-# https://github.com/agentmorris/MegaDetector/tree/master/api/batch_processing/postprocessing/repeat_detection_elimination
-#
-# Really, don't try to run this script without reading the user's guide, you'll think 
-# it's more magical than it is. 
-#
-# This script looks through a sequence of detections in the API output json file, and finds 
-# candidates that might be "repeated false positives", i.e. that random branch that the 
-# detector thinks is an animal/person/vehicle.
-#
-# Typically after running this script, you would do a manual step to remove 
-# true positives, then run remove_repeat_detections to produce a final output file.
-#
-# There's no way that statement was self-explanatory; see the user's guide.
-#
-########
+r"""
+
+find_repeat_detections.py
+
+If you want to use this script, we recommend that you read the RDE user's guide:
+
+https://github.com/agentmorris/MegaDetector/tree/master/api/batch_processing/postprocessing/repeat_detection_elimination
+
+Really, don't try to run this script without reading the user's guide, you'll think 
+it's more magical than it is. 
+
+This script looks through a sequence of detections in the API output json file, and finds 
+candidates that might be "repeated false positives", i.e. that random branch that the 
+detector thinks is an animal/person/vehicle.
+
+Typically after running this script, you would do a manual step to remove 
+true positives, then run remove_repeat_detections to produce a final output file.
+
+There's no way that statement was self-explanatory; see the user's guide.
+
+This script is just a command-line driver for repeat_detections_core.py.
+
+"""
 
 #%% Constants and imports
 
@@ -51,7 +53,7 @@ if False:
     options.maxSuspiciousDetectionSize = 0.2
 
     options.filterFileToLoad = ''
-    options.filterFileToLoad = os.path.join(baseDir,'...\detectionIndex.json')
+    options.filterFileToLoad = os.path.join(baseDir,r'..\detectionIndex.json')
 
     options.debugMaxDir = -1
     options.debugMaxRenderDir = -1
@@ -75,15 +77,16 @@ def main():
     defaultOptions = repeat_detections_core.RepeatDetectionOptions()
 
     parser = argparse.ArgumentParser()
-    parser.add_argument('inputFile')
+    
+    parser.add_argument('inputFile', type=str, help='MD results .json file to process')
     
     parser.add_argument('--outputFile', action='store', type=str, default=None,
-                        help=".json file to write filtered results to... don't use this " + \
-                            "if you're going to do manual review of the repeat detection images")
+                        help='.json file to write filtered results to... do not use this if you are going to ' + \
+                             'do manual review of the repeat detection images (which you should)')
         
     parser.add_argument('--imageBase', action='store', type=str, default='',
-                        help='Image base dir, relevant if renderHtml is True or if " + \
-                            "omitFilteringFolder is not set')
+                        help='Image base dir, relevant if renderHtml is True or if ' + \
+                             '"omitFilteringFolder" is not set')
                             
     parser.add_argument('--outputBase', action='store', type=str, default='',
                         help='HTML or filtering folder output dir')
@@ -99,22 +102,22 @@ def main():
     parser.add_argument('--iouThreshold', action='store', type=float,
                         default=defaultOptions.iouThreshold,
                         help='Detections with IOUs greater than this are considered ' + \
-                            '"the same detection"')
+                             '"the same detection"')
         
     parser.add_argument('--occurrenceThreshold', action='store', type=int,
                         default=defaultOptions.occurrenceThreshold,
                         help='More than this many near-identical detections in a group ' + \
-                            '(e.g. a folder) is considered suspicious')
+                             '(e.g. a folder) is considered suspicious')
         
     parser.add_argument('--minSuspiciousDetectionSize', action='store', type=float,
                         default=defaultOptions.minSuspiciousDetectionSize,
                         help='Detections smaller than this fraction of image area are not ' + \
-                            'considered suspicious')
+                             'considered suspicious')
 
     parser.add_argument('--maxSuspiciousDetectionSize', action='store', type=float,
                         default=defaultOptions.maxSuspiciousDetectionSize,
                         help='Detections larger than this fraction of image area are not ' + \
-                            'considered suspicious')
+                             'considered suspicious')
 
     parser.add_argument('--maxImagesPerFolder', action='store', type=int,
                         default=defaultOptions.maxImagesPerFolder,
@@ -138,26 +141,32 @@ def main():
     
     parser.add_argument('--filterFileToLoad', action='store', type=str, default='',  
                         help='Path to detectionIndex.json, which should be inside a ' + \
-                            'folder of images that are manually verified to _not_ ' + \
-                            'contain valid animals')
+                             'folder of images that are manually verified to _not_ ' + \
+                             'contain valid animals')
 
     parser.add_argument('--omitFilteringFolder', action='store_false',
                         dest='bWriteFilteringFolder',
                         help='Should we create a folder of rendered detections for post-filtering?')
     
-    parser.add_argument('--debugMaxDir', action='store', type=int, default=-1)
-    parser.add_argument('--debugMaxRenderDir', action='store', type=int, default=-1)
-    parser.add_argument('--debugMaxRenderDetection', action='store', type=int, default=-1)
-    parser.add_argument('--debugMaxRenderInstance', action='store', type=int, default=-1)
+    parser.add_argument('--debugMaxDir', action='store', type=int, default=-1, 
+                        help='For debugging only, limit the number of directories we process')
+    parser.add_argument('--debugMaxRenderDir', action='store', type=int, default=-1,
+                        help='For debugging only, limit the number of directories we render')
+    parser.add_argument('--debugMaxRenderDetection', action='store', type=int, default=-1,
+                        help='For debugging only, limit the number of detections we process per folder')
+    parser.add_argument('--debugMaxRenderInstance', action='store', type=int, default=-1,
+                        help='For debugging only, limit the number of instances we process per detection')
 
     parser.add_argument('--forceSerialComparisons', action='store_false',
-                        dest='bParallelizeComparisons')
+                        dest='bParallelizeComparisons',
+                        help='Disable parallelization during the comparison stage')
     parser.add_argument('--forceSerialRendering', action='store_false',
-                        dest='bParallelizeRendering')
+                        dest='bParallelizeRendering',
+                        help='Disable parallelization during the rendering stage')
     
     parser.add_argument('--maxOutputImageWidth', action='store', type=int,
                         default=defaultOptions.maxOutputImageWidth,
-                        help='Maximum output size for thumbnail images')    
+                        help='Maximum output size for thumbnail images') 
     
     parser.add_argument('--lineThickness', action='store', type=int,
                         default=defaultOptions.lineThickness,
@@ -170,16 +179,17 @@ def main():
     parser.add_argument('--nDirLevelsFromLeaf', type=int,
                         default=defaultOptions.nDirLevelsFromLeaf,
                         help='Number of levels from the leaf folders to use for repeat ' + \
-                            'detection (0 == leaves)')
+                             'detection (0 == leaves)')
 
     parser.add_argument('--bRenderOtherDetections', action='store_true',
                         help='Show non-target detections in light gray on each image')
     
     parser.add_argument('--bRenderDetectionTiles', action='store_true',
-                        help='Should we render a grid showing every instance for each detection?')
+                        help='Should we render a grid showing every instance (up to a limit) for each detection?')
     
     parser.add_argument('--detectionTilesPrimaryImageWidth', type=int,
-                        default=defaultOptions.detectionTilesPrimaryImageWidth)                        
+                        default=defaultOptions.detectionTilesPrimaryImageWidth,
+                        help='The width of the main image when rendering images with detection tiles')
 
     parser.add_argument('--renderHtml', action='store_true',
                         dest='bRenderHtml', help='Should we render HTML output?')
@@ -197,6 +207,5 @@ def main():
 
     repeat_detections_core.find_repeat_detections(args.inputFile, args.outputFile, options)
 
-
 if __name__ == '__main__':
     main()

api/batch_processing/postprocessing/repeat_detection_elimination/remove_repeat_detections.py

@@ -1,15 +1,15 @@
-########
-#
-# remove_repeat_detections.py
-#
-# Used after running find_repeat_detections, then manually filtering the results,
-# to create a final filtered output file.
-#
-# If you want to use this script, we recommend that you read the user's guide:
-#
-# https://github.com/agentmorris/MegaDetector/tree/master/api/batch_processing/postprocessing/repeat_detection_elimination
-#
-########
+"""
+
+remove_repeat_detections.py
+
+Used after running find_repeat_detections, then manually filtering the results,
+to create a final filtered output file.
+
+If you want to use this script, we recommend that you read the RDE user's guide:
+
+https://github.com/agentmorris/MegaDetector/tree/master/api/batch_processing/postprocessing/repeat_detection_elimination
+
+"""
 
 #%% Constants and imports
 
@@ -21,7 +21,19 @@ from api.batch_processing.postprocessing.repeat_detection_elimination import rep
 #%% Main function
 
 def remove_repeat_detections(inputFile,outputFile,filteringDir):
-
+    """
+    Given an index file that was produced in a first pass through find_repeat_detections,
+    and a folder of images (from which the user has deleted images they don't want removed),
+    remove the identified repeat detections from a set of MD results and write to a new file.
+    
+    Args:
+        inputFile (str): .json file of MD results, from which we should remove repeat detections
+        outputFile (str): output .json file to which we should write MD results (with repeat 
+            detections removed)
+        filteringDir (str): the folder produced by find_repeat_detections, containing a 
+            detectionIndex.json file        
+    """
+    
     assert os.path.isfile(inputFile), "Can't find file {}".format(inputFile)
     assert os.path.isdir(filteringDir), "Can't find folder {}".format(filteringDir)
     options = repeat_detections_core.RepeatDetectionOptions()
@@ -66,6 +78,5 @@ def main():
     args = parser.parse_args()
     remove_repeat_detections(args.inputFile, args.outputFile, args.filteringDir)
 
-
 if __name__ == '__main__':
     main()