megadetector 5.0.8__py3-none-any.whl → 5.0.9__py3-none-any.whl

api/batch_processing/postprocessing/compare_batch_results.py

@@ -1,19 +1,19 @@
-########
-#
-# compare_batch_results.py
-# 
-# Compare sets of batch results; typically used to compare:
-#
-# * Results from different MegaDetector versions
-# * Results before/after RDE
-# * Results with/without augmentation
-#
-# Makes pairwise comparisons, but can take lists of results files (will perform 
-# all pairwise comparisons).  Results are written to an HTML page that shows the number
-# and nature of disagreements (in the sense of each image being a detection or non-detection), 
-# with sample images for each category.
-# 
-########
+"""
+
+compare_batch_results.py
+
+Compare sets of batch results; typically used to compare:
+
+* Results from different MegaDetector versions
+* Results before/after RDE
+* Results with/without augmentation
+
+Makes pairwise comparisons, but can take lists of results files (will perform 
+all pairwise comparisons).  Results are written to an HTML page that shows the number
+and nature of disagreements (in the sense of each image being a detection or non-detection), 
+with sample images for each category.
+
+"""
 
 #%% Imports
 
@@ -43,16 +43,28 @@ class PairwiseBatchComparisonOptions:
     pairwise options sets is stored in the BatchComparisonsOptions class.
     """
     
+    #: First filename to compare
     results_filename_a = None
+    
+    #: Second filename to compare
     results_filename_b = None
     
+    #: Description to use in the output HTML for filename A
     results_description_a = None
+    
+    #: Description to use in the output HTML for filename B
     results_description_b = None
     
+    #: Per-class detection thresholds to use for filename A (including a 'default' threshold)
     detection_thresholds_a = {'animal':0.15,'person':0.15,'vehicle':0.15,'default':0.15}
+    
+    #: Per-class detection thresholds to use for filename B (including a 'default' threshold)
     detection_thresholds_b = {'animal':0.15,'person':0.15,'vehicle':0.15,'default':0.15}
 
+    #: Rendering threshold to use for all categories for filename A
     rendering_confidence_threshold_a = 0.1
+    
+    #: Rendering threshold to use for all categories for filename B
     rendering_confidence_threshold_b = 0.1
 
 # ...class PairwiseBatchComparisonOptions
@@ -63,33 +75,56 @@ class BatchComparisonOptions:
     Defines the options for a set of (possibly many) pairwise comparisons.
     """
     
+    #: Folder to which we should write HTML output
     output_folder = None
+    
+    #: Base folder for images (which are specified as relative files)
     image_folder = None
+    
+    #: Job name to use in the HTML output file
     job_name = ''
     
+    #: Maximum number of images to render for each category, where a "category" here is
+    #: "detections_a_only", "detections_b_only", etc., or None to render all images.
     max_images_per_category = 1000
+    
+    #: Maximum number of images per HTML page (paginates if a category page goes beyond this),
+    #: or None to disable pagination.
     max_images_per_page = None
+    
+    #: Colormap to use for detections in file A (maps detection categories to colors)
     colormap_a = ['Red']
+    
+    #: Colormap to use for detections in file B (maps detection categories to colors)
     colormap_b = ['RoyalBlue']
 
-    # Process-based parallelization isn't supported yet; this must be "True"
+    #: Process-based parallelization isn't supported yet; this must be "True"
     parallelize_rendering_with_threads = True
     
-    # List of filenames to include in the comparison, or None to use all files
+    #: List of filenames to include in the comparison, or None to use all files
     filenames_to_include = None
     
-    # Compare only detections/non-detections, ignore categories (still renders categories)
+    #: Compare only detections/non-detections, ignore categories (still renders categories)
     class_agnostic_comparison = False
     
+    #: Width of images to render in the output HTML
     target_width = 800
+    
+    #: Number of workers to use for rendering, or <=1 to disable parallelization
     n_rendering_workers = 20
+    
+    #: Random seed for image sampling (not used if max_images_per_category is None)
     random_seed = 0
     
-    # Default to sorting by filename
+    #: Whether to sort results by confidence; if this is False, sorts by filename
     sort_by_confidence = False
     
+    #: The expectation is that all results sets being compared will refer to the same images; if this
+    #: is True (default), we'll error if that's not the case, otherwise non-matching lists will just be
+    #: a warning.
     error_on_non_matching_lists = True
     
+    #: List of PairwiseBatchComparisonOptions that defines the comparisons we'll render.
     pairwise_options = []
     
 # ...class BatchComparisonOptions
@@ -100,18 +135,21 @@ class PairwiseBatchComparisonResults:
     The results from a single pairwise comparison.
     """
     
+    #: String of HTML content suitable for rendering to an HTML file
     html_content = None
+    
+    #: Possibly-modified version of the PairwiseBatchComparisonOptions supplied as input.
     pairwise_options = None
     
-    # A dictionary with keys including:
-    #
-    # common_detections
-    # common_non_detections
-    # detections_a_only
-    # detections_b_only
-    # class_transitions
+    #: A dictionary with keys including:
+    #:
+    #: common_detections
+    #: common_non_detections
+    #: detections_a_only
+    #: detections_b_only
+    #: class_transitions
     #
-    # Each of these maps a filename to a two-element list (the image in set A, the image in set B).
+    #: Each of these maps a filename to a two-element list (the image in set A, the image in set B).
     categories_to_image_pairs = None
 
 # ...class PairwiseBatchComparisonResults
@@ -122,9 +160,10 @@ class BatchComparisonResults:
     The results from a set of pairwise comparisons
     """
     
+    #: Filename containing HTML output
     html_output_file = None
     
-    # An list of PairwiseBatchComparisonResults
+    #: A list of PairwiseBatchComparisonResults
     pairwise_results = None
     
 # ...class BatchComparisonResults    
@@ -144,9 +183,20 @@ main_page_footer = '<br/><br/><br/></body></html>\n'
 
 #%% Comparison functions
 
-def render_image_pair(fn,image_pairs,category_folder,options,pairwise_options):
+def _render_image_pair(fn,image_pairs,category_folder,options,pairwise_options):
     """
     Render two sets of results (i.e., a comparison) for a single image.
+    
+    Args:
+        fn (str): image filename
+        image_pairs (dict): dict mapping filenames to pairs of image dicts
+        category_folder (str): folder to which to render this image, typically 
+            "detections_a_only", "detections_b_only", etc.
+        options (BatchComparisonOptions): job options
+        pairwise_options (PairwiseBatchComparisonOptions): pairwise comparison options
+            
+    Returns:
+        str: rendered image filename            
     """
     
     input_image_path = os.path.join(options.image_folder,fn)
@@ -194,20 +244,22 @@ def render_image_pair(fn,image_pairs,category_folder,options,pairwise_options):
     im.save(output_image_path)
     return output_image_path
 
-# ...def render_image_pair()
+# ...def _render_image_pair()
 
 
-def pairwise_compare_batch_results(options,output_index,pairwise_options):
+def _pairwise_compare_batch_results(options,output_index,pairwise_options):
     """
     The main entry point for this module is compare_batch_results(), which calls 
     this function for each pair of comparisons the caller has requested.  Generates an
     HTML page for this comparison.  Returns a BatchComparisonResults object.
     
-    options: an instance of BatchComparisonOptions
-    
-    output_index: a numeric index used for generating HTML titles
-    
-    pairwise_options: an instance of PairwiseBatchComparisonOptions    
+    Args:
+        options (BatchComparisonOptions): overall job options for this comparison group
+        output_index (int): a numeric index used for generating HTML titles    
+        pairwise_options (PairwiseBatchComparisonOptions): job options for this comparison
+        
+    Returns:
+        PairwiseBatchComparisonResults: the results of this pairwise comparison
     """
     
     # pairwise_options is passed as a parameter here, and should not be specified
@@ -463,11 +515,11 @@ def pairwise_compare_batch_results(options,output_index,pairwise_options):
         if options.n_rendering_workers <= 1:
             output_image_paths = []
             for fn in tqdm(image_filenames):        
-                output_image_paths.append(render_image_pair(fn,image_pairs,category_folder,
+                output_image_paths.append(_render_image_pair(fn,image_pairs,category_folder,
                                                             options,pairwise_options))
         else:            
             output_image_paths = list(tqdm(pool.imap(
-                partial(render_image_pair, image_pairs=image_pairs, 
+                partial(_render_image_pair, image_pairs=image_pairs, 
                         category_folder=category_folder,options=options,
                         pairwise_options=pairwise_options),
                 image_filenames), 
@@ -644,14 +696,20 @@ def pairwise_compare_batch_results(options,output_index,pairwise_options):
             
     return pairwise_results
         
-# ...def pairwise_compare_batch_results()
+# ...def _pairwise_compare_batch_results()
 
 
 def compare_batch_results(options):
     """
     The main entry point for this module.  Runs one or more batch results comparisons, 
-    writing results to an html page.  Most of the work is deferred to
-    pairwise_compare_batch_results().
+    writing results to an html page.  Most of the work is deferred to _pairwise_compare_batch_results().
+    
+    Args:
+        options (BatchComparisonOptions): job options to use for this comparison task, including the
+            list of specific pairswise comparisons to make (in the pairwise_options field)
+            
+    Returns:
+        BatchComparisonResults: the results of this comparison task
     """
     
     assert options.output_folder is not None
@@ -675,7 +733,7 @@ def compare_batch_results(options):
     for i_comparison,pairwise_options in enumerate(pairwise_options_list):
         print('Running comparison {} of {}'.format(i_comparison,n_comparisons))
         pairwise_results = \
-            pairwise_compare_batch_results(options,i_comparison,pairwise_options)
+            _pairwise_compare_batch_results(options,i_comparison,pairwise_options)
         html_content += pairwise_results.html_content
         all_pairwise_results.append(pairwise_results)
 
@@ -702,6 +760,18 @@ def n_way_comparison(filenames,options,detection_thresholds=None,rendering_thres
     """
     Performs N pairwise comparisons for the list of results files in [filenames], by generating
     sets of pairwise options and calling compare_batch_results.
+    
+    Args:
+        filenames (list): list of MD results filenames to compare
+        options (BatchComparisonOptions): task options set in which pairwise_options is still 
+            empty; that will get populated from [filenames]
+        detection_thresholds (list, optional): list of detection thresholds with the same length
+            as [filenames], or None to use sensible defaults
+        rendering_thresholds (list, optional): list of rendering thresholds with the same length
+            as [filenames], or None to use sensible defaults
+    
+    Returns:
+        BatchComparisonResults: the results of this comparison task
     """
     
     if detection_thresholds is None:

api/batch_processing/postprocessing/convert_output_format.py

@@ -1,13 +1,15 @@
-########
-#
-# convert_output_format.py
-#
-# Converts between file formats output by our batch processing API.  Currently
-# supports json <--> csv conversion, but this should be the landing place for any
-# conversion - including between hypothetical alternative .json versions - that we support 
-# in the future.
-#
-########
+"""
+
+convert_output_format.py
+
+Converts between file formats output by our batch processing API.  Currently
+supports json <--> csv conversion, but this should be the landing place for any
+conversion - including between hypothetical alternative .json versions - that we support 
+in the future.
+
+The .csv format is largely obsolete, don't use it unless you're super-duper sure you need it.
+
+"""
 
 #%% Constants and imports
 
@@ -33,13 +35,27 @@ def convert_json_to_csv(input_path,output_path=None,min_confidence=None,
                         omit_bounding_boxes=False,output_encoding=None,
                         overwrite=True):
     """
-    Convert .json to .csv
+    Converts a MD results .json file to a totally non-standard .csv format.
     
-    If output_path is None, will convert x.json to x.csv.
+    If [output_path] is None, will convert x.json to x.csv.
     
     TODO: this function should obviously be using Pandas or some other sensible structured
     representation of tabular data.  Even a list of dicts.  This implementation is quite
     brittle and depends on adding fields to every row in exactly the right order.
+    
+    Args:
+        input_path (str): the input .json file to convert
+        output_path (str, optional): the output .csv file to generate; if this is None, uses 
+            [input_path].csv
+        min_confidence (float, optional): the minimum-confidence detection we should include 
+            in the "detections" column; has no impact on the other columns
+        omit_bounding_boxes (bool): whether to leave out the json-formatted bounding boxes
+            that make up the "detections" column, which are not generally useful for someone who
+            wants to consume this data as a .csv file
+        output_encoding (str, optional): encoding to use for the .csv file
+        overwrite (bool): whether to overwrite an existing .csv file; if this is False and the
+            output file exists, no-ops and returns
+           
     """
     
     if output_path is None:
@@ -58,11 +74,12 @@ def convert_json_to_csv(input_path,output_path=None,min_confidence=None,
     
     # We add an output column for each class other than 'empty', 
     # containing the maximum probability of  that class for each image
-    n_non_empty_detection_categories = len(annotation_constants.annotation_bbox_categories) - 1
+    # n_non_empty_detection_categories = len(annotation_constants.annotation_bbox_categories) - 1
+    n_non_empty_detection_categories = annotation_constants.NUM_DETECTOR_CATEGORIES
     detection_category_column_names = []
-    assert annotation_constants.annotation_bbox_category_id_to_name[0] == 'empty'
+    assert annotation_constants.detector_bbox_categories[0] == 'empty'
     for cat_id in range(1,n_non_empty_detection_categories+1):
-        cat_name = annotation_constants.annotation_bbox_category_id_to_name[cat_id]
+        cat_name = annotation_constants.detector_bbox_categories[cat_id]
         detection_category_column_names.append('max_conf_' + cat_name)
     
     n_classification_categories = 0
@@ -206,6 +223,14 @@ def convert_json_to_csv(input_path,output_path=None,min_confidence=None,
 def convert_csv_to_json(input_path,output_path=None,overwrite=True):
     """
     Convert .csv to .json.  If output_path is None, will convert x.csv to x.json.
+    
+    Args:
+        input_path (str): .csv filename to convert to .json
+        output_path (str, optional): the output .json file to generate; if this is None, uses 
+            [input_path].json
+        overwrite (bool): whether to overwrite an existing .json file; if this is False and the
+            output file exists, no-ops and returns    
+        
     """
     
     if output_path is None:
@@ -231,7 +256,7 @@ def convert_csv_to_json(input_path,output_path=None,overwrite=True):
     }
     
     classification_categories = {}
-    detection_categories = annotation_constants.annotation_bbox_category_id_to_name
+    detection_categories = annotation_constants.detector_bbox_categories
 
     images = []
     

api/batch_processing/postprocessing/load_api_results.py

@@ -1,17 +1,17 @@
-########
-#
-# load_api_results.py
-#
-# DEPRECATED
-#
-# As of 2023.12, this module is used in postprocessing and RDE.  Not recommended
-# for new code.
-#
-# Loads the output of the batch processing API (json) into a Pandas dataframe.
-#
-# Includes functions to read/write the (very very old) .csv results format.
-#
-########
+"""
+
+load_api_results.py
+
+DEPRECATED
+
+As of 2023.12, this module is used in postprocessing and RDE.  Not recommended
+for new code.
+
+Loads the output of the batch processing API (json) into a Pandas dataframe.
+
+Includes functions to read/write the (very very old) .csv results format.
+
+"""
 
 #%% Imports
 
@@ -31,7 +31,7 @@ def load_api_results(api_output_path: str, normalize_paths: bool = True,
                      filename_replacements: Optional[Mapping[str, str]] = None,
                      force_forward_slashes: bool = True
                      ) -> Tuple[pd.DataFrame, Dict]:
-    """
+    r"""
     Loads json-formatted MegaDetector results to a Pandas DataFrame.
 
     Args:
@@ -44,8 +44,7 @@ def load_api_results(api_output_path: str, normalize_paths: bool = True,
             in filenames
 
     Returns:
-        detection_results: pd.DataFrame, contains at least the columns:
-                ['file', 'detections','failure']
+        detection_results: pd.DataFrame, contains at least the columns ['file', 'detections','failure']
         other_fields: a dict containing fields in the results other than 'images'
     """
     

api/batch_processing/postprocessing/md_to_coco.py

@@ -1,13 +1,13 @@
-########
-#
-# md_to_coco.py
-#
-# "Converts" MegaDetector output files to COCO format.  "Converts" is in quotes because
-# this is an opinionated transformation that requires a confidence threshold.
-#
-# Does not currently handle classification information.
-#
-########
+"""
+
+md_to_coco.py
+
+"Converts" MegaDetector output files to COCO format.  "Converts" is in quotes because
+this is an opinionated transformation that requires a confidence threshold.
+
+Does not currently handle classification information.
+
+"""
 
 #%% Constants and imports
 
@@ -38,18 +38,28 @@ def md_to_coco(md_results_file,
     
     A folder of images is required if width and height information are not available 
     in the MD results file.
+
+    Args:
+        md_results_file (str): MD results .json file to convert to COCO format
+        coco_output_file (str, optional): COCO .json file to write; if this is None, we'll return 
+            a COCO-formatted dict, but won't write it to disk
+        image_folder (str, optional): folder of images, required if 'width' and 'height' are not
+            present in the MD results file (they are not required by the format)
+        confidence_threshold (float, optional): boxes below this confidence threshold will not be
+            included in the output data
+        validate_image_sizes (bool, optional): if this is True, we'll check the image sizes 
+            regardless of whether "width" and "height" are present in the MD results file.
+        info (dict, optional): arbitrary metadata to include in an "info" field in the COCO-formatted
+            output
+        preserve_nonstandard_metadata (bool, optional): if this is True, confidence will be preserved in a 
+            non-standard "conf" field in each annotation, and any random fields present in each image's data
+            (e.g. EXIF metadata) will be propagated to COCO output    
+        include_failed_images (boo, optional): if this is True, failed images will be propagated to COCO output 
+            with a non-empty "failure" field and no other fields, otherwise failed images will be skipped.
     
-    If validate_image_sizes is True, we'll check the image sizes regardless of whether width
-    and height are present in the MD results file.
-    
-    If preserve_nonstandard_metadata is True, confidence will be preserved in a non-standard 
-    "conf" field in each annotation, and any random fields present in each image's data (e.g.
-    EXIF metadata) will be propagated to COCO output.
-    
-    If include_failed_images is True, failed images will be propagated to COCO output with
-    a non-empty "failure" field and no other fields, otherwise failed images will be skipped.
-    
-    Returns the COCO json dict.
+    Returns:
+        dict: the COCO data dict, identical to what's written to [coco_output_file] if [coco_output_file]
+        is not None.
     """
 
     with open(md_results_file,'r') as f:    

api/batch_processing/postprocessing/md_to_labelme.py

@@ -1,17 +1,17 @@
-########
-#
-# md_to_labelme.py
-#
-# "Converts" a MegaDetector output .json file to labelme format (one .json per image
-# file).  "Convert" is in quotes because this is an opinionated transformation that 
-# requires a confidence threshold.
-#
-# TODO:
-#    
-# * support variable confidence thresholds across classes
-# * support classification data
-#
-########
+"""
+
+md_to_labelme.py
+
+"Converts" a MegaDetector output .json file to labelme format (one .json per image
+file).  "Convert" is in quotes because this is an opinionated transformation that 
+requires a confidence threshold.
+
+TODO:
+   
+* support variable confidence thresholds across classes
+* support classification data
+
+"""
 
 #%% Imports and constants
 
@@ -26,6 +26,7 @@ from functools import partial
 
 from md_visualization.visualization_utils import open_image
 from md_utils.ct_utils import truncate_float
+from detection.run_detector import DEFAULT_DETECTOR_LABEL_MAP
 
 output_precision = 3
 default_confidence_threshold = 0.15
@@ -33,18 +34,33 @@ default_confidence_threshold = 0.15
 
 #%% Functions
 
-def get_labelme_dict_for_image(im,image_base_name,category_id_to_name,
+def get_labelme_dict_for_image(im,image_base_name=None,category_id_to_name=None,
                                info=None,confidence_threshold=None):
     """
     For the given image struct in MD results format, reformat the detections into
-    labelme format.  Returns a dict.
+    labelme format.
     
-    'height' and 'width' are required in [im].
+    Args:
+        im (dict): MegaDetector-formatted results dict, must include 'height' and 'width' fields
+        image_base_name (str, optional): written directly to the 'imagePath' field in the output; 
+            defaults to os.path.basename(im['file']).
+        category_id_to_name (dict, optional): maps string-int category IDs to category names, defaults
+            to the standard MD categories
+        info (dict, optional): arbitrary metadata to write to the "detector_info" field in the output
+            dict
+        confidence_threshold (float, optional): only detections at or above this confidence threshold
+            will be included in the output dict
     
-    image_base_name is written directly to the 'imagePath' field in the output; it should generally be
-    os.path.basename(your_image_file).
+    Return:
+        dict: labelme-formatted dictionary, suitable for writing directly to a labelme-formatted .json file
     """
     
+    if image_base_name is None:
+        image_base_name = os.path.basename(im['file'])
+        
+    if category_id_to_name:
+        category_id_to_name = DEFAULT_DETECTOR_LABEL_MAP
+        
     if confidence_threshold is None:        
         confidence_threshold = -1.0
      
@@ -130,7 +146,22 @@ def md_to_labelme(results_file,image_base,confidence_threshold=None,
     For all the images in [results_file], write a .json file in labelme format alongside the
     corresponding relative path within image_base.
     
-    If non-empty, "extension_prefix" will be inserted before the .json extension.
+    Args:
+        results_file (str): MD results .json file to convert to Labelme format
+        image_base (str): folder of images; filenames in [results_file] should be relative to
+            this folder
+        confidence_threshold (float, optional): only detections at or above this confidence threshold
+            will be included in the output dict
+        overwrite (bool, optional): whether to overwrite existing output files; if this is False
+            and the output file for an image exists, we'll skip that image
+        extension_prefix (str, optional): if non-empty, "extension_prefix" will be inserted before the .json 
+            extension
+        n_workers (int, optional): enables multiprocessing if > 1
+        use_threads (bool, optional): if [n_workers] > 1, determines whether we parallelize via threads (True)
+            or processes (False)
+        bypass_image_size_read (bool, optional): if True, skips reading image sizes and trusts whatever is in
+            the MD results file (don't set this to "True" if your MD results file doesn't contain image sizes)
+        verbose (bool, optional): enables additionald ebug output    
     """
     
     if extension_prefix is None:
@@ -294,7 +325,6 @@ def main():
     args = parser.parse_args()
 
     md_to_labelme(args.results_file,args.image_base,args.confidence_threshold,args.overwrite)
-    
-    
+
 if __name__ == '__main__':
     main()

api/batch_processing/postprocessing/merge_detections.py

@@ -1,17 +1,17 @@
-########
-#
-# merge_detections.py
-#
-# Merge high-confidence detections from one or more results files into another 
-# file.  Typically used to combine results from MDv5b and/or MDv4 into a "primary"
-# results file from MDv5a.
-#
-# Detection categories must be the same in both files; if you want to first remap
-# one file's category mapping to be the same as another's, see remap_detection_categories.
-#
-# If you want to literally merge two .json files, see combine_api_outputs.py.
-#
-########
+"""
+
+merge_detections.py
+
+Merge high-confidence detections from one or more results files into another 
+file.  Typically used to combine results from MDv5b and/or MDv4 into a "primary"
+results file from MDv5a.
+
+Detection categories must be the same in both files; if you want to first remap
+one file's category mapping to be the same as another's, see remap_detection_categories.
+
+If you want to literally merge two .json files, see combine_api_outputs.py.
+
+"""
 
 #%% Constants and imports