megadetector 5.0.9__py3-none-any.whl → 5.0.11__py3-none-any.whl

api/batch_processing/postprocessing/postprocess_batch_results.py

@@ -1,1904 +0,0 @@
-"""
-
-postprocess_batch_results.py
-
-Given a .json or .csv file containing MD results, do one or more of the following:
-
-* Sample detections/non-detections and render to HTML (when ground truth isn't
-  available) (this is 99.9% of what this module is for)
-* Evaluate detector precision/recall, optionally rendering results (requires
-  ground truth)
-* Sample true/false positives/negatives and render to HTML (requires ground
-  truth)
-
-Ground truth, if available, must be in COCO Camera Traps format:
-    
-https://github.com/agentmorris/MegaDetector/blob/main/data_management/README.md#coco-camera-traps-format
-
-"""
-
-#%% Constants and imports
-
-import argparse
-import collections
-import copy
-import errno
-import io
-import os
-import sys
-import time
-import uuid
-import warnings
-import random
-
-from enum import IntEnum
-from multiprocessing.pool import ThreadPool
-from multiprocessing.pool import Pool
-from functools import partial
-
-import matplotlib.pyplot as plt
-import numpy as np
-import humanfriendly
-import pandas as pd
-
-from sklearn.metrics import precision_recall_curve, confusion_matrix, average_precision_score
-from tqdm import tqdm
-
-import md_visualization.visualization_utils as vis_utils
-import md_visualization.plot_utils as plot_utils
-
-from md_utils.write_html_image_list import write_html_image_list
-from md_utils import path_utils
-from data_management.cct_json_utils import (CameraTrapJsonUtils, IndexedJsonDb)
-from api.batch_processing.postprocessing.load_api_results import load_api_results
-from md_utils.ct_utils import args_to_object, sets_overlap
-
-from detection.run_detector import get_typical_confidence_threshold_from_results
-
-warnings.filterwarnings('ignore', '(Possibly )?corrupt EXIF data', UserWarning)
-
-
-#%% Options
-
-DEFAULT_NEGATIVE_CLASSES = ['empty']
-DEFAULT_UNKNOWN_CLASSES = ['unknown', 'unlabeled', 'ambiguous']
-
-# Make sure there is no overlap between the two sets, because this will cause
-# issues in the code
-assert not sets_overlap(DEFAULT_NEGATIVE_CLASSES, DEFAULT_UNKNOWN_CLASSES), (
-        'Default negative and unknown classes cannot overlap.')
-
-
-class PostProcessingOptions:
-    """
-    Options used to parameterize process_batch_results().
-    """
-    
-    ### Required inputs
-
-    #: MD results .json file to process
-    md_results_file = ''
-    
-    #: Folder to which we should write HTML output
-    output_dir = ''
-
-    ### Options
-
-    #: Folder where images live (filenames in [md_results_file] should be relative to this folder)
-    image_base_dir = '.'
-
-    ## These apply only when we're doing ground-truth comparisons
-    
-    #: Optional .json file containing ground truth information
-    ground_truth_json_file = ''
-
-    #: Classes we'll treat as negative
-    #:
-    #: Include the token "#NO_LABELS#" to indicate that an image with no annotations
-    #: should be considered empty.
-    negative_classes = DEFAULT_NEGATIVE_CLASSES
-    
-    #: Classes we'll treat as neither positive nor negative
-    unlabeled_classes = DEFAULT_UNKNOWN_CLASSES
-
-    #: A list of output sets that we should count, but not render images for.
-    #:
-    #: Typically used to preview sets with lots of empties, where you don't want to
-    #: subset but also don't want to render 100,000 empty images.
-    #:
-    #: detections, non_detections
-    #: detections_animal, detections_person, detections_vehicle
-    rendering_bypass_sets = []
-
-    #: If this is None, choose a confidence threshold based on the detector version.
-    #:
-    #: This can either be a float or a dictionary mapping category names (not IDs) to 
-    #: thresholds.  The category "default" can be used to specify thresholds for 
-    #: other categories.  Currently the use of a dict here is not supported when 
-    #: ground truth is supplied.
-    confidence_threshold = None
-    
-    #: Confidence threshold to apply to classification (not detection) results
-    #:
-    #: Only a float is supported here (unlike the "confidence_threshold" parameter, which
-    #: can be a dict).    
-    classification_confidence_threshold = 0.5
-
-    #: Used for summary statistics only
-    target_recall = 0.9
-
-    #: Number of images to sample, -1 for "all images"
-    num_images_to_sample = 500
-
-    #: Random seed for sampling, or None
-    sample_seed = 0 # None
-
-    #: Image width for images in the HTML output
-    viz_target_width = 800
-
-    #: Line width (in pixels) for rendering detections
-    line_thickness = 4
-    
-    #: Box expansion (in pixels) for rendering detections
-    box_expansion = 0
-
-    #: Job name to include in big letters in the output HTML
-    job_name_string = None
-    
-    #: Model version string to include in the output HTML
-    model_version_string = None
-    
-    #: Sort order for the output, should be one of "filename", "confidence", or "random"
-    html_sort_order = 'filename'
-   
-    #: If True, images in the output HTML will be links back to the original images
-    link_images_to_originals = True
-    
-    #: Optionally separate detections into categories (animal/vehicle/human)
-    #: 
-    #: Currently only supported when ground truth is unavailable
-    separate_detections_by_category = True
-
-    #: Optionally replace one or more strings in filenames with other strings;
-    #: useful for taking a set of results generated for one folder structure
-    #: and applying them to a slightly different folder structure.
-    api_output_filename_replacements = {}
-    
-    #: Optionally replace one or more strings in filenames with other strings;
-    #: useful for taking a set of results generated for one folder structure
-    #: and applying them to a slightly different folder structure.
-    ground_truth_filename_replacements = {}
-
-    #: Allow bypassing API output loading when operating on previously-loaded
-    #: results.  If present, this is a Pandas DataFrame.  Almost never useful.
-    api_detection_results = None
-    
-    #: Allow bypassing API output loading when operating on previously-loaded
-    #: results.  If present, this is a str --> obj dict.  Almost never useful.
-    api_other_fields = None
-
-    #: Should we also split out a separate report about the detections that were
-    #: just below our main confidence threshold?
-    #:
-    #: Currently only supported when ground truth is unavailable.
-    include_almost_detections = False
-    
-    #: Only a float is supported here (unlike the "confidence_threshold" parameter, which
-    #: can be a dict).
-    almost_detection_confidence_threshold = None
-
-    #: Enable/disable rendering parallelization    
-    parallelize_rendering = False
-    
-    #: Number of threads/processes to use for rendering parallelization
-    parallelize_rendering_n_cores = 25
-    
-    #: Whether to use threads (True) or processes (False) for rendering parallelization
-    parallelize_rendering_with_threads = True
-    
-    #: When classification results are present, should be sort alphabetically by class name (False)
-    #: or in descending order by frequency (True)?
-    sort_classification_results_by_count = False    
-    
-    #: Should we split individual pages up into smaller pages if there are more than
-    #: N images?
-    max_figures_per_html_file = None
-    
-# ...PostProcessingOptions
-
-
-class PostProcessingResults:
-    """
-    Return format from process_batch_results
-    """
-    
-    #: HTML file to which preview information was written
-    output_html_file = ''
-    
-    #: Pandas Dataframe containing detection results
-    api_detection_results = None
-    
-    #: str --> obj dictionary containing other information loaded from the results file
-    api_other_fields = None
-
-
-##%% Helper classes and functions
-
-class DetectionStatus(IntEnum):
-    """
-    Flags used to mark images as positive or negative for P/R analysis
-    (according to ground truth and/or detector output)
-        
-    :meta private:
-    """
-    
-    DS_NEGATIVE = 0
-    DS_POSITIVE = 1
-
-    # Anything greater than this isn't clearly positive or negative
-    DS_MAX_DEFINITIVE_VALUE = DS_POSITIVE
-
-    # image has annotations suggesting both negative and positive
-    DS_AMBIGUOUS = 2
-
-    # image is not annotated or is annotated with 'unknown', 'unlabeled', ETC.
-    DS_UNKNOWN = 3
-
-    # image has not yet been assigned a state
-    DS_UNASSIGNED = 4
-
-    # In some analyses, we add an additional class that lets us look at
-    # detections just below our main confidence threshold
-    DS_ALMOST = 5
-
-
-def _mark_detection_status(indexed_db,
-                           negative_classes=DEFAULT_NEGATIVE_CLASSES,
-                           unknown_classes=DEFAULT_UNKNOWN_CLASSES):
-    """
-    For each image in indexed_db.db['images'], add a '_detection_status' field
-    to indicate whether to treat this image as positive, negative, ambiguous,
-    or unknown.
-
-    Makes modifications in-place.
-
-    returns (n_negative, n_positive, n_unknown, n_ambiguous)
-    """
-    
-    negative_classes = set(negative_classes)
-    unknown_classes = set(unknown_classes)
-
-    # count the # of images with each type of DetectionStatus
-    n_unknown = 0
-    n_ambiguous = 0
-    n_positive = 0
-    n_negative = 0
-
-    print('Preparing ground-truth annotations')
-    for im in tqdm(indexed_db.db['images']):
-
-        image_id = im['id']
-        annotations = indexed_db.image_id_to_annotations[image_id]
-        categories = [ann['category_id'] for ann in annotations]
-        category_names = set(indexed_db.cat_id_to_name[cat] for cat in categories)
-
-        # Check whether this image has:
-        # - unknown / unassigned-type labels
-        # - negative-type labels
-        # - positive labels (i.e., labels that are neither unknown nor negative)
-        has_unknown_labels = sets_overlap(category_names, unknown_classes)
-        has_negative_labels = sets_overlap(category_names, negative_classes)
-        has_positive_labels = 0 < len(category_names - (unknown_classes | negative_classes))
-        # assert has_unknown_labels is False, '{} has unknown labels'.format(annotations)
-
-        # If there are no image annotations...
-        if len(categories) == 0:
-                        
-            if '#NO_LABELS#' in negative_classes:
-                n_negative += 1
-                im['_detection_status'] = DetectionStatus.DS_NEGATIVE
-            else:
-                n_unknown += 1
-                im['_detection_status'] = DetectionStatus.DS_UNKNOWN
-
-            # n_negative += 1
-            # im['_detection_status'] = DetectionStatus.DS_NEGATIVE
-
-        # If the image has more than one type of labels, it's ambiguous
-        # note: bools are automatically converted to 0/1, so we can sum
-        elif (has_unknown_labels + has_negative_labels + has_positive_labels) > 1:
-            n_ambiguous += 1
-            im['_detection_status'] = DetectionStatus.DS_AMBIGUOUS
-
-        # After the check above, we can be sure it's only one of positive,
-        # negative, or unknown.
-        #
-        # Important: do not merge the following 'unknown' branch with the first
-        # 'unknown' branch above, where we tested 'if len(categories) == 0'
-        #
-        # If the image has only unknown labels
-        elif has_unknown_labels:
-            n_unknown += 1
-            im['_detection_status'] = DetectionStatus.DS_UNKNOWN
-
-        # If the image has only negative labels
-        elif has_negative_labels:
-            n_negative += 1
-            im['_detection_status'] = DetectionStatus.DS_NEGATIVE
-
-        # If the images has only positive labels
-        elif has_positive_labels:
-            n_positive += 1
-            im['_detection_status'] = DetectionStatus.DS_POSITIVE
-
-            # Annotate the category, if it is unambiguous
-            if len(category_names) == 1:
-                im['_unambiguous_category'] = list(category_names)[0]
-
-        else:
-            raise Exception('Invalid detection state')
-
-    # ...for each image
-
-    return n_negative, n_positive, n_unknown, n_ambiguous
-
-# ..._mark_detection_status()
-
-
-def is_sas_url(s) -> bool:
-    """
-    Placeholder for a more robust way to verify that a link is a SAS URL.
-    99.999% of the time this will suffice for what we're using it for right now.
-    
-    :meta private:
-    """
-    
-    return (s.startswith(('http://', 'https://')) and ('core.windows.net' in s)
-            and ('?' in s))
-
-
-def relative_sas_url(folder_url, relative_path):
-    """
-    Given a container-level or folder-level SAS URL, create a SAS URL to the
-    specified relative path.
-    
-    :meta private:
-    """
-    
-    relative_path = relative_path.replace('%','%25')
-    relative_path = relative_path.replace('#','%23')
-    relative_path = relative_path.replace(' ','%20')
-
-    if not is_sas_url(folder_url):
-        return None
-    tokens = folder_url.split('?')
-    assert len(tokens) == 2
-    if not tokens[0].endswith('/'):
-        tokens[0] = tokens[0] + '/'
-    if relative_path.startswith('/'):
-        relative_path = relative_path[1:]
-    return tokens[0] + relative_path + '?' + tokens[1]
-
-
-def _render_bounding_boxes(
-        image_base_dir,
-        image_relative_path,
-        display_name,
-        detections,
-        res,
-        ground_truth_boxes=None,
-        detection_categories=None,
-        classification_categories=None,
-        options=None):
-    """
-    Renders detection bounding boxes on a single image.
-    
-    This is an internal function; if you want tools for rendering boxes on images, see 
-    md_visualization.visualization_utils.
-
-    The source image is:
-
-        image_base_dir / image_relative_path
-
-    The target image is, for example:
-
-        [options.output_dir] / 
-        ['detections' or 'non_detections'] / 
-        [filename with slashes turned into tildes]
-
-    "res" is a result type, e.g. "detections", "non-detections"; this determines the
-    output folder for the rendered image.
-    
-    Only very preliminary support is provided for ground truth box rendering.
-    
-    Returns the html info struct for this image in the format that's used for
-    write_html_image_list.
-    
-    :meta private:
-    """
-
-    if options is None:
-        options = PostProcessingOptions()
-
-    # Leaving code in place for reading from blob storage, may support this
-    # in the future.
-    """
-    stream = io.BytesIO()
-    _ = blob_service.get_blob_to_stream(container_name, image_id, stream)
-    # resize is to display them in this notebook or in the HTML more quickly
-    image = Image.open(stream).resize(viz_size)
-    """
-
-    image_full_path = None
-    
-    if res in options.rendering_bypass_sets:
-
-        sample_name = res + '_' + path_utils.flatten_path(image_relative_path)
-
-    else:
-
-        if is_sas_url(image_base_dir):
-            image_full_path = relative_sas_url(image_base_dir, image_relative_path)
-        else:
-            image_full_path = os.path.join(image_base_dir, image_relative_path)
-
-        # os.path.isfile() is slow when mounting remote directories; much faster
-        # to just try/except on the image open.
-        try:
-            image = vis_utils.open_image(image_full_path)
-        except:
-            print('Warning: could not open image file {}'.format(image_full_path))
-            image = None
-            # return ''
-        
-        # Render images to a flat folder
-        sample_name = res + '_' + path_utils.flatten_path(image_relative_path)
-        fullpath = os.path.join(options.output_dir, res, sample_name)
-
-        if image is not None:
-            
-            original_size = image.size
-            
-            if options.viz_target_width is not None:
-                image = vis_utils.resize_image(image, options.viz_target_width)
-    
-            if ground_truth_boxes is not None and len(ground_truth_boxes) > 0:
-                
-                # Create class labels like "gt_1" or "gt_27"
-                gt_classes = [0] * len(ground_truth_boxes)
-                label_map = {0:'ground truth'}
-                # for i_box,box in enumerate(ground_truth_boxes):
-                #    gt_classes.append('_' + str(box[-1]))
-                vis_utils.render_db_bounding_boxes(ground_truth_boxes, gt_classes, image,
-                                                   original_size=original_size,label_map=label_map,
-                                                   thickness=4,expansion=4)
-        
-            # render_detection_bounding_boxes expects either a float or a dict mapping
-            # category IDs to names.
-            if isinstance(options.confidence_threshold,float):
-                rendering_confidence_threshold = options.confidence_threshold
-            else:
-                category_ids = set()
-                for d in detections:
-                    category_ids.add(d['category'])
-                rendering_confidence_threshold = {}
-                for category_id in category_ids:
-                    rendering_confidence_threshold[category_id] = \
-                        _get_threshold_for_category_id(category_id, options, detection_categories)
-                
-            vis_utils.render_detection_bounding_boxes(
-                detections, image,
-                label_map=detection_categories,
-                classification_label_map=classification_categories,
-                confidence_threshold=rendering_confidence_threshold,
-                thickness=options.line_thickness,
-                expansion=options.box_expansion)
-    
-            try:
-                image.save(fullpath)
-            except OSError as e:
-                # errno.ENAMETOOLONG doesn't get thrown properly on Windows, so
-                # we awkwardly check against a hard-coded limit
-                if (e.errno == errno.ENAMETOOLONG) or (len(fullpath) >= 259):
-                    extension = os.path.splitext(sample_name)[1]
-                    sample_name = res + '_' + str(uuid.uuid4()) + extension
-                    image.save(os.path.join(options.output_dir, res, sample_name))
-                else:
-                    raise
-
-    # Use slashes regardless of os
-    file_name = '{}/{}'.format(res,sample_name)
-
-    info = {
-        'filename': file_name,
-        'title': display_name,
-        'textStyle':\
-         'font-family:verdana,arial,calibri;font-size:80%;text-align:left;margin-top:20;margin-bottom:5'
-    }
-    
-    # Optionally add links back to the original images
-    if options.link_images_to_originals and (image_full_path is not None):
-                
-        # Handling special characters in links has been pushed down into
-        # write_html_image_list
-        #
-        # link_target = image_full_path.replace('\\','/')
-        # link_target  = urllib.parse.quote(link_target)
-        link_target = image_full_path
-        info['linkTarget'] = link_target 
-        
-    return info
-
-# ..._render_bounding_boxes
-
-
-def _prepare_html_subpages(images_html, output_dir, options=None):
-    """
-    Write out a series of html image lists, e.g. the "detections" or "non-detections"
-    pages.
-
-    image_html is a dictionary mapping an html page name (e.g. "detections_animal") to 
-    a list of image structs friendly to write_html_image_list.
-    
-    Returns a dictionary mapping category names to image counts.
-    """
-    
-    if options is None:
-        options = PostProcessingOptions()
-
-    # Count items in each category
-    image_counts = {}
-    for res, array in images_html.items():
-        image_counts[res] = len(array)
-
-    # Optionally sort by filename before writing to html
-    if options.html_sort_order == 'filename':
-        images_html_sorted = {}
-        for res, array in images_html.items():
-            sorted_array = sorted(array, key=lambda x: x['filename'])
-            images_html_sorted[res] = sorted_array
-        images_html = images_html_sorted
-    
-    # Optionally sort by confidence before writing to html
-    elif options.html_sort_order == 'confidence':
-        images_html_sorted = {}
-        for res, array in images_html.items():
-            
-            if not all(['max_conf' in d for d in array]):
-                print("Warning: some elements in the {} page don't have confidence values, can't sort by confidence".format(res))
-            else:
-                sorted_array = sorted(array, key=lambda x: x['max_conf'], reverse=True)
-                images_html_sorted[res] = sorted_array
-                images_html = images_html_sorted
-        
-    else:
-        assert options.html_sort_order == 'random',\
-            'Unrecognized sort order {}'.format(options.html_sort_order)
-        images_html_sorted = {}
-        for res, array in images_html.items():
-            sorted_array = random.sample(array,len(array))
-            images_html_sorted[res] = sorted_array
-        images_html = images_html_sorted
-        
-    # Write the individual HTML files
-    for res, array in images_html.items():
-        
-        html_image_list_options = {}    
-        html_image_list_options['maxFiguresPerHtmlFile'] = options.max_figures_per_html_file
-        html_image_list_options['headerHtml'] = '<h1>{}</h1>'.format(res.upper())
-        
-        # Don't write empty pages
-        if len(array) == 0:
-            continue
-        else:
-            write_html_image_list(
-                filename=os.path.join(output_dir, '{}.html'.format(res)),
-                images=array,
-                options=html_image_list_options)
-
-    return image_counts
-
-# ..._prepare_html_subpages()
-
-
-def _get_threshold_for_category_name(category_name,options):
-    """
-    Determines the confidence threshold we should use for a specific category name.
-    """
-    
-    if isinstance(options.confidence_threshold,float):
-        return options.confidence_threshold
-    else:
-        assert isinstance(options.confidence_threshold,dict), \
-            'confidence_threshold must either be a float or a dict'
-                           
-    if category_name in options.confidence_threshold:
-        
-        return options.confidence_threshold[category_name]
-    
-    else:
-        assert 'default' in options.confidence_threshold, \
-            'category {} not in confidence_threshold dict, and no default supplied'.format(
-                category_name)
-        return options.confidence_threshold['default']
-
-    
-def _get_threshold_for_category_id(category_id,options,detection_categories):
-    """
-    Determines the confidence threshold we should use for a specific category ID.
-    
-    [detection_categories] is a dict mapping category IDs to names.
-    """
-    
-    if isinstance(options.confidence_threshold,float):        
-        return options.confidence_threshold
-    
-    assert category_id in detection_categories, \
-        'Invalid category ID {}'.format(category_id)
-        
-    category_name = detection_categories[category_id]
-    
-    return _get_threshold_for_category_name(category_name,options)
-    
-    
-def _get_positive_categories(detections,options,detection_categories):
-    """
-    Gets a sorted list of unique categories (as string IDs) above the threshold for this image
-    
-    [detection_categories] is a dict mapping category IDs to names.
-    """
-    
-    positive_categories = set()
-    for d in detections:
-        threshold = _get_threshold_for_category_id(d['category'], options, detection_categories)
-        if d['conf'] >= threshold:
-            positive_categories.add(d['category'])
-    return sorted(positive_categories)
-
-
-def _has_positive_detection(detections,options,detection_categories):
-    """
-    Determines whether any positive detections are present in the detection list
-    [detections].
-    """    
-    
-    found_positive_detection = False
-    for d in detections:
-        threshold = _get_threshold_for_category_id(d['category'], options, detection_categories)
-        if d['conf'] >= threshold:
-            found_positive_detection = True
-            break
-    return found_positive_detection
-        
-
-def _render_image_no_gt(file_info,detection_categories_to_results_name,
-                       detection_categories,classification_categories,
-                       options):
-    """
-    Renders an image (with no ground truth information)
-    
-    Returns a list of rendering structs, where the first item is a category (e.g. "detections_animal"), 
-    and  the second is a dict of information needed for rendering.  E.g.:
-        
-    [['detections_animal', 
-    {
-     'filename': 'detections_animal/detections_animal_blah~01060415.JPG', 
-     'title': '<b>Result type</b>: detections_animal, 
-               <b>Image</b>: blah\\01060415.JPG,
-               <b>Max conf</b>: 0.897',
-      'textStyle': 'font-family:verdana,arial,calibri;font-size:80%;text-align:left;margin-top:20;margin-bottom:5',
-      'linkTarget': 'full_path_to_%5C01060415.JPG'
-    }]]
-     
-    When no classification data is present, this list will always be length-1.  When
-    classification data is present, an image may appear in multiple categories.
-    
-    Populates the 'max_conf' field of the first element of the list.
-    
-    Returns None if there are any errors.
-    """
-    
-    image_relative_path = file_info[0]
-    max_conf = file_info[1]
-    detections = file_info[2]
-
-    # Determine whether any positive detections are present (using a threshold that
-    # may vary by category)
-    found_positive_detection = _has_positive_detection(detections,options,detection_categories)
-        
-    detection_status = DetectionStatus.DS_UNASSIGNED
-    if found_positive_detection:
-        detection_status = DetectionStatus.DS_POSITIVE
-    else:
-        if options.include_almost_detections:
-            if max_conf >= options.almost_detection_confidence_threshold:
-                detection_status = DetectionStatus.DS_ALMOST
-            else:
-                detection_status = DetectionStatus.DS_NEGATIVE
-        else:
-            detection_status = DetectionStatus.DS_NEGATIVE
-
-    if detection_status == DetectionStatus.DS_POSITIVE:
-        if options.separate_detections_by_category:
-            positive_categories = tuple(_get_positive_categories(detections,options,detection_categories))
-            if positive_categories not in detection_categories_to_results_name:
-                raise ValueError('Error: {} not in category mapping (file {})'.format(
-                    str(positive_categories),image_relative_path))
-            res = detection_categories_to_results_name[positive_categories]
-        else:
-            res = 'detections'
-
-    elif detection_status == DetectionStatus.DS_NEGATIVE:
-        res = 'non_detections'
-    else:
-        assert detection_status == DetectionStatus.DS_ALMOST
-        res = 'almost_detections'
-
-    display_name = '<b>Result type</b>: {}, <b>Image</b>: {}, <b>Max conf</b>: {:0.3f}'.format(
-        res, image_relative_path, max_conf)
-
-    rendering_options = copy.copy(options)
-    if detection_status == DetectionStatus.DS_ALMOST:
-        rendering_options.confidence_threshold = \
-            rendering_options.almost_detection_confidence_threshold
-            
-    rendered_image_html_info = _render_bounding_boxes(
-        image_base_dir=options.image_base_dir,
-        image_relative_path=image_relative_path,
-        display_name=display_name,
-        detections=detections,
-        res=res,
-        ground_truth_boxes=None,
-        detection_categories=detection_categories,
-        classification_categories=classification_categories,
-        options=rendering_options)
-
-    image_result = None
-
-    if len(rendered_image_html_info) > 0:
-
-        image_result = [[res, rendered_image_html_info]]
-
-        max_conf = 0
-        
-        for det in detections:
-            
-            if det['conf'] > max_conf:
-                max_conf = det['conf']
-
-            if ('classifications' in det):
-
-                # This is a list of [class,confidence] pairs, sorted by confidence
-                classifications = det['classifications']
-                top1_class_id = classifications[0][0]
-                top1_class_name = classification_categories[top1_class_id]
-                top1_class_score = classifications[0][1]
-
-                # If we either don't have a confidence threshold, or we've met our
-                # confidence threshold
-                if (options.classification_confidence_threshold < 0) or \
-                    (top1_class_score >= options.classification_confidence_threshold):
-                    image_result.append(['class_{}'.format(top1_class_name),
-                                         rendered_image_html_info])
-                else:
-                    image_result.append(['class_unreliable',
-                                         rendered_image_html_info])
-
-            # ...if this detection has classification info
-
-        # ...for each detection
-
-        image_result[0][1]['max_conf'] = max_conf
-        
-    # ...if we got valid rendering info back from _render_bounding_boxes()
-    
-    return image_result
-
-# ...def _render_image_no_gt()
-    
-
-def _render_image_with_gt(file_info,ground_truth_indexed_db,
-                         detection_categories,classification_categories,options):
-    """
-    Render an image with ground truth information.  See _render_image_no_gt for return
-    data format.
-    """
-    
-    image_relative_path = file_info[0]
-    max_conf = file_info[1]
-    detections = file_info[2]
-
-    # This should already have been normalized to either '/' or '\'
-
-    image_id = ground_truth_indexed_db.filename_to_id.get(image_relative_path, None)
-    if image_id is None:
-        print('Warning: couldn''t find ground truth for image {}'.format(image_relative_path))
-        return None
-
-    image = ground_truth_indexed_db.image_id_to_image[image_id]
-    annotations = ground_truth_indexed_db.image_id_to_annotations[image_id]
-
-    ground_truth_boxes = []
-    for ann in annotations:
-        if 'bbox' in ann:
-            ground_truth_box = [x for x in ann['bbox']]
-            ground_truth_box.append(ann['category_id'])
-            ground_truth_boxes.append(ground_truth_box)
-    
-    gt_status = image['_detection_status']
-
-    gt_presence = bool(gt_status)
-
-    gt_classes = CameraTrapJsonUtils.annotations_to_class_names(
-        annotations, ground_truth_indexed_db.cat_id_to_name)
-    gt_class_summary = ','.join(gt_classes)
-
-    if gt_status > DetectionStatus.DS_MAX_DEFINITIVE_VALUE:
-        print(f'Skipping image {image_id}, does not have a definitive '
-              f'ground truth status (status: {gt_status}, classes: {gt_class_summary})')
-        return None
-
-    detected = _has_positive_detection(detections, options, detection_categories)
-
-    if gt_presence and detected:
-        if '_classification_accuracy' not in image.keys():
-            res = 'tp'
-        elif np.isclose(1, image['_classification_accuracy']):
-            res = 'tpc'
-        else:
-            res = 'tpi'
-    elif not gt_presence and detected:
-        res = 'fp'
-    elif gt_presence and not detected:
-        res = 'fn'
-    else:
-        res = 'tn'
-
-    display_name = '<b>Result type</b>: {}, <b>Presence</b>: {}, <b>Class</b>: {}, <b>Max conf</b>: {:0.3f}%, <b>Image</b>: {}'.format(
-        res.upper(), str(gt_presence), gt_class_summary,
-        max_conf * 100, image_relative_path)
-
-    rendered_image_html_info = _render_bounding_boxes(
-        image_base_dir=options.image_base_dir,
-        image_relative_path=image_relative_path,
-        display_name=display_name,
-        detections=detections,
-        res=res,
-        ground_truth_boxes=ground_truth_boxes,
-        detection_categories=detection_categories,
-        classification_categories=classification_categories,
-        options=options)
-
-    image_result = None
-    if len(rendered_image_html_info) > 0:
-        image_result = [[res, rendered_image_html_info]]
-        for gt_class in gt_classes:
-            image_result.append(['class_{}'.format(gt_class), rendered_image_html_info])
-
-    return image_result
-
-# ...def _render_image_with_gt()
-
-    
-#%% Main function
-
-def process_batch_results(options):
-
-    """
-    Given a .json or .csv file containing MD results, do one or more of the following:
-
-    * Sample detections/non-detections and render to HTML (when ground truth isn't
-      available) (this is 99.9% of what this module is for)
-    * Evaluate detector precision/recall, optionally rendering results (requires
-      ground truth)
-    * Sample true/false positives/negatives and render to HTML (requires ground
-      truth)
-
-    Ground truth, if available, must be in COCO Camera Traps format:
-        
-    https://github.com/agentmorris/MegaDetector/blob/main/data_management/README.md#coco-camera-traps-format
-
-    Args:
-        options (PostProcessingOptions): everything we need to render a preview/analysis for
-            this set of results; see the PostProcessingOptions class for details.
-           
-    Returns:
-        PostProcessingResults: information about the results/preview, most importantly the HTML filename
-            of the output.  See the PostProcessingResults class for details.
-    """
-    ppresults = PostProcessingResults()
-
-    ##%% Expand some options for convenience
-
-    output_dir = options.output_dir
-
-
-    ##%% Prepare output dir
-
-    os.makedirs(output_dir, exist_ok=True)
-
-
-    ##%% Load ground truth if available
-
-    ground_truth_indexed_db = None
-
-    if (options.ground_truth_json_file is not None) and (len(options.ground_truth_json_file) > 0):
-        assert (options.confidence_threshold is None) or (isinstance(options.confidence_threshold,float)), \
-            'Variable confidence thresholds are not supported when supplying ground truth'
-            
-    if (options.ground_truth_json_file is not None) and (len(options.ground_truth_json_file) > 0):
-
-        if options.separate_detections_by_category:
-            print("Warning: I don't know how to separate categories yet when doing " + \
-                  "a P/R analysis, disabling category separation")
-            options.separate_detections_by_category = False
-
-        ground_truth_indexed_db = IndexedJsonDb(
-            options.ground_truth_json_file, b_normalize_paths=True,
-            filename_replacements=options.ground_truth_filename_replacements)
-
-        # Mark images in the ground truth as positive or negative
-        n_negative, n_positive, n_unknown, n_ambiguous = _mark_detection_status(
-            ground_truth_indexed_db, negative_classes=options.negative_classes,
-            unknown_classes=options.unlabeled_classes)
-        print(f'Finished loading and indexing ground truth: {n_negative} '
-              f'negative, {n_positive} positive, {n_unknown} unknown, '
-              f'{n_ambiguous} ambiguous')
-
-
-    ##%% Load detection (and possibly classification) results
-
-    # If the caller hasn't supplied results, load them
-    if options.api_detection_results is None:
-        detections_df, other_fields = load_api_results(
-            options.api_output_file, force_forward_slashes=True,
-            filename_replacements=options.api_output_filename_replacements)
-        ppresults.api_detection_results = detections_df
-        ppresults.api_other_fields = other_fields        
-
-    else:
-        print('Bypassing detection results loading...')
-        assert options.api_other_fields is not None
-        detections_df = options.api_detection_results
-        other_fields = options.api_other_fields
-
-    # Determine confidence thresholds if necessary
-    
-    if options.confidence_threshold is None:
-        options.confidence_threshold = \
-            get_typical_confidence_threshold_from_results(other_fields)
-        print('Choosing default confidence threshold of {} based on MD version'.format(
-            options.confidence_threshold))    
-            
-    if options.almost_detection_confidence_threshold is None and options.include_almost_detections:
-        assert isinstance(options.confidence_threshold,float), \
-            'If you are using a dictionary of confidence thresholds and almost-detections are enabled, ' + \
-            'you need to supply a threshold for almost detections.'
-        options.almost_detection_confidence_threshold = options.confidence_threshold - 0.05
-        if options.almost_detection_confidence_threshold < 0:
-            options.almost_detection_confidence_threshold = 0
-            
-    # Remove rows with inference failures (typically due to corrupt images)
-    n_failures = 0
-    if 'failure' in detections_df.columns:
-        n_failures = detections_df['failure'].count()
-        print('Ignoring {} failed images'.format(n_failures))
-        # Explicitly forcing a copy() operation here to suppress "trying to be set
-        # on a copy" warnings (and associated risks) below.
-        detections_df = detections_df[detections_df['failure'].isna()].copy()
-        
-    assert other_fields is not None
-
-    detection_categories = other_fields['detection_categories']
-    
-    # Convert keys and values to lowercase
-    classification_categories = other_fields.get('classification_categories', {})
-    if classification_categories is not None:
-        classification_categories = {
-            k.lower(): v.lower()
-            for k, v in classification_categories.items()
-        }
-
-    # Count detections and almost-detections for reporting purposes
-    n_positives = 0
-    n_almosts = 0
-    
-    for i_row,row in tqdm(detections_df.iterrows(),total=len(detections_df)):
-        
-        detections = row['detections']
-        max_conf = row['max_detection_conf']
-        if _has_positive_detection(detections, options, detection_categories):
-            n_positives += 1
-        elif (options.almost_detection_confidence_threshold is not None) and \
-             (max_conf >= options.almost_detection_confidence_threshold):
-            n_almosts += 1        
-        
-    print(f'Finished loading and preprocessing {len(detections_df)} rows '
-          f'from detector output, predicted {n_positives} positives.')
-
-    if options.include_almost_detections:
-        print('...and {} almost-positives'.format(n_almosts))
-
-
-    ##%% Find descriptive metadata to include at the top of the page
-
-    if options.job_name_string is not None:
-        job_name_string = options.job_name_string
-    else:
-        # This is rare; it only happens during debugging when the caller
-        # is supplying already-loaded API results.
-        if options.api_output_file is None:
-            job_name_string = 'unknown'
-        else:
-            job_name_string = os.path.basename(options.api_output_file)
-    
-    if options.model_version_string is not None:
-        model_version_string = options.model_version_string
-    else:
-        
-        if 'info' not in other_fields or 'detector' not in other_fields['info']:
-            print('No model metadata supplied, assuming MDv4')
-            model_version_string = 'MDv4 (assumed)'
-        else:            
-            model_version_string = other_fields['info']['detector']
-    
-        
-    ##%% If we have ground truth, remove images we can't match to ground truth
-
-    if ground_truth_indexed_db is not None:
-
-        b_match = detections_df['file'].isin(
-            ground_truth_indexed_db.filename_to_id)
-        print(f'Confirmed filename matches to ground truth for {sum(b_match)} '
-              f'of {len(detections_df)} files')
-
-        detections_df = detections_df[b_match]
-        detector_files = detections_df['file'].tolist()
-
-        assert len(detector_files) > 0, (
-            'No detection files available, possible path issue?')
-
-        print('Trimmed detection results to {} files'.format(len(detector_files)))
-
-
-    ##%% (Optionally) sample from the full set of images
-
-    images_to_visualize = detections_df
-
-    if options.num_images_to_sample is not None and options.num_images_to_sample > 0:
-        images_to_visualize = images_to_visualize.sample(
-            n=min(options.num_images_to_sample, len(images_to_visualize)),
-            random_state=options.sample_seed)
-
-    output_html_file = ''
-
-    style_header = """<head>
-        <style type="text/css">
-        a { text-decoration: none; }
-        body { font-family: segoe ui, calibri, "trebuchet ms", verdana, arial, sans-serif; }
-        div.contentdiv { margin-left: 20px; }
-        </style>
-        </head>"""
-
-
-    ##%% Fork here depending on whether or not ground truth is available
-
-    # If we have ground truth, we'll compute precision/recall and sample tp/fp/tn/fn.
-    #
-    # Otherwise we'll just visualize detections/non-detections.
-
-    if ground_truth_indexed_db is not None:
-
-        ##%% Detection evaluation: compute precision/recall
-
-        # numpy array of detection probabilities
-        p_detection = detections_df['max_detection_conf'].values
-        n_detections = len(p_detection)
-
-        # numpy array of bools (0.0/1.0), and -1 as null value
-        gt_detections = np.zeros(n_detections, dtype=float)
-
-        for i_detection, fn in enumerate(detector_files):
-            image_id = ground_truth_indexed_db.filename_to_id[fn]
-            image = ground_truth_indexed_db.image_id_to_image[image_id]
-            detection_status = image['_detection_status']
-
-            if detection_status == DetectionStatus.DS_NEGATIVE:
-                gt_detections[i_detection] = 0.0
-            elif detection_status == DetectionStatus.DS_POSITIVE:
-                gt_detections[i_detection] = 1.0
-            else:
-                gt_detections[i_detection] = -1.0
-
-        # Don't include ambiguous/unknown ground truth in precision/recall analysis
-        b_valid_ground_truth = gt_detections >= 0.0
-
-        p_detection_pr = p_detection[b_valid_ground_truth]
-        gt_detections_pr = (gt_detections[b_valid_ground_truth] == 1.)
-
-        print('Including {} of {} values in p/r analysis'.format(np.sum(b_valid_ground_truth),
-              len(b_valid_ground_truth)))
-
-        precisions, recalls, thresholds = precision_recall_curve(gt_detections_pr, p_detection_pr)
-
-        # For completeness, include the result at a confidence threshold of 1.0
-        thresholds = np.append(thresholds, [1.0])
-
-        precisions_recalls = pd.DataFrame(data={
-            'confidence_threshold': thresholds,
-            'precision': precisions,
-            'recall': recalls
-        })
-
-        # Compute and print summary statistics
-        average_precision = average_precision_score(gt_detections_pr, p_detection_pr)
-        print('Average precision: {:.1%}'.format(average_precision))
-
-        # Thresholds go up throughout precisions/recalls/thresholds; find the last
-        # value where recall is at or above target.  That's our precision @ target recall.
-        
-        i_above_target_recall = (np.where(recalls >= options.target_recall))
-        
-        # np.where returns a tuple of arrays, but in this syntax where we're 
-        # comparing an array with a scalar, there will only be one element.
-        assert len (i_above_target_recall) == 1
-        
-        # Convert back to a list
-        i_above_target_recall = i_above_target_recall[0].tolist()
-        
-        if len(i_above_target_recall) == 0:
-            precision_at_target_recall = 0.0
-        else:
-            precision_at_target_recall = precisions[i_above_target_recall[-1]]
-        print('Precision at {:.1%} recall: {:.1%}'.format(options.target_recall,
-                                                          precision_at_target_recall))
-
-        cm_predictions = np.array(p_detection_pr) > options.confidence_threshold
-        cm = confusion_matrix(gt_detections_pr, cm_predictions, labels=[False,True])
-
-        # Flatten the confusion matrix
-        tn, fp, fn, tp = cm.ravel()
-
-        precision_at_confidence_threshold = tp / (tp + fp)
-        recall_at_confidence_threshold = tp / (tp + fn)
-        f1 = 2.0 * (precision_at_confidence_threshold * recall_at_confidence_threshold) / \
-            (precision_at_confidence_threshold + recall_at_confidence_threshold)
-
-        print('At a confidence threshold of {:.1%}, precision={:.1%}, recall={:.1%}, f1={:.1%}'.format(
-                options.confidence_threshold, precision_at_confidence_threshold,
-                recall_at_confidence_threshold, f1))
-
-        ##%% Collect classification results, if they exist
-
-        classifier_accuracies = []
-
-        # Mapping of classnames to idx for the confusion matrix.
-        #
-        # The lambda is actually kind of a hack, because we use assume that
-        # the following code does not reassign classname_to_idx
-        classname_to_idx = collections.defaultdict(lambda: len(classname_to_idx))
-
-        # Confusion matrix as defaultdict of defaultdict
-        #
-        # Rows / first index is ground truth, columns / second index is predicted category
-        classifier_cm = collections.defaultdict(lambda: collections.defaultdict(lambda: 0))
-
-        # iDetection = 0; fn = detector_files[iDetection]; print(fn)
-        assert len(detector_files) == len(detections_df)
-        for iDetection, fn in enumerate(detector_files):
-
-            image_id = ground_truth_indexed_db.filename_to_id[fn]
-            image = ground_truth_indexed_db.image_id_to_image[image_id]
-            detections = detections_df['detections'].iloc[iDetection]
-            pred_class_ids = [det['classifications'][0][0] \
-                for det in detections if 'classifications' in det.keys()]
-            pred_classnames = [classification_categories[pd] for pd in pred_class_ids]
-
-            # If this image has classification predictions, and an unambiguous class
-            # annotated, and is a positive image...
-            if len(pred_classnames) > 0 \
-                    and '_unambiguous_category' in image.keys() \
-                    and image['_detection_status'] == DetectionStatus.DS_POSITIVE:
-
-                # The unambiguous category, we make this a set for easier handling afterward
-                gt_categories = set([image['_unambiguous_category']])
-                pred_categories = set(pred_classnames)
-
-                # Compute the accuracy as intersection of union,
-                # i.e. (# of categories in both prediction and GT)
-                #      divided by (# of categories in either prediction or GT
-                #
-                # In case of only one GT category, the result will be 1.0, if
-                # prediction is one category and this category matches GT
-                #
-                # It is 1.0/(# of predicted top-1 categories), if the GT is
-                # one of the predicted top-1 categories.
-                #
-                # It is 0.0, if none of the predicted categories is correct
-
-                classifier_accuracies.append(
-                    len(gt_categories & pred_categories)
-                    / len(gt_categories | pred_categories)
-                )
-                image['_classification_accuracy'] = classifier_accuracies[-1]
-
-                # Distribute this accuracy across all predicted categories in the
-                # confusion matrix
-                assert len(gt_categories) == 1
-                gt_class_idx = classname_to_idx[list(gt_categories)[0]]
-                for pred_category in pred_categories:
-                    pred_class_idx = classname_to_idx[pred_category]
-                    classifier_cm[gt_class_idx][pred_class_idx] += 1
-
-        # ...for each file in the detection results
-
-        # If we have classification results
-        if len(classifier_accuracies) > 0:
-
-            # Build confusion matrix as array from classifier_cm
-            all_class_ids = sorted(classname_to_idx.values())
-            classifier_cm_array = np.array(
-                [[classifier_cm[r_idx][c_idx] for c_idx in all_class_ids] for \
-                 r_idx in all_class_ids], dtype=float)
-            classifier_cm_array /= (classifier_cm_array.sum(axis=1, keepdims=True) + 1e-7)
-
-            # Print some statistics
-            print('Finished computation of {} classification results'.format(
-                len(classifier_accuracies)))
-            print('Mean accuracy: {}'.format(np.mean(classifier_accuracies)))
-
-            # Prepare confusion matrix output
-
-            # Get confusion matrix as string
-            sio = io.StringIO()
-            np.savetxt(sio, classifier_cm_array * 100, fmt='%5.1f')
-            cm_str = sio.getvalue()
-            # Get fixed-size classname for each idx
-            idx_to_classname = {v:k for k,v in classname_to_idx.items()}
-            classname_list = [idx_to_classname[idx] for idx in sorted(classname_to_idx.values())]
-            classname_headers = ['{:<5}'.format(cname[:5]) for cname in classname_list]
-
-            # Prepend class name on each line and add to the top
-            cm_str_lines = [' ' * 16 + ' '.join(classname_headers)]
-            cm_str_lines += ['{:>15}'.format(cn[:15]) + ' ' + cm_line for cn, cm_line in \
-                             zip(classname_list, cm_str.splitlines())]
-
-            # Print formatted confusion matrix
-            if False:
-                # Actually don't, this gets really messy in all but the widest consoles
-                print('Confusion matrix: ')
-                print(*cm_str_lines, sep='\n')
-
-            # Plot confusion matrix
-
-            # To manually add more space at bottom: plt.rcParams['figure.subplot.bottom'] = 0.1
-            #
-            # Add 0.5 to figsize for every class. For two classes, this will result in
-            # fig = plt.figure(figsize=[4,4])
-            fig = plot_utils.plot_confusion_matrix(
-                classifier_cm_array,
-                classname_list,
-                normalize=False,
-                title='Confusion matrix',
-                cmap=plt.cm.Blues,
-                vmax=1.0,
-                use_colorbar=True,
-                y_label=True)
-            cm_figure_relative_filename = 'confusion_matrix.png'
-            cm_figure_filename = os.path.join(output_dir, cm_figure_relative_filename)
-            plt.savefig(cm_figure_filename)
-            plt.close(fig)
-
-        # ...if we have classification results
-
-
-        ##%% Render output
-
-        # Write p/r table to .csv file in output directory
-        pr_table_filename = os.path.join(output_dir, 'prec_recall.csv')
-        precisions_recalls.to_csv(pr_table_filename, index=False)
-
-        # Write precision/recall plot to .png file in output directory
-        t = 'Precision-Recall curve: AP={:0.1%}, P@{:0.1%}={:0.1%}'.format(
-            average_precision, options.target_recall, precision_at_target_recall)
-        fig = plot_utils.plot_precision_recall_curve(precisions, recalls, t)
-        
-        pr_figure_relative_filename = 'prec_recall.png'
-        pr_figure_filename = os.path.join(output_dir, pr_figure_relative_filename)
-        fig.savefig(pr_figure_filename)
-        plt.close(fig)
-
-
-        ##%% Sampling
-
-        # Sample true/false positives/negatives with correct/incorrect top-1
-        # classification and render to html
-
-        # Accumulate html image structs (in the format expected by write_html_image_lists)
-        # for each category, e.g. 'tp', 'fp', ..., 'class_bird', ...
-        images_html = collections.defaultdict(list)
-        
-        # Add default entries by accessing them for the first time
-        [images_html[res] for res in ['tp', 'tpc', 'tpi', 'fp', 'tn', 'fn']]
-        for res in images_html.keys():
-            os.makedirs(os.path.join(output_dir, res), exist_ok=True)
-
-        image_count = len(images_to_visualize)
-
-        # Each element will be a list of 2-tuples, with elements [collection name,html info struct]
-        rendering_results = []
-
-        # Each element will be a three-tuple with elements file,max_conf,detections
-        files_to_render = []
-
-        # Assemble the information we need for rendering, so we can parallelize without
-        # dealing with Pandas
-        # i_row = 0; row = images_to_visualize.iloc[0]
-        for _, row in images_to_visualize.iterrows():
-
-            # Filenames should already have been normalized to either '/' or '\'
-            files_to_render.append([row['file'], row['max_detection_conf'], row['detections']])
-
-        start_time = time.time()
-        if options.parallelize_rendering:
-            if options.parallelize_rendering_n_cores is None:                
-                if options.parallelize_rendering_with_threads:
-                    pool = ThreadPool()
-                else:
-                    pool = Pool()
-            else:
-                if options.parallelize_rendering_with_threads:
-                    pool = ThreadPool(options.parallelize_rendering_n_cores)
-                    worker_string = 'threads'
-                else:
-                    pool = Pool(options.parallelize_rendering_n_cores)
-                    worker_string = 'processes'
-                print('Rendering images with {} {}'.format(options.parallelize_rendering_n_cores,
-                                                           worker_string))
-                
-            rendering_results = list(tqdm(pool.imap(
-                partial(_render_image_with_gt,
-                        ground_truth_indexed_db=ground_truth_indexed_db,
-                        detection_categories=detection_categories,
-                        classification_categories=classification_categories,
-                        options=options), 
-                files_to_render), total=len(files_to_render)))
-        else:
-            for file_info in tqdm(files_to_render):
-                rendering_results.append(_render_image_with_gt(
-                    file_info,ground_truth_indexed_db,
-                    detection_categories,classification_categories,
-                    options=options))
-        elapsed = time.time() - start_time
-
-        # Map all the rendering results in the list rendering_results into the
-        # dictionary images_html, which maps category names to lists of results
-        image_rendered_count = 0
-        for rendering_result in rendering_results:
-            if rendering_result is None:
-                continue
-            image_rendered_count += 1
-            for assignment in rendering_result:
-                images_html[assignment[0]].append(assignment[1])
-
-        # Prepare the individual html image files
-        image_counts = _prepare_html_subpages(images_html, output_dir, options)
-
-        print('{} images rendered (of {})'.format(image_rendered_count,image_count))
-
-        # Write index.html
-        all_tp_count = image_counts['tp'] + image_counts['tpc'] + image_counts['tpi']
-        total_count = all_tp_count + image_counts['tn'] + image_counts['fp'] + image_counts['fn']
-
-        classification_detection_results = """&nbsp;&nbsp;&nbsp;&nbsp;<a href="tpc.html">with all correct top-1 predictions (TPC)</a> ({})<br/>
-           &nbsp;&nbsp;&nbsp;&nbsp;<a href="tpi.html">with one or more incorrect top-1 prediction (TPI)</a> ({})<br/>
-           &nbsp;&nbsp;&nbsp;&nbsp;<a href="tp.html">without classification evaluation</a><sup>*</sup> ({})<br/>""".format(
-            image_counts['tpc'],
-            image_counts['tpi'],
-            image_counts['tp']
-        )
-
-        confidence_threshold_string = ''
-        if isinstance(options.confidence_threshold,float):
-            confidence_threshold_string = '{:.2%}'.format(options.confidence_threshold)
-        else:
-            confidence_threshold_string = str(options.confidence_threshold)
-        
-        index_page = """<html>
-        {}
-        <body>
-        <h2>Evaluation</h2>
-
-        <h3>Job metadata</h3>
-        
-        <div class="contentdiv">
-        <p>Job name: {}<br/>
-        <p>Model version: {}</p>
-        </div>
-        
-        <h3>Sample images</h3>
-        <div class="contentdiv">
-        <p>A sample of {} images, annotated with detections above confidence {}.</p>
-        <a href="tp.html">True positives (TP)</a> ({}) ({:0.1%})<br/>
-        CLASSIFICATION_PLACEHOLDER_1
-        <a href="tn.html">True negatives (TN)</a> ({}) ({:0.1%})<br/>
-        <a href="fp.html">False positives (FP)</a> ({}) ({:0.1%})<br/>
-        <a href="fn.html">False negatives (FN)</a> ({}) ({:0.1%})<br/>
-        CLASSIFICATION_PLACEHOLDER_2
-        </div>
-        """.format(
-            style_header,job_name_string,model_version_string,
-            image_count, confidence_threshold_string,
-            all_tp_count, all_tp_count/total_count,
-            image_counts['tn'], image_counts['tn']/total_count,
-            image_counts['fp'], image_counts['fp']/total_count,
-            image_counts['fn'], image_counts['fn']/total_count
-        )
-
-        index_page += """
-            <h3>Detection results</h3>
-            <div class="contentdiv">
-            <p>At a confidence threshold of {}, precision={:0.1%}, recall={:0.1%}</p>
-            <p><strong>Precision/recall summary for all {} images</strong></p><img src="{}"><br/>
-            </div>
-            """.format(
-                confidence_threshold_string, precision_at_confidence_threshold, recall_at_confidence_threshold,
-                len(detections_df), pr_figure_relative_filename
-           )
-
-        if len(classifier_accuracies) > 0:
-            index_page = index_page.replace('CLASSIFICATION_PLACEHOLDER_1',classification_detection_results)
-            index_page = index_page.replace('CLASSIFICATION_PLACEHOLDER_2',"""<p><sup>*</sup>We do not evaluate the classification result of images
-                if the classification information is missing, if the image contains
-                categories like &lsquo;empty&rsquo; or &lsquo;human&rsquo;, or if the image has multiple
-                classification labels.</p>""")
-        else:
-            index_page = index_page.replace('CLASSIFICATION_PLACEHOLDER_1','')
-            index_page = index_page.replace('CLASSIFICATION_PLACEHOLDER_2','')
-
-        if len(classifier_accuracies) > 0:
-            index_page += """
-                <h3>Classification results</h3>
-                <div class="contentdiv">
-                <p>Classification accuracy: {:.2%}<br>
-                The accuracy is computed only for images with exactly one classification label.
-                The accuracy of an image is computed as 1/(number of unique detected top-1 classes),
-                i.e. if the model detects multiple boxes with different top-1 classes, then the accuracy
-                decreases and the image is put into 'TPI'.</p>
-                <p>Confusion matrix:</p>
-                <p><img src="{}"></p>
-                <div style='font-family:monospace;display:block;'>{}</div>
-                </div>
-                """.format(
-                    np.mean(classifier_accuracies),
-                    cm_figure_relative_filename,
-                    "<br>".join(cm_str_lines).replace(' ', '&nbsp;')
-                )
-
-        # Show links to each GT class
-        #
-        # We could do this without classification results; currently we don't.
-        if len(classname_to_idx) > 0:
-
-            index_page += '<h3>Images of specific classes</h3><br/><div class="contentdiv">'
-            # Add links to all available classes
-            for cname in sorted(classname_to_idx.keys()):
-                index_page += '<a href="class_{0}.html">{0}</a> ({1})<br>'.format(
-                    cname,
-                    len(images_html['class_{}'.format(cname)]))
-            index_page += '</div>'
-
-        # Close body and html tags
-        index_page += '</body></html>'
-        output_html_file = os.path.join(output_dir, 'index.html')
-        with open(output_html_file, 'w') as f:
-            f.write(index_page)
-
-        print('Finished writing html to {}'.format(output_html_file))
-
-    # ...for each image
-
-
-    ##%% Otherwise, if we don't have ground truth...
-
-    else:
-
-        ##%% Sample detections/non-detections
-
-        # Accumulate html image structs (in the format expected by write_html_image_list)
-        # for each category
-        images_html = collections.defaultdict(list)
-        
-
-        # Add default entries by accessing them for the first time
-
-        # Maps sorted tuples of detection category IDs (string ints) - e.g. ("1"), ("1", "4", "7") - to 
-        # result set names, e.g. "detections_human", "detections_cat_truck".
-        detection_categories_to_results_name = {}
-        
-        # Keep track of which categories are single-class (e.g. "animal") and which are
-        # combinations (e.g. "animal_vehicle")
-        detection_categories_to_category_count = {}
-        
-        # For the creation of a "non-detections" category
-        images_html['non_detections']
-        detection_categories_to_category_count['non_detections'] = 0
-        
-        
-        if not options.separate_detections_by_category:
-            # For the creation of a "detections" category
-            images_html['detections']
-            detection_categories_to_category_count['detections'] = 0            
-        else:
-            # Add a set of results for each category and combination of categories, e.g.
-            # "detections_animal_vehicle".  When we're using this script for non-MegaDetector
-            # results, this can generate lots of categories, e.g. detections_bear_bird_cat_dog_pig.
-            # We'll keep that huge set of combinations in this map, but we'll only write
-            # out links for the ones that are non-empty.            
-            used_combinations = set()
-            
-            # row = images_to_visualize.iloc[0]
-            for i_row, row in images_to_visualize.iterrows():
-                detections_this_row = row['detections']
-                above_threshold_category_ids_this_row = set()
-                for detection in detections_this_row:
-                    threshold = _get_threshold_for_category_id(detection['category'], options, detection_categories)
-                    if detection['conf'] >= threshold:
-                        above_threshold_category_ids_this_row.add(detection['category'])
-                if len(above_threshold_category_ids_this_row) == 0:
-                    continue
-                sorted_categories_this_row = tuple(sorted(above_threshold_category_ids_this_row))
-                used_combinations.add(sorted_categories_this_row)
-            
-            for sorted_subset in used_combinations:
-                assert len(sorted_subset) > 0
-                results_name = 'detections'
-                for category_id in sorted_subset:
-                    results_name = results_name + '_' + detection_categories[category_id]
-                images_html[results_name]
-                detection_categories_to_results_name[sorted_subset] = results_name
-                detection_categories_to_category_count[results_name] = len(sorted_subset)                
-
-        if options.include_almost_detections:
-            images_html['almost_detections']
-            detection_categories_to_category_count['almost_detections'] = 0
-
-        # Create output directories
-        for res in images_html.keys():
-            os.makedirs(os.path.join(output_dir, res), exist_ok=True)
-
-        image_count = len(images_to_visualize)
-        
-        # Each element will be a list of 2-tuples, with elements [collection name,html info struct]
-        rendering_results = []
-
-        # list of 3-tuples with elements (file, max_conf, detections)
-        files_to_render = []
-
-        # Assemble the information we need for rendering, so we can parallelize without
-        # dealing with Pandas
-        # i_row = 0; row = images_to_visualize.iloc[0]
-        for _, row in images_to_visualize.iterrows():
-
-            assert isinstance(row['detections'],list)
-            
-            # Filenames should already have been normalized to either '/' or '\'
-            files_to_render.append([row['file'],
-                                    row['max_detection_conf'],
-                                    row['detections']])
-
-        start_time = time.time()
-        if options.parallelize_rendering:
-            
-            if options.parallelize_rendering_n_cores is None:                
-                if options.parallelize_rendering_with_threads:
-                    pool = ThreadPool()
-                else:
-                    pool = Pool()
-            else:
-                if options.parallelize_rendering_with_threads:
-                    pool = ThreadPool(options.parallelize_rendering_n_cores)
-                    worker_string = 'threads'
-                else:
-                    pool = Pool(options.parallelize_rendering_n_cores)
-                    worker_string = 'processes'
-                print('Rendering images with {} {}'.format(options.parallelize_rendering_n_cores,
-                                                           worker_string))
-                
-            # _render_image_no_gt(file_info,detection_categories_to_results_name,
-            # detection_categories,classification_categories)
-
-            rendering_results = list(tqdm(pool.imap(
-                partial(_render_image_no_gt, 
-                        detection_categories_to_results_name=detection_categories_to_results_name,
-                        detection_categories=detection_categories,
-                        classification_categories=classification_categories,
-                        options=options),
-                        files_to_render), total=len(files_to_render)))
-        else:
-            for file_info in tqdm(files_to_render):
-                rendering_results.append(_render_image_no_gt(file_info,
-                                                            detection_categories_to_results_name,
-                                                            detection_categories,
-                                                            classification_categories,
-                                                            options=options))
-                
-        elapsed = time.time() - start_time
-
-        # Do we have classification results in addition to detection results?
-        has_classification_info = False
-        
-        # Map all the rendering results in the list rendering_results into the
-        # dictionary images_html
-        image_rendered_count = 0
-        for rendering_result in rendering_results:
-            if rendering_result is None:
-                continue
-            image_rendered_count += 1
-            for assignment in rendering_result:
-                if 'class' in assignment[0]:
-                    has_classification_info = True
-                images_html[assignment[0]].append(assignment[1])
-
-        # Prepare the individual html image files
-        image_counts = _prepare_html_subpages(images_html, output_dir, options)
-        
-        if image_rendered_count == 0:
-            seconds_per_image = 0.0
-        else:
-            seconds_per_image = elapsed/image_rendered_count
-
-        print('Rendered {} images (of {}) in {} ({} per image)'.format(image_rendered_count,
-              image_count,humanfriendly.format_timespan(elapsed),
-              humanfriendly.format_timespan(seconds_per_image)))
-
-        # Write index.html
-
-        # We can't just sum these, because image_counts includes images in both their
-        # detection and classification classes
-        # total_images = sum(image_counts.values())
-        total_images = 0
-        for k in image_counts.keys():
-            v = image_counts[k]
-            if has_classification_info and k.startswith('class_'):
-                continue
-            total_images += v
-
-        if total_images != image_count:
-            print('Warning, missing images: image_count is {}, total_images is {}'.format(total_images,image_count))
-        
-        almost_detection_string = ''
-        if options.include_almost_detections:
-            almost_detection_string = ' (&ldquo;almost detection&rdquo; threshold at {:.1%})'.format(
-                options.almost_detection_confidence_threshold)
-
-        confidence_threshold_string = ''
-        if isinstance(options.confidence_threshold,float):
-            confidence_threshold_string = '{:.2%}'.format(options.confidence_threshold)
-        else:
-            confidence_threshold_string = str(options.confidence_threshold)
-            
-        index_page = """<html>\n{}\n<body>\n
-        <h2>Visualization of results for {}</h2>\n
-        <p>A sample of {} images (of {} total)FAILURE_PLACEHOLDER, annotated with detections above confidence {}{}.</p>\n
-        
-        <div class="contentdiv">
-        <p>Model version: {}</p>
-        </div>
-        
-        <h3>Sample images</h3>\n
-        <div class="contentdiv">\n""".format(
-            style_header, job_name_string, image_count, len(detections_df), confidence_threshold_string,
-            almost_detection_string, model_version_string)
-
-        failure_string = ''
-        if n_failures is not None:
-            failure_string = ' ({} failures)'.format(n_failures)        
-        index_page = index_page.replace('FAILURE_PLACEHOLDER',failure_string)
-        
-        def result_set_name_to_friendly_name(result_set_name):
-            friendly_name = ''
-            friendly_name = result_set_name.replace('_','-')
-            if friendly_name.startswith('detections-'):
-                friendly_name = friendly_name.replace('detections-', 'detections: ')
-            friendly_name = friendly_name.capitalize()
-            return friendly_name
-
-        sorted_result_set_names = sorted(list(images_html.keys()))
-        
-        result_set_name_to_count = {}
-        for result_set_name in sorted_result_set_names:
-            image_count = image_counts[result_set_name]
-            result_set_name_to_count[result_set_name] = image_count
-        sorted_result_set_names = sorted(sorted_result_set_names,
-                                         key=lambda x: result_set_name_to_count[x],
-                                         reverse=True)
-            
-        for result_set_name in sorted_result_set_names:
-
-            # Don't print classification classes here; we'll do that later with a slightly
-            # different structure
-            if has_classification_info and result_set_name.lower().startswith('class_'):
-                continue
-
-            filename = result_set_name + '.html'
-            label = result_set_name_to_friendly_name(result_set_name)
-            image_count = image_counts[result_set_name]
-            
-            # Don't include line items for empty multi-category pages
-            if image_count == 0 and \
-                detection_categories_to_category_count[result_set_name] > 1:
-                    continue
-            
-            if total_images == 0:
-                image_fraction = -1
-            else:
-                image_fraction = image_count / total_images
-                
-            # Write the line item for this category, including a link only if the
-            # category is non-empty
-            if image_count == 0:
-                index_page += '{} ({}, {:.1%})<br/>\n'.format(
-                    label,image_count,image_fraction)
-            else:
-                index_page += '<a href="{}">{}</a> ({}, {:.1%})<br/>\n'.format(
-                    filename,label,image_count,image_fraction)
-
-        index_page += '</div>\n'
-
-        if has_classification_info:
-            index_page += '<h3>Images of detected classes</h3>'
-            index_page += '<p>The same image might appear under multiple classes ' + \
-                'if multiple species were detected.</p>\n'
-            index_page += '<p>Classifications with confidence less than {:.1%} confidence are considered "unreliable".</p>\n'.format(
-                options.classification_confidence_threshold)
-            index_page += '<div class="contentdiv">\n'
-
-            # Add links to all available classes
-            class_names = sorted(classification_categories.values())
-            if 'class_unreliable' in images_html.keys():
-                class_names.append('unreliable')
-
-            if options.sort_classification_results_by_count:
-                class_name_to_count = {}                
-                for cname in class_names:
-                    ccount = len(images_html['class_{}'.format(cname)])
-                    class_name_to_count[cname] = ccount
-                class_names = sorted(class_names,key=lambda x: class_name_to_count[x],reverse=True)                
-                    
-            for cname in class_names:
-                ccount = len(images_html['class_{}'.format(cname)])
-                if ccount > 0:
-                    index_page += '<a href="class_{}.html">{}</a> ({})<br/>\n'.format(
-                        cname, cname.lower(), ccount)
-            index_page += '</div>\n'
-
-        index_page += '</body></html>'
-        output_html_file = os.path.join(output_dir, 'index.html')
-        with open(output_html_file, 'w') as f:
-            f.write(index_page)
-
-        print('Finished writing html to {}'.format(output_html_file))
-
-        # os.startfile(output_html_file)
-
-    # ...if we do/don't have ground truth
-
-    ppresults.output_html_file = output_html_file
-    return ppresults
-
-# ...process_batch_results
-
-
-#%% Interactive driver(s)
-
-if False:
-
-    #%%
-
-    base_dir = r'g:\temp'
-    options = PostProcessingOptions()
-    options.image_base_dir = base_dir
-    options.output_dir = os.path.join(base_dir, 'preview')
-    options.api_output_file = os.path.join(base_dir, 'results.json')
-    options.confidence_threshold = {'person':0.5,'animal':0.5,'vehicle':0.01}
-    options.include_almost_detections = True
-    options.almost_detection_confidence_threshold = 0.001
-
-    ppresults = process_batch_results(options)
-    # from md_utils.path_utils import open_file; open_file(ppresults.output_html_file)
-
-
-#%% Command-line driver
-
-def main():
-    
-    options = PostProcessingOptions()
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        'api_output_file',
-        help='path to .json file produced by the batch inference API')
-    parser.add_argument(
-        'output_dir',
-        help='base directory for output')
-    parser.add_argument(
-        '--image_base_dir', default=options.image_base_dir,
-        help='base directory for images (optional, can compute statistics '
-             'without images)')
-    parser.add_argument(
-        '--ground_truth_json_file', default=options.ground_truth_json_file,
-        help='ground truth labels (optional, can render detections without '
-             'ground truth), in the COCO Camera Traps format')
-    parser.add_argument(
-        '--confidence_threshold', type=float,
-        default=options.confidence_threshold,
-        help='Confidence threshold for statistics and visualization')
-    parser.add_argument(
-        '--almost_detection_confidence_threshold', type=float,
-        default=options.almost_detection_confidence_threshold,
-        help='Almost-detection confidence threshold for statistics and visualization')
-    parser.add_argument(
-        '--target_recall', type=float, default=options.target_recall,
-        help='Target recall (for statistics only)')
-    parser.add_argument(
-        '--num_images_to_sample', type=int,
-        default=options.num_images_to_sample,
-        help='number of images to visualize, -1 for all images (default: 500)')
-    parser.add_argument(
-        '--viz_target_width', type=int, default=options.viz_target_width,
-        help='Output image width')
-    parser.add_argument(
-        '--include_almost_detections', action='store_true',
-        help='Include a separate category for images just above a second confidence threshold')
-    parser.add_argument(
-        '--html_sort_order', type=str, default='filename',
-        help='Sort order for output pages, should be one of [filename,confidence,random] (defaults to filename)')
-    parser.add_argument(
-        '--sort_by_confidence', action='store_true',
-        help='Sort output in decreasing order by confidence (defaults to sorting by filename)')
-    parser.add_argument(
-        '--n_cores', type=int, default=1,
-        help='Number of threads to use for rendering (default: 1)')
-    parser.add_argument(
-        '--parallelize_rendering_with_processes', 
-        action='store_true',
-        help='Should we use processes (instead of threads) for parallelization?')
-    parser.add_argument(
-        '--no_separate_detections_by_category', 
-        action='store_true',
-        help='Collapse all categories into just "detections" and "non-detections"')    
-    parser.add_argument(
-        '--open_output_file', 
-        action='store_true',
-        help='Open the HTML output file when finished')    
-    parser.add_argument(
-        '--max_figures_per_html_file', 
-        type=int, default=None,
-        help='Maximum number of images to put on a single HTML page')
-    
-    if len(sys.argv[1:]) == 0:
-        parser.print_help()
-        parser.exit()
-
-    args = parser.parse_args()
-    
-    if args.n_cores != 1:
-        assert (args.n_cores > 1), 'Illegal number of cores: {}'.format(args.n_cores)
-        if args.parallelize_rendering_with_processes:
-            args.parallelize_rendering_with_threads = False
-        args.parallelize_rendering = True
-        args.parallelize_rendering_n_cores = args.n_cores        
-
-    args_to_object(args, options)    
-
-    if args.no_separate_detections_by_category:
-        options.separate_detections_by_category = False
-    
-    ppresults = process_batch_results(options)
-    
-    if options.open_output_file:
-        path_utils.open_file(ppresults.output_html_file)
-
-if __name__ == '__main__':
-    main()