megadetector 5.0.9__py3-none-any.whl → 5.0.11__py3-none-any.whl

api/batch_processing/postprocessing/compare_batch_results.py

@@ -1,958 +0,0 @@
-"""
-
-compare_batch_results.py
-
-Compare sets of batch results; typically used to compare:
-
-* Results from different MegaDetector versions
-* Results before/after RDE
-* Results with/without augmentation
-
-Makes pairwise comparisons, but can take lists of results files (will perform 
-all pairwise comparisons).  Results are written to an HTML page that shows the number
-and nature of disagreements (in the sense of each image being a detection or non-detection), 
-with sample images for each category.
-
-"""
-
-#%% Imports
-
-import json
-import os
-import random
-import copy
-import urllib
-import itertools
-
-from tqdm import tqdm
-from functools import partial
-
-from multiprocessing.pool import ThreadPool
-from multiprocessing.pool import Pool
-
-from md_visualization import visualization_utils
-from md_utils.write_html_image_list import write_html_image_list
-from md_utils import path_utils
-
-
-#%% Constants and support classes
-    
-class PairwiseBatchComparisonOptions:
-    """
-    Defines the options used for a single pairwise comparison; a list of these
-    pairwise options sets is stored in the BatchComparisonsOptions class.
-    """
-    
-    #: First filename to compare
-    results_filename_a = None
-    
-    #: Second filename to compare
-    results_filename_b = None
-    
-    #: Description to use in the output HTML for filename A
-    results_description_a = None
-    
-    #: Description to use in the output HTML for filename B
-    results_description_b = None
-    
-    #: Per-class detection thresholds to use for filename A (including a 'default' threshold)
-    detection_thresholds_a = {'animal':0.15,'person':0.15,'vehicle':0.15,'default':0.15}
-    
-    #: Per-class detection thresholds to use for filename B (including a 'default' threshold)
-    detection_thresholds_b = {'animal':0.15,'person':0.15,'vehicle':0.15,'default':0.15}
-
-    #: Rendering threshold to use for all categories for filename A
-    rendering_confidence_threshold_a = 0.1
-    
-    #: Rendering threshold to use for all categories for filename B
-    rendering_confidence_threshold_b = 0.1
-
-# ...class PairwiseBatchComparisonOptions
-
-
-class BatchComparisonOptions:
-    """
-    Defines the options for a set of (possibly many) pairwise comparisons.
-    """
-    
-    #: Folder to which we should write HTML output
-    output_folder = None
-    
-    #: Base folder for images (which are specified as relative files)
-    image_folder = None
-    
-    #: Job name to use in the HTML output file
-    job_name = ''
-    
-    #: Maximum number of images to render for each category, where a "category" here is
-    #: "detections_a_only", "detections_b_only", etc., or None to render all images.
-    max_images_per_category = 1000
-    
-    #: Maximum number of images per HTML page (paginates if a category page goes beyond this),
-    #: or None to disable pagination.
-    max_images_per_page = None
-    
-    #: Colormap to use for detections in file A (maps detection categories to colors)
-    colormap_a = ['Red']
-    
-    #: Colormap to use for detections in file B (maps detection categories to colors)
-    colormap_b = ['RoyalBlue']
-
-    #: Process-based parallelization isn't supported yet; this must be "True"
-    parallelize_rendering_with_threads = True
-    
-    #: List of filenames to include in the comparison, or None to use all files
-    filenames_to_include = None
-    
-    #: Compare only detections/non-detections, ignore categories (still renders categories)
-    class_agnostic_comparison = False
-    
-    #: Width of images to render in the output HTML
-    target_width = 800
-    
-    #: Number of workers to use for rendering, or <=1 to disable parallelization
-    n_rendering_workers = 20
-    
-    #: Random seed for image sampling (not used if max_images_per_category is None)
-    random_seed = 0
-    
-    #: Whether to sort results by confidence; if this is False, sorts by filename
-    sort_by_confidence = False
-    
-    #: The expectation is that all results sets being compared will refer to the same images; if this
-    #: is True (default), we'll error if that's not the case, otherwise non-matching lists will just be
-    #: a warning.
-    error_on_non_matching_lists = True
-    
-    #: List of PairwiseBatchComparisonOptions that defines the comparisons we'll render.
-    pairwise_options = []
-    
-# ...class BatchComparisonOptions
-    
-
-class PairwiseBatchComparisonResults:
-    """
-    The results from a single pairwise comparison.
-    """
-    
-    #: String of HTML content suitable for rendering to an HTML file
-    html_content = None
-    
-    #: Possibly-modified version of the PairwiseBatchComparisonOptions supplied as input.
-    pairwise_options = None
-    
-    #: A dictionary with keys including:
-    #:
-    #: common_detections
-    #: common_non_detections
-    #: detections_a_only
-    #: detections_b_only
-    #: class_transitions
-    #
-    #: Each of these maps a filename to a two-element list (the image in set A, the image in set B).
-    categories_to_image_pairs = None
-
-# ...class PairwiseBatchComparisonResults
-    
-    
-class BatchComparisonResults:
-    """
-    The results from a set of pairwise comparisons
-    """
-    
-    #: Filename containing HTML output
-    html_output_file = None
-    
-    #: A list of PairwiseBatchComparisonResults
-    pairwise_results = None
-    
-# ...class BatchComparisonResults    
-
-
-main_page_style_header = """<head>
-    <style type="text/css">
-    a { text-decoration: none; }
-    body { font-family: segoe ui, calibri, "trebuchet ms", verdana, arial, sans-serif; }
-    div.contentdiv { margin-left: 20px; }
-    </style>
-    </head>"""
-
-main_page_header = '<html>\n{}\n<body>\n'.format(main_page_style_header)
-main_page_footer = '<br/><br/><br/></body></html>\n'
-
-
-#%% Comparison functions
-
-def _render_image_pair(fn,image_pairs,category_folder,options,pairwise_options):
-    """
-    Render two sets of results (i.e., a comparison) for a single image.
-    
-    Args:
-        fn (str): image filename
-        image_pairs (dict): dict mapping filenames to pairs of image dicts
-        category_folder (str): folder to which to render this image, typically 
-            "detections_a_only", "detections_b_only", etc.
-        options (BatchComparisonOptions): job options
-        pairwise_options (PairwiseBatchComparisonOptions): pairwise comparison options
-            
-    Returns:
-        str: rendered image filename            
-    """
-    
-    input_image_path = os.path.join(options.image_folder,fn)
-    assert os.path.isfile(input_image_path), 'Image {} does not exist'.format(input_image_path)
-    
-    im = visualization_utils.open_image(input_image_path)
-    image_pair = image_pairs[fn]
-    detections_a = image_pair[0]['detections']
-    detections_b = image_pair[1]['detections']
-    
-    custom_strings_a = [''] * len(detections_a)
-    custom_strings_b = [''] * len(detections_b)    
-    
-    # This function is often used to compare results before/after various merging
-    # steps, so we have some special-case formatting based on the "transferred_from"
-    # field generated in merge_detections.py.
-    for i_det,det in enumerate(detections_a):
-        if 'transferred_from' in det:
-            custom_strings_a[i_det] = '({})'.format(
-                det['transferred_from'].split('.')[0])
-    
-    for i_det,det in enumerate(detections_b):
-        if 'transferred_from' in det:
-            custom_strings_b[i_det] = '({})'.format(
-                det['transferred_from'].split('.')[0])
-    
-    if options.target_width is not None:
-        im = visualization_utils.resize_image(im, options.target_width)
-        
-    visualization_utils.render_detection_bounding_boxes(detections_a,im,
-        confidence_threshold=pairwise_options.rendering_confidence_threshold_a,
-        thickness=4,expansion=0,
-        colormap=options.colormap_a,
-        textalign=visualization_utils.TEXTALIGN_LEFT,
-        custom_strings=custom_strings_a)
-    visualization_utils.render_detection_bounding_boxes(detections_b,im,
-        confidence_threshold=pairwise_options.rendering_confidence_threshold_b,
-        thickness=2,expansion=0,
-        colormap=options.colormap_b,
-        textalign=visualization_utils.TEXTALIGN_RIGHT,
-        custom_strings=custom_strings_b)
-
-    output_image_fn = path_utils.flatten_path(fn)
-    output_image_path = os.path.join(category_folder,output_image_fn)
-    im.save(output_image_path)
-    return output_image_path
-
-# ...def _render_image_pair()
-
-
-def _pairwise_compare_batch_results(options,output_index,pairwise_options):
-    """
-    The main entry point for this module is compare_batch_results(), which calls 
-    this function for each pair of comparisons the caller has requested.  Generates an
-    HTML page for this comparison.  Returns a BatchComparisonResults object.
-    
-    Args:
-        options (BatchComparisonOptions): overall job options for this comparison group
-        output_index (int): a numeric index used for generating HTML titles    
-        pairwise_options (PairwiseBatchComparisonOptions): job options for this comparison
-        
-    Returns:
-        PairwiseBatchComparisonResults: the results of this pairwise comparison
-    """
-    
-    # pairwise_options is passed as a parameter here, and should not be specified
-    # in the options object.
-    assert options.pairwise_options is None
-    
-    if options.random_seed is not None:
-        random.seed(options.random_seed)
-
-    # Warn the user if some "detections" might not get rendered
-    max_classification_threshold_a = max(list(pairwise_options.detection_thresholds_a.values()))
-    max_classification_threshold_b = max(list(pairwise_options.detection_thresholds_b.values()))
-    
-    if pairwise_options.rendering_confidence_threshold_a > max_classification_threshold_a:
-        print('*** Warning: rendering threshold A ({}) is higher than max confidence threshold A ({}) ***'.format(
-            pairwise_options.rendering_confidence_threshold_a,max_classification_threshold_a))
-    
-    if pairwise_options.rendering_confidence_threshold_b > max_classification_threshold_b:
-        print('*** Warning: rendering threshold B ({}) is higher than max confidence threshold B ({}) ***'.format(
-            pairwise_options.rendering_confidence_threshold_b,max_classification_threshold_b))
-    
-
-    ##%% Validate inputs
-    
-    assert os.path.isfile(pairwise_options.results_filename_a), \
-        "Can't find results file {}".format(pairwise_options.results_filename_a)
-    assert os.path.isfile(pairwise_options.results_filename_b), \
-        "Can't find results file {}".format(pairwise_options.results_filename_b)
-    assert os.path.isdir(options.image_folder), \
-        "Can't find image folder {}".format(pairwise_options.image_folder)
-    os.makedirs(options.output_folder,exist_ok=True)
-    
-    
-    ##%% Load both result sets
-    
-    with open(pairwise_options.results_filename_a,'r') as f:
-        results_a = json.load(f)
-    
-    with open(pairwise_options.results_filename_b,'r') as f:
-        results_b = json.load(f)
-        
-    # Don't let path separators confuse things
-    for im in results_a['images']:
-        if 'file' in im:
-            im['file'] = im['file'].replace('\\','/')
-    for im in results_b['images']:
-        if 'file' in im:
-            im['file'] = im['file'].replace('\\','/')
-    
-    if not options.class_agnostic_comparison:
-        assert results_a['detection_categories'] == results_b['detection_categories'], \
-            "Cannot perform a class-sensitive comparison across results with different categories"
-            
-    detection_categories_a = results_a['detection_categories']
-    detection_categories_b = results_b['detection_categories']
-    
-    if pairwise_options.results_description_a is None:
-        if 'detector' not in results_a['info']:
-            print('No model metadata supplied for results-A, assuming MDv4')
-            pairwise_options.results_description_a = 'MDv4 (assumed)'
-        else:            
-            pairwise_options.results_description_a = results_a['info']['detector']
-    
-    if pairwise_options.results_description_b is None:
-        if 'detector' not in results_b['info']:
-            print('No model metadata supplied for results-B, assuming MDv4')
-            pairwise_options.results_description_b = 'MDv4 (assumed)'
-        else:            
-            pairwise_options.results_description_b = results_b['info']['detector']
-    
-    images_a = results_a['images']
-    images_b = results_b['images']
-    
-    filename_to_image_a = {im['file']:im for im in images_a}
-    filename_to_image_b = {im['file']:im for im in images_b}
-    
-    
-    ##%% Make sure they represent the same set of images
-    
-    filenames_a = [im['file'] for im in images_a]
-    filenames_b_set = set([im['file'] for im in images_b])
-    
-    if len(images_a) != len(images_b):
-        s = 'set A has {} images, set B has {}'.format(len(images_a),len(images_b))
-        if options.error_on_non_matching_lists:
-            raise ValueError(s)
-        else:
-            print('Warning: ' + s)
-    else:
-        if options.error_on_non_matching_lists:
-            for fn in filenames_a:
-                assert fn in filenames_b_set
-
-    assert len(filenames_a) == len(images_a)
-    assert len(filenames_b_set) == len(images_b)    
-    
-    if options.filenames_to_include is None:
-        filenames_to_compare = filenames_a
-    else:
-        filenames_to_compare = options.filenames_to_include
-    
-    ##%% Find differences
-    
-    # Each of these maps a filename to a two-element list (the image in set A, the image in set B)
-    #
-    # Right now, we only handle a very simple notion of class transition, where the detection
-    # of maximum confidence changes class *and* both images have an above-threshold detection.
-    common_detections = {}
-    common_non_detections = {}
-    detections_a_only = {}
-    detections_b_only = {}
-    class_transitions = {}
-        
-    # fn = filenames_to_compare[0]
-    for fn in tqdm(filenames_to_compare):
-    
-        if fn not in filename_to_image_b:
-            
-            # We shouldn't have gotten this far if error_on_non_matching_lists is set
-            assert not options.error_on_non_matching_lists
-            
-            print('Skipping filename {}, not in image set B'.format(fn))
-            continue
-        
-        im_a = filename_to_image_a[fn]
-        im_b = filename_to_image_b[fn]
-        
-        categories_above_threshold_a = set()
-
-        if not 'detections' in im_a or im_a['detections'] is None:
-            assert 'failure' in im_a and im_a['failure'] is not None
-            continue
-        
-        if not 'detections' in im_b or im_b['detections'] is None:
-            assert 'failure' in im_b and im_b['failure'] is not None
-            continue
-                
-        invalid_category_error = False
-
-        # det = im_a['detections'][0]
-        for det in im_a['detections']:
-            
-            category_id = det['category']
-            
-            if category_id not in detection_categories_a:
-                print('Warning: unexpected category {} for model A on file {}'.format(category_id,fn))
-                invalid_category_error = True
-                break
-                
-            conf = det['conf']
-            
-            if detection_categories_a[category_id] in pairwise_options.detection_thresholds_a:
-                conf_thresh = pairwise_options.detection_thresholds_a[detection_categories_a[category_id]]
-            else:
-                conf_thresh = pairwise_options.detection_thresholds_a['default']
-                
-            if conf >= conf_thresh:
-                categories_above_threshold_a.add(category_id)
-                            
-        if invalid_category_error:
-            continue
-        
-        categories_above_threshold_b = set()
-        
-        for det in im_b['detections']:
-            
-            category_id = det['category']
-            
-            if category_id not in detection_categories_b:
-                print('Warning: unexpected category {} for model B on file {}'.format(category_id,fn))
-                invalid_category_error = True
-                break
-            
-            conf = det['conf']
-            
-            if detection_categories_b[category_id] in pairwise_options.detection_thresholds_b:
-                conf_thresh = pairwise_options.detection_thresholds_b[detection_categories_b[category_id]]
-            else:
-                conf_thresh = pairwise_options.detection_thresholds_a['default']
-                
-            if conf >= conf_thresh:
-                categories_above_threshold_b.add(category_id)
-                            
-        if invalid_category_error:
-            continue
-        
-        im_pair = (im_a,im_b)
-        
-        detection_a = (len(categories_above_threshold_a) > 0)
-        detection_b = (len(categories_above_threshold_b) > 0)
-                
-        if detection_a and detection_b:            
-            if (categories_above_threshold_a == categories_above_threshold_b) or \
-                options.class_agnostic_comparison:
-                common_detections[fn] = im_pair
-            else:
-                class_transitions[fn] = im_pair
-        elif (not detection_a) and (not detection_b):
-            common_non_detections[fn] = im_pair
-        elif detection_a and (not detection_b):
-            detections_a_only[fn] = im_pair
-        else:
-            assert detection_b and (not detection_a)
-            detections_b_only[fn] = im_pair
-            
-    # ...for each filename
-    
-    print('Of {} files:\n{} common detections\n{} common non-detections\n{} A only\n{} B only\n{} class transitions'.format(
-        len(filenames_to_compare),len(common_detections),
-        len(common_non_detections),len(detections_a_only),
-        len(detections_b_only),len(class_transitions)))
-        
-    
-    ##%% Sample and plot differences
-    
-    if options.n_rendering_workers > 1:
-       worker_type = 'processes'
-       if options.parallelize_rendering_with_threads:
-           worker_type = 'threads'
-       print('Rendering images with {} {}'.format(options.n_rendering_workers,worker_type))
-       if options.parallelize_rendering_with_threads:
-           pool = ThreadPool(options.n_rendering_workers)    
-       else:
-           pool = Pool(options.n_rendering_workers)    
-        
-    categories_to_image_pairs = {
-        'common_detections':common_detections,
-        'common_non_detections':common_non_detections,
-        'detections_a_only':detections_a_only,
-        'detections_b_only':detections_b_only,
-        'class_transitions':class_transitions
-    }
-    
-    categories_to_page_titles = {
-        'common_detections':'Detections common to both models',
-        'common_non_detections':'Non-detections common to both models',
-        'detections_a_only':'Detections reported by model A only',
-        'detections_b_only':'Detections reported by model B only',
-        'class_transitions':'Detections reported as different classes by models A and B'
-    }
-
-    local_output_folder = os.path.join(options.output_folder,'cmp_' + \
-                                       str(output_index).zfill(3))
-    
-    def render_detection_comparisons(category,image_pairs,image_filenames):
-        
-        print('Rendering detections for category {}'.format(category))
-                
-        category_folder = os.path.join(local_output_folder,category)
-        os.makedirs(category_folder,exist_ok=True)
-        
-        # fn = image_filenames[0]
-        if options.n_rendering_workers <= 1:
-            output_image_paths = []
-            for fn in tqdm(image_filenames):        
-                output_image_paths.append(_render_image_pair(fn,image_pairs,category_folder,
-                                                            options,pairwise_options))
-        else:            
-            output_image_paths = list(tqdm(pool.imap(
-                partial(_render_image_pair, image_pairs=image_pairs, 
-                        category_folder=category_folder,options=options,
-                        pairwise_options=pairwise_options),
-                image_filenames), 
-                total=len(image_filenames)))
-        
-        return output_image_paths
-    
-    # ...def render_detection_comparisons()
-    
-    # For each category, generate comparison images and the 
-    # comparison HTML page.
-    #
-    # category = 'common_detections'
-    for category in categories_to_image_pairs.keys():
-        
-        # Choose detection pairs we're going to render for this category
-        image_pairs = categories_to_image_pairs[category]
-        image_filenames = list(image_pairs.keys())
-        
-        if options.max_images_per_category is not None and options.max_images_per_category > 0:
-            if len(image_filenames) > options.max_images_per_category:
-                print('Sampling {} of {} image pairs for category {}'.format(
-                    options.max_images_per_category,
-                    len(image_filenames),
-                    category))
-                image_filenames = random.sample(image_filenames,
-                                                options.max_images_per_category)
-            assert len(image_filenames) <= options.max_images_per_category
-
-        input_image_absolute_paths = [os.path.join(options.image_folder,fn) for fn in image_filenames]
-        
-        category_image_output_paths = render_detection_comparisons(category,
-                                                            image_pairs,image_filenames)
-        
-        category_html_filename = os.path.join(local_output_folder,
-                                              category + '.html')
-        category_image_output_paths_relative = [os.path.relpath(s,local_output_folder) \
-                                         for s in category_image_output_paths]
-        
-        image_info = []
-        
-        assert len(category_image_output_paths_relative) == len(input_image_absolute_paths)
-        
-        for i_fn,fn in enumerate(category_image_output_paths_relative): 
-            
-            input_path_relative = image_filenames[i_fn]
-            image_pair = image_pairs[input_path_relative]
-            assert len(image_pair) == 2; image_a = image_pair[0]; image_b = image_pair[1]
-            
-            def maxempty(L):
-                if len(L) == 0:
-                    return 0
-                else:
-                    return max(L)
-                
-            max_conf_a = maxempty([det['conf'] for det in image_a['detections']])
-            max_conf_b = maxempty([det['conf'] for det in image_b['detections']])
-            
-            title = input_path_relative + ' (max conf {:.2f},{:.2f})'.format(max_conf_a,max_conf_b)
-            
-            # Only used if sort_by_confidence is True
-            if category == 'common_detections':
-                sort_conf = max(max_conf_a,max_conf_b)
-            elif category == 'common_non_detections':
-                sort_conf = max(max_conf_a,max_conf_b)
-            elif category == 'detections_a_only':
-                sort_conf = max_conf_a
-            elif category == 'detections_b_only':
-                sort_conf = max_conf_b
-            elif category == 'class_transitions':
-                sort_conf = max(max_conf_a,max_conf_b)
-            else:
-                print('Warning: unknown sort category {}'.format(category))
-                sort_conf = max(max_conf_a,max_conf_b)
-                
-            info = {
-                'filename': fn,
-                'title': title,
-                'textStyle': 'font-family:verdana,arial,calibri;font-size:' + \
-                    '80%;text-align:left;margin-top:20;margin-bottom:5',
-                'linkTarget': urllib.parse.quote(input_image_absolute_paths[i_fn]),
-                'sort_conf':sort_conf
-            }
-            image_info.append(info)
-    
-        # ...for each image
-        
-        category_page_header_string = '<h1>{}</h1>'.format(categories_to_page_titles[category])
-        category_page_header_string += '<p style="font-weight:bold;">\n'
-        category_page_header_string += 'Model A: {}<br/>\n'.format(
-            pairwise_options.results_description_a)
-        category_page_header_string += 'Model B: {}'.format(pairwise_options.results_description_b)
-        category_page_header_string += '</p>\n'
-        
-        category_page_header_string += '<p>\n'
-        category_page_header_string += 'Detection thresholds for A ({}):\n{}<br/>'.format(
-            pairwise_options.results_description_a,str(pairwise_options.detection_thresholds_a))
-        category_page_header_string += 'Detection thresholds for B ({}):\n{}<br/>'.format(
-            pairwise_options.results_description_b,str(pairwise_options.detection_thresholds_b))
-        category_page_header_string += 'Rendering threshold for A ({}):\n{}<br/>'.format(
-            pairwise_options.results_description_a,
-            str(pairwise_options.rendering_confidence_threshold_a))
-        category_page_header_string += 'Rendering threshold for B ({}):\n{}<br/>'.format(
-            pairwise_options.results_description_b,
-            str(pairwise_options.rendering_confidence_threshold_b))
-        category_page_header_string += '</p>\n'        
-        
-        # Default to sorting by filename
-        if options.sort_by_confidence:
-            image_info = sorted(image_info, key=lambda d: d['sort_conf'], reverse=True)
-        else:
-            image_info = sorted(image_info, key=lambda d: d['filename'])
-            
-        write_html_image_list(
-            category_html_filename,
-            images=image_info,
-            options={
-                'headerHtml': category_page_header_string,
-                'maxFiguresPerHtmlFile': options.max_images_per_page
-            })
-        
-    # ...for each category
-    
-    
-    ##%% Write the top-level HTML file content
-
-    html_output_string  = ''
-    
-    html_output_string += '<p>Comparing <b>{}</b> (A, red) to <b>{}</b> (B, blue)</p>'.format(
-        pairwise_options.results_description_a,pairwise_options.results_description_b)
-    html_output_string += '<div class="contentdiv">\n'
-    html_output_string += 'Detection thresholds for {}:\n{}<br/>'.format(
-        pairwise_options.results_description_a,
-        str(pairwise_options.detection_thresholds_a))
-    html_output_string += 'Detection thresholds for {}:\n{}<br/>'.format(
-        pairwise_options.results_description_b,
-        str(pairwise_options.detection_thresholds_b))
-    html_output_string += 'Rendering threshold for {}:\n{}<br/>'.format(
-        pairwise_options.results_description_a,
-        str(pairwise_options.rendering_confidence_threshold_a))
-    html_output_string += 'Rendering threshold for {}:\n{}<br/>'.format(
-        pairwise_options.results_description_b,
-        str(pairwise_options.rendering_confidence_threshold_b))
-    
-    html_output_string += '<br/>'
-    
-    html_output_string += 'Rendering a maximum of {} images per category<br/>'.format(
-        options.max_images_per_category)
-    
-    html_output_string += '<br/>'
-    
-    html_output_string += ('Of {} total files:<br/><br/><div style="margin-left:15px;">{} common detections<br/>{} common non-detections<br/>{} A only<br/>{} B only<br/>{} class transitions</div><br/>'.format(
-        len(filenames_to_compare),len(common_detections),
-        len(common_non_detections),len(detections_a_only),
-        len(detections_b_only),len(class_transitions)))
-    
-    html_output_string += 'Comparison pages:<br/><br/>\n'
-    html_output_string += '<div style="margin-left:15px;">\n'
-        
-    comparison_path_relative = os.path.relpath(local_output_folder,options.output_folder)    
-    for category in categories_to_image_pairs.keys():
-        category_html_filename = os.path.join(comparison_path_relative,category + '.html')
-        html_output_string += '<a href="{}">{}</a><br/>\n'.format(
-            category_html_filename,category)
-    
-    html_output_string += '</div>\n'
-    html_output_string += '</div>\n'
-    
-    pairwise_results = PairwiseBatchComparisonResults()
-    
-    pairwise_results.html_content = html_output_string
-    pairwise_results.pairwise_options = pairwise_options
-    pairwise_results.categories_to_image_pairs = categories_to_image_pairs
-            
-    return pairwise_results
-        
-# ...def _pairwise_compare_batch_results()
-
-
-def compare_batch_results(options):
-    """
-    The main entry point for this module.  Runs one or more batch results comparisons, 
-    writing results to an html page.  Most of the work is deferred to _pairwise_compare_batch_results().
-    
-    Args:
-        options (BatchComparisonOptions): job options to use for this comparison task, including the
-            list of specific pairswise comparisons to make (in the pairwise_options field)
-            
-    Returns:
-        BatchComparisonResults: the results of this comparison task
-    """
-    
-    assert options.output_folder is not None
-    assert options.image_folder is not None
-    assert options.pairwise_options is not None
-
-    options = copy.deepcopy(options)
- 
-    if not isinstance(options.pairwise_options,list):
-        options.pairwise_options = [options.pairwise_options]
-    
-    pairwise_options_list = options.pairwise_options
-    n_comparisons = len(pairwise_options_list)
-    
-    options.pairwise_options = None
-    
-    html_content = ''
-    all_pairwise_results = []
-    
-    # i_comparison = 0; pairwise_options = pairwise_options_list[i_comparison]
-    for i_comparison,pairwise_options in enumerate(pairwise_options_list):
-        print('Running comparison {} of {}'.format(i_comparison,n_comparisons))
-        pairwise_results = \
-            _pairwise_compare_batch_results(options,i_comparison,pairwise_options)
-        html_content += pairwise_results.html_content
-        all_pairwise_results.append(pairwise_results)
-
-    html_output_string = main_page_header
-    job_name_string = ''
-    if len(options.job_name) > 0:
-        job_name_string = ' for {}'.format(options.job_name)
-    html_output_string += '<h2>Comparison of results{}</h2>\n'.format(
-        job_name_string)
-    html_output_string += html_content
-    html_output_string += main_page_footer
-    
-    html_output_file = os.path.join(options.output_folder,'index.html')    
-    with open(html_output_file,'w') as f:
-        f.write(html_output_string) 
-    
-    results = BatchComparisonResults()
-    results.html_output_file = html_output_file
-    results.pairwise_results = all_pairwise_results
-    return results
-
-
-def n_way_comparison(filenames,options,detection_thresholds=None,rendering_thresholds=None):
-    """
-    Performs N pairwise comparisons for the list of results files in [filenames], by generating
-    sets of pairwise options and calling compare_batch_results.
-    
-    Args:
-        filenames (list): list of MD results filenames to compare
-        options (BatchComparisonOptions): task options set in which pairwise_options is still 
-            empty; that will get populated from [filenames]
-        detection_thresholds (list, optional): list of detection thresholds with the same length
-            as [filenames], or None to use sensible defaults
-        rendering_thresholds (list, optional): list of rendering thresholds with the same length
-            as [filenames], or None to use sensible defaults
-    
-    Returns:
-        BatchComparisonResults: the results of this comparison task
-    """
-    
-    if detection_thresholds is None:
-        detection_thresholds = [0.15] * len(filenames)
-    assert len(detection_thresholds) == len(filenames)
-
-    if rendering_thresholds is not None:
-        assert len(rendering_thresholds) == len(detection_thresholds)
-    else:
-        rendering_thresholds = [(x*0.6666) for x in detection_thresholds]
-
-    # Choose all pairwise combinations of the files in [filenames]
-    for i, j in itertools.combinations(list(range(0,len(filenames))),2):
-            
-        pairwise_options = PairwiseBatchComparisonOptions()
-        
-        pairwise_options.results_filename_a = filenames[i]
-        pairwise_options.results_filename_b = filenames[j]
-        
-        pairwise_options.rendering_confidence_threshold_a = rendering_thresholds[i]
-        pairwise_options.rendering_confidence_threshold_b = rendering_thresholds[j]
-        
-        pairwise_options.detection_thresholds_a = {'default':detection_thresholds[i]}
-        pairwise_options.detection_thresholds_b = {'default':detection_thresholds[j]}
-        
-        options.pairwise_options.append(pairwise_options)
-
-    return compare_batch_results(options)
-
-# ...n_way_comparison()
-
-
-#%% Interactive driver
-
-if False:
-        
-    #%% Test two-way comparison
-    
-    options = BatchComparisonOptions()
-
-    options.parallelize_rendering_with_threads = True
-    
-    options.job_name = 'BCT'
-    options.output_folder = r'g:\temp\comparisons'
-    options.image_folder = r'g:\camera_traps\camera_trap_images'
-    options.max_images_per_category = 100
-    options.sort_by_confidence = True
-    
-    options.pairwise_options = []
-
-    results_base = os.path.expanduser('~/postprocessing/bellevue-camera-traps')
-    filenames = [                
-        os.path.join(results_base,r'bellevue-camera-traps-2023-12-05-v5a.0.0\combined_api_outputs\bellevue-camera-traps-2023-12-05-v5a.0.0_detections.json'),
-        os.path.join(results_base,r'bellevue-camera-traps-2023-12-05-aug-v5a.0.0\combined_api_outputs\bellevue-camera-traps-2023-12-05-aug-v5a.0.0_detections.json')
-        ]
-
-    detection_thresholds = [0.15,0.15]
-    rendering_thresholds = None
-    
-    results = n_way_comparison(filenames,options,detection_thresholds,rendering_thresholds=rendering_thresholds)
-    
-    from md_utils.path_utils import open_file
-    open_file(results.html_output_file)
-
-    
-    #%% Test three-way comparison
-    
-    options = BatchComparisonOptions()
-
-    options.parallelize_rendering_with_threads = False
-
-    options.job_name = 'KGA-test'
-    options.output_folder = os.path.expanduser('~/tmp/md-comparison-test')
-    options.image_folder = os.path.expanduser('~/data/KGA')
-
-    options.pairwise_options = []
-
-    filenames = [
-        os.path.expanduser('~/data/KGA-4.json'),
-        os.path.expanduser('~/data/KGA-5a.json'),
-        os.path.expanduser('~/data/KGA-5b.json')
-        ]
-
-    detection_thresholds = [0.7,0.15,0.15]
-
-    results = n_way_comparison(filenames,options,detection_thresholds,rendering_thresholds=None)
-    
-    from md_utils.path_utils import open_file
-    open_file(results.html_output_file)
-
-
-#%% Command-line driver
-
-"""
-python compare_batch_results.py ~/tmp/comparison-test ~/data/KGA ~/data/KGA-5a.json ~/data/KGA-5b.json ~/data/KGA-4.json --detection_thresholds 0.15 0.15 0.7 --rendering_thresholds 0.1 0.1 0.6 --use_processes
-"""
-
-import sys,argparse,textwrap
-
-def main():
-    
-    options = BatchComparisonOptions()
-    
-    parser = argparse.ArgumentParser(
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog=textwrap.dedent('''\
-           Example:
-        
-           python compare_batch_results.py output_folder image_folder mdv5a.json mdv5b.json mdv4.json --detection_thresholds 0.15 0.15 0.7
-           '''))
-          
-    parser.add_argument('output_folder', type=str, help='folder to which to write html results')
-
-    parser.add_argument('image_folder', type=str, help='image source folder')
-
-    parser.add_argument('results_files', nargs='*', type=str, help='list of .json files to be compared')
-
-    parser.add_argument('--detection_thresholds', nargs='*', type=float,
-                        help='list of detection thresholds, same length as the number of .json files, ' + \
-                             'defaults to 0.15 for all files')
-    
-    parser.add_argument('--rendering_thresholds', nargs='*', type=float,
-                        help='list of rendering thresholds, same length as the number of .json files, ' + \
-                             'defaults to 0.10 for all files')
-    
-    parser.add_argument('--max_images_per_category', type=int, default=options.max_images_per_category,
-                       help='number of images to sample for each agreement category (common detections, etc.)')
-    
-    parser.add_argument('--target_width', type=int, default=options.target_width, 
-                        help='output image width, defaults to {}'.format(options.target_width))
-    
-    parser.add_argument('--use_processes', action='store_true', 
-                        help='use processes rather than threads for parallelization')
-    
-    parser.add_argument('--open_results', action='store_true', 
-                        help='open the output html file when done')
-    
-    parser.add_argument('--n_rendering_workers', type=int, default=options.n_rendering_workers,
-                        help='number of workers for parallel rendering, defaults to {}'.format(
-                            options.n_rendering_workers))
-        
-    if len(sys.argv[1:])==0:
-        parser.print_help()
-        parser.exit()
-        
-    args = parser.parse_args()
-    
-    print('Output folder:')
-    print(args.output_folder)
-    
-    print('\nResults files:')
-    print(args.results_files)
-    
-    print('\nDetection thresholds:')
-    print(args.detection_thresholds)
-    
-    print('\nRendering thresholds:')
-    print(args.rendering_thresholds)    
-    
-    # Convert to options objects
-    options = BatchComparisonOptions()
-    
-    options.output_folder = args.output_folder
-    options.image_folder = args.image_folder
-    options.target_width = args.target_width
-    options.n_rendering_workers = args.n_rendering_workers    
-    options.max_images_per_category = args.max_images_per_category
-    
-    if args.use_processes:
-        options.parallelize_rendering_with_threads = False
-        
-    results = n_way_comparison(args.results_files,options,args.detection_thresholds,args.rendering_thresholds)
-    
-    if args.open_results:
-        path_utils.open_file(results.html_output_file)
-        
-    print('Wrote results to {}'.format(results.html_output_file))
-    
-# ...main()
-
-    
-if __name__ == '__main__':
-    
-    main()