megadetector 5.0.15__py3-none-any.whl → 5.0.17__py3-none-any.whl

megadetector/utils/md_tests.py

@@ -29,6 +29,10 @@ import subprocess
 import argparse
 import inspect
 
+#: IoU threshold used to determine whether boxes in two detection files likely correspond
+#: to the same box.
+iou_threshold_for_file_comparison = 0.9
+
 
 #%% Classes
 
@@ -359,6 +363,100 @@ def output_files_are_identical(fn1,fn2,verbose=False):
 # ...def output_files_are_identical(...)
    
 
+def compare_detection_lists(detections_a,detections_b,options,bidirectional_comparison=True):
+    """
+    Compare two lists of MD-formatted detections, matching detections across lists using IoU
+    criteria.  Generally used to compare detections for the same image when two sets of results 
+    are expected to be more or less the same.
+    
+    Args:
+        detections_a (list): the first set of detection dicts
+        detections_b (list): the second set of detection dicts
+        options (MDTestOptions): options that determine tolerable differences between files    
+        bidirectional_comparison (bool, optional): reverse the arguments and make a recursive
+            call.
+            
+    Returns:
+        dict: a dictionary with keys 'max_conf_error' and 'max_coord_error'.
+    """
+    from megadetector.utils.ct_utils import get_iou
+    
+    max_conf_error = 0
+    max_coord_error = 0
+        
+    # i_det_a = 0
+    for i_det_a in range(0,len(detections_a)):
+        
+        det_a = detections_a[i_det_a]
+        
+        # Don't process very-low-confidence boxes
+        if det_a['conf'] < options.max_conf_error:
+            continue
+        
+        matching_det_b = None
+        highest_iou = -1
+        
+        # Find the closest match in the detections_b list
+        
+        # i_det_b = 0
+        for i_det_b in range(0,len(detections_b)):
+            
+            b_det = detections_b[i_det_b]
+            
+            if b_det['category'] != det_a['category']:
+                continue
+            
+            iou = get_iou(det_a['bbox'],b_det['bbox'])
+            
+            # Is this likely the same detection as det_a?
+            if iou >= iou_threshold_for_file_comparison and iou > highest_iou:
+                matching_det_b = b_det
+                highest_iou = iou
+                
+        # If there are no detections in this category in detections_b
+        if matching_det_b is None:
+            if det_a['conf'] > max_conf_error:
+                max_conf_error = det_a['conf']
+            # max_coord_error = 1.0
+            continue
+        
+        assert det_a['category'] == matching_det_b['category']
+        conf_err = abs(det_a['conf'] - matching_det_b['conf'])
+        coord_differences = []
+        for i_coord in range(0,4):
+            coord_differences.append(abs(det_a['bbox'][i_coord]-\
+                                         matching_det_b['bbox'][i_coord]))
+        coord_err = max(coord_differences)
+        
+        if conf_err >= max_conf_error:
+            max_conf_error = conf_err
+            
+        if coord_err >= max_coord_error:
+            max_coord_error = coord_err            
+    
+    # ...for each detection in detections_a
+    
+    if bidirectional_comparison:
+        
+        reverse_comparison_results = compare_detection_lists(detections_b,
+                                                             detections_a, 
+                                                             options, 
+                                                             bidirectional_comparison=False)
+        
+        if reverse_comparison_results['max_conf_error'] > max_conf_error:
+            max_conf_error = reverse_comparison_results['max_conf_error']
+        if reverse_comparison_results['max_coord_error'] > max_coord_error:
+            max_coord_error = reverse_comparison_results['max_coord_error']
+    
+    list_comparison_results = {}
+    list_comparison_results['max_coord_error'] = max_coord_error
+    list_comparison_results['max_conf_error'] = max_conf_error
+    
+    return list_comparison_results
+
+# ...def compare_detection_lists(...)
+
+
 def compare_results(inference_output_file,expected_results_file,options):
     """
     Compare two MD-formatted output files that should be nearly identical, allowing small
@@ -369,6 +467,9 @@ def compare_results(inference_output_file,expected_results_file,options):
         inference_output_file (str): the first results file to compare
         expected_results_file (str): the second results file to compare
         options (MDTestOptions): options that determine tolerable differences between files
+        
+    Returns:
+        dict: dictionary with keys 'max_coord_error' and 'max_conf_error'
     """
     
     # Read results
@@ -386,8 +487,11 @@ def compare_results(inference_output_file,expected_results_file,options):
             len(filename_to_results_expected),
             len(filename_to_results))
     
-    max_coord_error = 0
     max_conf_error = 0
+    max_conf_error_file = None
+    
+    max_coord_error = 0
+    max_coord_error_file = None    
     
     # fn = next(iter(filename_to_results.keys()))
     for fn in filename_to_results.keys():
@@ -405,51 +509,43 @@ def compare_results(inference_output_file,expected_results_file,options):
         actual_detections = actual_image_results['detections']
         expected_detections = expected_image_results['detections']
         
-        s = 'expected {} detections for file {}, found {}'.format(
-            len(expected_detections),fn,len(actual_detections))
-        s += '\nExpected results file: {}\nActual results file: {}'.format(
-            expected_results_file,inference_output_file)
+        comparison_results_this_image = compare_detection_lists(
+            detections_a=actual_detections,
+            detections_b=expected_detections,
+            options=options,
+            bidirectional_comparison=True)
         
-        if options.warning_mode:
-            if len(actual_detections) != len(expected_detections):
-                print('Warning: {}'.format(s))
-            continue
-        assert len(actual_detections) == len(expected_detections), \
-            'Error: {}'.format(s)
-        
-        # i_det = 0
-        for i_det in range(0,len(actual_detections)):
-            actual_det = actual_detections[i_det]
-            expected_det = expected_detections[i_det]
-            assert actual_det['category'] == expected_det['category']
-            conf_err = abs(actual_det['conf'] - expected_det['conf'])
-            coord_differences = []
-            for i_coord in range(0,4):
-                coord_differences.append(abs(actual_det['bbox'][i_coord]-expected_det['bbox'][i_coord]))
-            coord_err = max(coord_differences)
+        if comparison_results_this_image['max_conf_error'] > max_conf_error:
+            max_conf_error = comparison_results_this_image['max_conf_error']
+            max_conf_error_file = fn
             
-            if conf_err > max_conf_error:
-                max_conf_error = conf_err
-            if coord_err > max_coord_error:
-                max_coord_error = coord_err
-        
-        # ...for each detection
-        
+        if comparison_results_this_image['max_coord_error'] > max_coord_error:
+            max_coord_error = comparison_results_this_image['max_coord_error']
+            max_coord_error_file = fn
+                
     # ...for each image
     
     if not options.warning_mode:
         
         assert max_conf_error <= options.max_conf_error, \
-            'Confidence error {} is greater than allowable ({})'.format(
-                max_conf_error,options.max_conf_error)
+            'Confidence error {} is greater than allowable ({}), on file:\n{}'.format(
+                max_conf_error,options.max_conf_error,max_conf_error_file)
         
         assert max_coord_error <= options.max_coord_error, \
-            'Coord error {} is greater than allowable ({})'.format(
-                max_coord_error,options.max_coord_error)
+            'Coord error {} is greater than allowable ({}), on file:\n{}'.format(
+                max_coord_error,options.max_coord_error,max_coord_error_file)
         
-    print('Max conf error: {}'.format(max_conf_error))
-    print('Max coord error: {}'.format(max_coord_error))
+    print('Max conf error: {} (file {})'.format(
+        max_conf_error,max_conf_error_file))
+    print('Max coord error: {} (file {})'.format(
+        max_coord_error,max_coord_error_file))
     
+    comparison_results = {}
+    comparison_results['max_conf_error'] = max_conf_error
+    comparison_results['max_coord_error'] = max_coord_error
+
+    return comparison_results
+
 # ...def compare_results(...)
 
 
@@ -557,6 +653,8 @@ def run_python_tests(options):
     
     ## Run inference on an image
     
+    print('\n** Running MD on a single image (module) **\n')
+    
     from megadetector.detection import run_detector
     from megadetector.visualization import visualization_utils as vis_utils
     image_fn = os.path.join(options.scratch_dir,options.test_images[0])
@@ -567,7 +665,7 @@ def run_python_tests(options):
     
     ## Run inference on a folder
 
-    print('\n** Running MD on images **\n')
+    print('\n** Running MD on a folder of images (module) **\n')
     
     from megadetector.detection.run_detector_batch import load_and_run_detector_batch,write_results_to_file
     from megadetector.utils import path_utils
@@ -584,15 +682,18 @@ def run_python_tests(options):
 
     
     ## Verify results
-
+    
     expected_results_file = get_expected_results_filename(is_gpu_available(verbose=False),
                                                           options=options)
     compare_results(inference_output_file,expected_results_file,options)
         
-
+    # Make note of this filename, we will use it again later
+    inference_output_file_standard_inference = inference_output_file
+    
+    
     ## Run and verify again with augmentation enabled
     
-    print('\n** Running MD on images with augmentation **\n')
+    print('\n** Running MD on images with augmentation (module) **\n')
     
     from megadetector.utils.path_utils import insert_before_extension
     
@@ -611,6 +712,8 @@ def run_python_tests(options):
     
     ## Postprocess results
     
+    print('\n** Post-processing results (module) **\n')
+    
     from megadetector.postprocessing.postprocess_batch_results import \
         PostProcessingOptions,process_batch_results
     postprocessing_options = PostProcessingOptions()
@@ -626,6 +729,8 @@ def run_python_tests(options):
         
     ## Partial RDE test
     
+    print('\n** Testing RDE (module) **\n')
+    
     from megadetector.postprocessing.repeat_detection_elimination.repeat_detections_core import \
         RepeatDetectionOptions, find_repeat_detections
     
@@ -648,9 +753,12 @@ def run_python_tests(options):
         print('Skipping YOLO val inference tests, no YOLO folder supplied')
         
     else:
-            
+        
+        print('\n** Running YOLO val inference test (module) **\n')
+        
         from megadetector.detection.run_inference_with_yolov5_val import \
             YoloInferenceOptions, run_inference_with_yolo_val
+        from megadetector.utils.path_utils import insert_before_extension
         
         inference_output_file_yolo_val = os.path.join(options.scratch_dir,'folder_inference_output_yolo_val.json')
         
@@ -661,12 +769,21 @@ def run_python_tests(options):
         yolo_inference_options.model_filename = options.default_model
         yolo_inference_options.augment = False
         yolo_inference_options.overwrite_handling = 'overwrite'
+        from megadetector.detection.run_detector import DEFAULT_OUTPUT_CONFIDENCE_THRESHOLD
+        yolo_inference_options.conf_thres = DEFAULT_OUTPUT_CONFIDENCE_THRESHOLD
         
         run_inference_with_yolo_val(yolo_inference_options)
         
+        ## Confirm this matches the standard inference path
+        
+        if False:
+            # TODO: compare_results() isn't quite ready for this yet
+            compare_results(inference_output_file=inference_output_file_yolo_val, 
+                            expected_results_file=inference_output_file_standard_inference, 
+                            options=options)
+                        
         # Run again, without symlinks this time
         
-        from megadetector.utils.path_utils import insert_before_extension
         inference_output_file_yolo_val_no_links = insert_before_extension(inference_output_file_yolo_val,
                                                                           'no-links')
         yolo_inference_options.output_file = inference_output_file_yolo_val_no_links
@@ -709,9 +826,10 @@ def run_python_tests(options):
         
         ## Video test (single video)
        
-        print('\n** Running MD on a single video **\n')
+        print('\n** Running MD on a single video (module) **\n')
         
         from megadetector.detection.process_video import ProcessVideoOptions, process_video
+        from megadetector.utils.path_utils import insert_before_extension
         
         video_options = ProcessVideoOptions()
         video_options.model_file = options.default_model
@@ -747,25 +865,26 @@ def run_python_tests(options):
         
         ## Video test (folder)
         
-        print('\n** Running MD on a folder of videos **\n')
+        print('\n** Running MD on a folder of videos (module) **\n')
         
         from megadetector.detection.process_video import ProcessVideoOptions, process_video_folder
+        from megadetector.utils.path_utils import insert_before_extension
         
         video_options = ProcessVideoOptions()
         video_options.model_file = options.default_model
         video_options.input_video_file = os.path.join(options.scratch_dir,
                                                       os.path.dirname(options.test_videos[0]))
         video_options.output_json_file = os.path.join(options.scratch_dir,'video_folder_output.json')
-        # video_options.output_video_file = None
+        video_options.output_video_file = None
         video_options.frame_folder = os.path.join(options.scratch_dir,'video_scratch/frame_folder')
         video_options.frame_rendering_folder = os.path.join(options.scratch_dir,'video_scratch/rendered_frame_folder')    
         video_options.render_output_video = False
-        # video_options.keep_rendered_frames = False
-        # video_options.keep_rendered_frames = False
+        video_options.keep_rendered_frames = False
+        video_options.keep_rendered_frames = False
         video_options.force_extracted_frame_folder_deletion = True
         video_options.force_rendered_frame_folder_deletion = True
-        # video_options.reuse_results_if_available = False
-        # video_options.reuse_frames_if_available = False
+        video_options.reuse_results_if_available = False
+        video_options.reuse_frames_if_available = False
         video_options.recursive = True
         video_options.verbose = True
         video_options.fourcc = options.video_fourcc
@@ -773,12 +892,16 @@ def run_python_tests(options):
         # video_options.json_confidence_threshold = 0.005
         video_options.frame_sample = 10  
         video_options.n_cores = 5        
+        
+        # Force frame extraction to disk, since that's how we generated our expected results file
+        video_options.force_on_disk_frame_extraction = True
         # video_options.debug_max_frames = -1
         # video_options.class_mapping_filename = None
         
         # Use quality == None, because we can't control whether YOLOv5 has patched cm2.imread,
         # and therefore can't rely on using the quality parameter
-        video_options.quality = None        
+        video_options.quality = None
+        video_options.max_width = None
         
         _ = process_video_folder(video_options)
     
@@ -796,6 +919,29 @@ def run_python_tests(options):
         assert os.path.isfile(expected_results_file)
         compare_results(frame_output_file,expected_results_file,options)
         
+        
+        ## Run again, this time in memory, and make sure the results are *almost* the same
+        
+        # They won't be quite the same, because the on-disk path goes through a jpeg intermediate
+        
+        print('\n** Running MD on a folder of videos (in memory) (module) **\n')
+        
+        video_options.output_json_file = insert_before_extension(video_options.output_json_file,'in-memory')
+        video_options.force_on_disk_frame_extraction = False
+        _ = process_video_folder(video_options)
+                
+        frame_output_file_in_memory = insert_before_extension(video_options.output_json_file,'frames')
+        assert os.path.isfile(frame_output_file_in_memory)
+        
+        from copy import deepcopy
+        options_loose = deepcopy(options)
+        options_loose.max_conf_error = 0.05
+        options_loose.max_coord_error = 0.01
+        
+        compare_results(inference_output_file=frame_output_file,
+                        expected_results_file=frame_output_file_in_memory,
+                        options=options_loose)
+                
     # ...if we're not skipping video tests
     
     print('\n*** Finished module tests ***\n')
@@ -835,6 +981,8 @@ def run_cli_tests(options):
     
     ## Run inference on an image
     
+    print('\n** Running MD on a single image (CLI) **\n')
+    
     image_fn = os.path.join(options.scratch_dir,options.test_images[0])
     output_dir = os.path.join(options.scratch_dir,'single_image_test')
     if options.cli_working_dir is None:
@@ -857,6 +1005,7 @@ def run_cli_tests(options):
     
     ## Run inference on a folder
     
+    print('\n** Running MD on a folder (CLI) **\n')
     
     image_folder = os.path.join(options.scratch_dir,'md-test-images')
     assert os.path.isdir(image_folder), 'Test image folder {} is not available'.format(image_folder)
@@ -876,6 +1025,8 @@ def run_cli_tests(options):
     
     ## Run again with checkpointing enabled, make sure the results are the same
     
+    print('\n** Running MD on a folder (with checkpoints) (CLI) **\n')
+    
     from megadetector.utils.path_utils import insert_before_extension
         
     checkpoint_string = ' --checkpoint_frequency 5'
@@ -885,11 +1036,14 @@ def run_cli_tests(options):
     cmd_results = execute_and_print(cmd)
     
     assert output_files_are_identical(fn1=inference_output_file, 
-                                      fn2=inference_output_file_checkpoint,verbose=True)
+                                      fn2=inference_output_file_checkpoint,
+                                      verbose=True)
     
     
     ## Run again with the image queue enabled, make sure the results are the same
     
+    print('\n** Running MD on a folder (with image queue) (CLI) **\n')
+    
     cmd = base_cmd + ' --use_image_queue'
     from megadetector.utils.path_utils import insert_before_extension
     inference_output_file_queue = insert_before_extension(inference_output_file,'_queue')
@@ -897,7 +1051,8 @@ def run_cli_tests(options):
     cmd_results = execute_and_print(cmd)
     
     assert output_files_are_identical(fn1=inference_output_file, 
-                                      fn2=inference_output_file_queue,verbose=True)
+                                      fn2=inference_output_file_queue,
+                                      verbose=True)
     
     
     ## Run again on multiple cores, make sure the results are the same
@@ -916,11 +1071,16 @@ def run_cli_tests(options):
     if not gpu_available:
         inference_output_file_cpu = inference_output_file
     else:
+        
+        print('\n** Running MD on a folder (single CPU) (CLI) **\n')
+                
         inference_output_file_cpu = insert_before_extension(inference_output_file,'cpu')    
         cmd = base_cmd
         cmd = cmd.replace(inference_output_file,inference_output_file_cpu)    
         cmd_results = execute_and_print(cmd)
         
+    print('\n** Running MD on a folder (multiple CPUs) (CLI) **\n')
+    
     cpu_string = ' --ncores 4'
     cmd = base_cmd + cpu_string
     from megadetector.utils.path_utils import insert_before_extension
@@ -935,11 +1095,14 @@ def run_cli_tests(options):
         del os.environ['CUDA_VISIBLE_DEVICES']
         
     assert output_files_are_identical(fn1=inference_output_file_cpu, 
-                                      fn2=inference_output_file_cpu_multicore,verbose=True)
+                                      fn2=inference_output_file_cpu_multicore,
+                                      verbose=True)
     
     
     ## Postprocessing
     
+    print('\n** Testing post-processing (CLI) **\n')
+    
     postprocessing_output_dir = os.path.join(options.scratch_dir,'postprocessing_output_cli')
     
     if options.cli_working_dir is None:
@@ -954,6 +1117,8 @@ def run_cli_tests(options):
     
     ## RDE
     
+    print('\n** Running RDE (CLI) **\n')
+    
     rde_output_dir = os.path.join(options.scratch_dir,'rde_output_cli')
     
     if options.cli_working_dir is None:
@@ -990,6 +1155,8 @@ def run_cli_tests(options):
     
     ## Run inference on a folder (tiled)
     
+    print('\n** Running tiled inference (CLI) **\n')
+    
     image_folder = os.path.join(options.scratch_dir,'md-test-images')
     tiling_folder = os.path.join(options.scratch_dir,'tiling-folder')
     inference_output_file_tiled = os.path.join(options.scratch_dir,'folder_inference_output_tiled.json')
@@ -1014,6 +1181,8 @@ def run_cli_tests(options):
         
     else:
     
+        print('\n** Running YOLOv5 val tests (CLI) **\n')
+        
         image_folder = os.path.join(options.scratch_dir,'md-test-images')
         yolo_results_folder = os.path.join(options.scratch_dir,'yolo-output-folder')
         yolo_symlink_folder = os.path.join(options.scratch_dir,'yolo-symlink_folder')
@@ -1041,12 +1210,15 @@ def run_cli_tests(options):
         cmd_results = execute_and_print(cmd)
         
         assert output_files_are_identical(fn1=inference_output_file_yolo_val,
-                                          fn2=inference_output_file_yolo_val_checkpoint)
+                                          fn2=inference_output_file_yolo_val_checkpoint,
+                                          verbose=True)
         
     if not options.skip_video_tests:
             
         ## Video test
         
+        print('\n** Testing video rendering (CLI) **\n')
+        
         video_inference_output_file = os.path.join(options.scratch_dir,'video_inference_output.json')
         output_video_file = os.path.join(options.scratch_dir,'video_scratch/cli_rendered_video.mp4')
         frame_folder = os.path.join(options.scratch_dir,'video_scratch/frame_folder_cli')
@@ -1073,6 +1245,8 @@ def run_cli_tests(options):
     
     ## Run inference on a folder (with MDV5B, so we can do a comparison)
     
+    print('\n** Running MDv5b (CLI) **\n')
+    
     image_folder = os.path.join(options.scratch_dir,'md-test-images')
     inference_output_file_alt = os.path.join(options.scratch_dir,'folder_inference_output_alt.json')
     if options.cli_working_dir is None:
@@ -1166,14 +1340,16 @@ if False:
     options.test_data_url = 'https://lila.science/public/md-test-package.zip'
     options.force_data_download = False
     options.force_data_unzip = False
-    options.warning_mode = True
+    options.warning_mode = False
     options.max_coord_error = 0.001
     options.max_conf_error = 0.005
     options.cli_working_dir = r'c:\git\MegaDetector'
     options.yolo_working_dir = r'c:\git\yolov5-md'
 
-    import os
     
+    #%%
+    
+    import os
     if 'PYTHONPATH' not in os.environ or options.yolo_working_dir not in os.environ['PYTHONPATH']:
         os.environ['PYTHONPATH'] += ';' + options.yolo_working_dir
 
@@ -1332,4 +1508,3 @@ if False:
     fn1 = r"G:\temp\md-test-package\mdv5a-image-cpu-pt1.10.1.json"
     fn2 = r"G:\temp\md-test-package\mdv5a-augment-image-cpu-pt1.10.1.json"
     print(output_files_are_identical(fn1,fn2,verbose=True))
-    

megadetector/utils/path_utils.py

@@ -924,8 +924,8 @@ def zip_files_into_single_zipfile(input_files, output_fn, arc_name_base,
     
 def zip_folder(input_folder, output_fn=None, overwrite=False, verbose=False, compresslevel=9):
     """
-    Recursively zip everything in [input_folder] into a single zipfile, storing outputs as relative 
-    paths.
+    Recursively zip everything in [input_folder] into a single zipfile, storing files as paths 
+    relative to [input_folder].
     
     Args: 
         input_folder (str): folder to zip

megadetector/utils/torch_test.py

@@ -0,0 +1,32 @@
+"""
+
+ torch_test.py
+
+ Simple script to verify CUDA availability, used to verify a CUDA/PyTorch
+ environment.
+
+"""
+
+def torch_test():
+    """
+    Print diagnostic information about Torch/CUDA status, including Torch/CUDA versions
+    and all available CUDA device names.
+    """
+    
+    import torch
+
+    print('Torch version: {}'.format(str(torch.__version__)))
+    print('CUDA available: {}'.format(torch.cuda.is_available()))
+
+    device_ids = list(range(torch.cuda.device_count()))
+    print('Found {} CUDA devices:'.format(len(device_ids)))
+    for device_id in device_ids:
+        device_name = 'unknown'
+        try:
+            device_name = torch.cuda.get_device_name(device=device_id)
+        except Exception as e:
+            pass
+        print('{}: {}'.format(device_id,device_name))
+       
+if __name__ == '__main__':    
+    torch_test()