megadetector 5.0.5__py3-none-any.whl → 5.0.7__py3-none-any.whl

md_utils/md_tests.py

@@ -1,24 +1,17 @@
 ########
 #
-# md-tests.py
+# md_tests.py
 #
 # A series of tests to validate basic repo functionality and verify either "correct"
 # inference behavior, or - when operating in environments other than the training
 # environment - acceptable deviation from the correct results.
 #
-# This module should not depend on anything else in this repo outside of the, 
-# tests themselves, even if it means some duplicated code (e.g. for downloading files), since 
-# much of what it tries to test is, e.g., imports.
+# This module should not depend on anything else in this repo outside of the 
+# tests themselves, even if it means some duplicated code (e.g. for downloading files),
+# since much of what it tries to test is, e.g., imports.
 #
 ########
 
-#%% TODO
-
-# Video tests
-# Augmented inference tests
-# Checkpoint tests
-
-
 #%% Imports and constants
 
 ### Only standard imports belong here, not MD-specific imports ###
@@ -42,28 +35,61 @@ class MDTestOptions:
     
     disable_gpu = False
     cpu_execution_is_error = False
-    disable_video_tests = False
+    skip_video_tests = False
+    skip_python_tests = False
+    skip_cli_tests = False
     scratch_dir = None
     test_data_url = 'https://lila.science/public/md-test-package.zip'
     force_data_download = False
     force_data_unzip = False
+    warning_mode = False
     test_image_subdir = 'md-test-images'
     max_coord_error = 0.001
     max_conf_error = 0.005
     cli_working_dir = None
+    yolo_working_folder = None
 
 
 #%% Support functions
 
 def get_expected_results_filename(gpu_is_available):
+    """
+    Expected results vary just a little across inference environments, particularly
+    between PT 1.x and 2.x, so when making sure things are working acceptably, we 
+    compare to a reference file that matches the current environment.
+    """
     
     if gpu_is_available:
-        return 'md-test-results-gpu-pt1.10.1.json'
+        hw_string = 'gpu'
+    else:
+        hw_string = 'cpu'
+    import torch
+    torch_version = str(torch.__version__)
+    if torch_version.startswith('1'):
+        assert torch_version == '1.10.1', 'Only tested against PT 1.10.1 and PT 2.x'
+        pt_string = 'pt1.10.1'
     else:
-        return 'md-test-results-cpu-pt1.10.1.json'
+        assert torch_version.startswith('2'), 'Unknown torch version: {}'.format(torch_version)
+        pt_string = 'pt2.x'
+    
+    # A hack for now to account for the fact that even with acceleration enabled and PT2 
+    # installed, Apple silicon appears to provide the same results as CPU/PT1 inference
+    try:
+        import torch
+        m1_inference = torch.backends.mps.is_built and torch.backends.mps.is_available()
+        if m1_inference:
+            hw_string = 'cpu'
+        pt_string = 'pt1.10.1'
+    except Exception:
+        pass
+    
+    return 'md-test-results-{}-{}.json'.format(hw_string,pt_string)
     
     
 def download_test_data(options):
+    """
+    Download the test zipfile if necessary, unzip if necessary.
+    """
     
     if options.scratch_dir is None:        
         tempdir_base = tempfile.gettempdir()
@@ -87,9 +113,9 @@ def download_test_data(options):
     if download_zipfile:
         print('Downloading test data zipfile')
         urllib.request.urlretrieve(options.test_data_url, local_zipfile)
-        print('Finished download')
+        print('Finished download to {}'.format(local_zipfile))
     else:
-        print('Bypassing test data zipfile download')
+        print('Bypassing test data zipfile download for {}'.format(local_zipfile))
     
     
     ## Unzip data
@@ -132,23 +158,38 @@ def download_test_data(options):
     options.all_test_files = test_files
     options.test_images = [fn for fn in test_files if os.path.splitext(fn.lower())[1] in ('.jpg','.jpeg','.png')]
     options.test_videos = [fn for fn in test_files if os.path.splitext(fn.lower())[1] in ('.mp4','.avi')]    
+    options.test_videos = [fn for fn in options.test_videos if 'rendered' not in fn]
         
 # ...def download_test_data(...)
 
 
 def is_gpu_available(verbose=True):
-        
+    """
+    Check whether a GPU (including M1/M2 MPS) is available.
+    """
+    
     # Import torch inside this function, so we have a chance to set CUDA_VISIBLE_DEVICES
     # before checking GPU availability.
     import torch
     gpu_available = torch.cuda.is_available()
     
-    if verbose:    
-        print('CUDA available: {}'.format(gpu_available))
-        device_ids = list(range(torch.cuda.device_count()))
-        if len(device_ids) > 1:
-            print('Found multiple devices: {}'.format(str(device_ids)))
-    
+    if gpu_available:
+        if verbose:
+            print('CUDA available: {}'.format(gpu_available))
+            device_ids = list(range(torch.cuda.device_count()))
+            if len(device_ids) > 1:
+                print('Found multiple devices: {}'.format(str(device_ids)))
+    else:
+        try:
+            gpu_available = torch.backends.mps.is_built and torch.backends.mps.is_available()
+        except AttributeError:
+            pass
+        if gpu_available:
+            print('Metal performance shaders available')
+    
+    if not gpu_available:
+        print('No GPU available')
+        
     return gpu_available            
         
     
@@ -201,6 +242,8 @@ def execute_and_print(cmd,print_output=True):
 
 def run_python_tests(options):
 
+    print('\n*** Starting module tests ***\n')
+    
     ## Prepare data
     
     download_test_data(options)
@@ -237,8 +280,6 @@ def run_python_tests(options):
 
     ## Verify results
 
-    #%%
-    
     # Read expected results
     expected_results_filename = get_expected_results_filename(is_gpu_available(verbose=False))
     
@@ -272,9 +313,17 @@ def run_python_tests(options):
         actual_detections = actual_image_results['detections']
         expected_detections = expected_image_results['detections']
         
+        s = 'expected {} detections for file {}, found {}'.format(
+            len(expected_detections),fn,len(actual_detections))
+        s += '\nExpected results file: {}\nActual results file: {}'.format(
+            expected_results_filename,inference_output_file)
+        
+        if options.warning_mode:
+            if len(actual_detections) != len(expected_detections):
+                print('Warning: {}'.format(s))
+            continue
         assert len(actual_detections) == len(expected_detections), \
-            'Error: expected {} detections for file {}, found {}'.format(
-                len(expected_detections),fn,len(actual_detections))
+            'Error: {}'.format(s)
         
         # i_det = 0
         for i_det in range(0,len(actual_detections)):
@@ -296,16 +345,16 @@ def run_python_tests(options):
         
     # ...for each image
     
-    #%%
-    
-    assert max_conf_error <= options.max_conf_error, \
-        'Confidence error {} is greater than allowable ({})'.format(
-            max_conf_error,options.max_conf_error)
-    
-    assert max_coord_error <= options.max_coord_error, \
-        'Coord error {} is greater than allowable ({})'.format(
-            max_coord_error,options.max_coord_error)
-    
+    if not options.warning_mode:
+        
+        assert max_conf_error <= options.max_conf_error, \
+            'Confidence error {} is greater than allowable ({})'.format(
+                max_conf_error,options.max_conf_error)
+        
+        assert max_coord_error <= options.max_coord_error, \
+            'Coord error {} is greater than allowable ({})'.format(
+                max_coord_error,options.max_coord_error)
+        
     print('Max conf error: {}'.format(max_conf_error))
     print('Max coord error: {}'.format(max_coord_error))
 
@@ -341,7 +390,84 @@ def run_python_tests(options):
     assert os.path.isfile(rde_results.filterFile),\
         'Could not find RDE output file {}'.format(rde_results.filterFile)
         
-    print('Finished running Python tests')
+    
+    # TODO: add remove_repeat_detections test here
+    #
+    # It's already tested in the CLI tests, so this is not urgent.
+    
+
+    ## Video test (single video)
+    
+    from detection.process_video import ProcessVideoOptions, process_video
+    
+    video_options = ProcessVideoOptions()
+    video_options.model_file = 'MDV5A'
+    video_options.input_video_file = os.path.join(options.scratch_dir,options.test_videos[0])
+    video_options.output_json_file = os.path.join(options.scratch_dir,'single_video_output.json')
+    video_options.output_video_file = os.path.join(options.scratch_dir,'video_scratch/rendered_video.mp4')
+    video_options.frame_folder = os.path.join(options.scratch_dir,'video_scratch/frame_folder')
+    video_options.frame_rendering_folder = os.path.join(options.scratch_dir,'video_scratch/rendered_frame_folder')    
+    video_options.render_output_video = True
+    # video_options.keep_rendered_frames = False
+    # video_options.keep_rendered_frames = False
+    video_options.force_extracted_frame_folder_deletion = True
+    video_options.force_rendered_frame_folder_deletion = True
+    # video_options.reuse_results_if_available = False
+    # video_options.reuse_frames_if_available = False
+    video_options.recursive = True
+    video_options.verbose = False
+    video_options.fourcc = 'mp4v'
+    # video_options.rendering_confidence_threshold = None
+    # video_options.json_confidence_threshold = 0.005
+    video_options.frame_sample = 5    
+    video_options.n_cores = 5
+    # video_options.debug_max_frames = -1
+    # video_options.class_mapping_filename = None
+    
+    _ = process_video(video_options)
+
+    assert os.path.isfile(video_options.output_video_file), \
+        'Python video test failed to render output video file'
+    assert os.path.isfile(video_options.output_json_file), \
+        'Python video test failed to render output .json file'
+        
+    
+    ## Video test (folder)
+    
+    from detection.process_video import ProcessVideoOptions, process_video_folder
+    
+    video_options = ProcessVideoOptions()
+    video_options.model_file = 'MDV5A'
+    video_options.input_video_file = os.path.join(options.scratch_dir,
+                                                  os.path.dirname(options.test_videos[0]))
+    video_options.output_json_file = os.path.join(options.scratch_dir,'video_folder_output.json')
+    # video_options.output_video_file = None
+    video_options.frame_folder = os.path.join(options.scratch_dir,'video_scratch/frame_folder')
+    video_options.frame_rendering_folder = os.path.join(options.scratch_dir,'video_scratch/rendered_frame_folder')    
+    video_options.render_output_video = False
+    # video_options.keep_rendered_frames = False
+    # video_options.keep_rendered_frames = False
+    video_options.force_extracted_frame_folder_deletion = True
+    video_options.force_rendered_frame_folder_deletion = True
+    # video_options.reuse_results_if_available = False
+    # video_options.reuse_frames_if_available = False
+    video_options.recursive = True
+    video_options.verbose = False
+    # video_options.fourcc = None
+    # video_options.rendering_confidence_threshold = None
+    # video_options.json_confidence_threshold = 0.005
+    video_options.frame_sample = 5    
+    video_options.n_cores = 5
+    # video_options.debug_max_frames = -1
+    # video_options.class_mapping_filename = None
+    
+    _ = process_video_folder(video_options)
+
+    assert os.path.isfile(video_options.output_json_file), \
+        'Python video test failed to render output .json file'
+    
+    
+    print('\n*** Finished module tests ***\n')
 
 # ...def run_python_tests(...)
 
@@ -350,6 +476,8 @@ def run_python_tests(options):
 
 def run_cli_tests(options):
     
+    print('\n*** Starting CLI tests ***\n')
+    
     ## chdir if necessary
     
     if options.cli_working_dir is not None:
@@ -366,7 +494,11 @@ def run_cli_tests(options):
     model_file = 'MDV5A'
     image_fn = os.path.join(options.scratch_dir,options.test_images[0])
     output_dir = os.path.join(options.scratch_dir,'single_image_test')
-    cmd = 'python detection/run_detector.py {} --image_file {} --output_dir {}'.format(
+    if options.cli_working_dir is None:
+        cmd = 'python -m detection.run_detector'
+    else:
+        cmd = 'python detection/run_detector.py'
+    cmd += ' {} --image_file {} --output_dir {}'.format(
         model_file,image_fn,output_dir)
     print('Running: {}'.format(cmd))
     cmd_results = execute_and_print(cmd)
@@ -386,9 +518,14 @@ def run_cli_tests(options):
     image_folder = os.path.join(options.scratch_dir,'md-test-images')
     assert os.path.isdir(image_folder), 'Test image folder {} is not available'.format(image_folder)
     inference_output_file = os.path.join(options.scratch_dir,'folder_inference_output.json')
-    cmd = 'python detection/run_detector_batch.py {} {} {} --recursive'.format(
+    if options.cli_working_dir is None:
+        cmd = 'python -m detection.run_detector_batch'
+    else:
+        cmd = 'python detection/run_detector_batch.py'
+    cmd += ' {} {} {} --recursive'.format(
         model_file,image_folder,inference_output_file)
-    cmd += ' --output_relative_filenames --quiet --include_image_size --include_image_timestamp --include_exif_data'
+    cmd += ' --output_relative_filenames --quiet --include_image_size'
+    cmd += ' --include_image_timestamp --include_exif_data'
     print('Running: {}'.format(cmd))
     cmd_results = execute_and_print(cmd)
     
@@ -402,16 +539,171 @@ def run_cli_tests(options):
     
     postprocessing_output_dir = os.path.join(options.scratch_dir,'postprocessing_output_cli')
     
-    cmd = 'python api/batch_processing/postprocessing/postprocess_batch_results.py {} {}'.format(
+    if options.cli_working_dir is None:
+        cmd = 'python -m api.batch_processing.postprocessing.postprocess_batch_results'
+    else:
+        cmd = 'python api/batch_processing/postprocessing/postprocess_batch_results.py'
+    cmd += ' {} {}'.format(
         inference_output_file,postprocessing_output_dir)
     cmd += ' --image_base_dir {}'.format(image_folder)
     print('Running: {}'.format(cmd))
     cmd_results = execute_and_print(cmd)
                 
+    
+    ## RDE
+    
+    rde_output_dir = os.path.join(options.scratch_dir,'rde_output_cli')
+    
+    if options.cli_working_dir is None:
+        cmd = 'python -m api.batch_processing.postprocessing.repeat_detection_elimination.find_repeat_detections'
+    else:
+        cmd = 'python  api/batch_processing/postprocessing/repeat_detection_elimination/find_repeat_detections.py'
+    cmd += ' {}'.format(inference_output_file)
+    cmd += ' --imageBase {}'.format(image_folder)
+    cmd += ' --outputBase {}'.format(rde_output_dir)
+    cmd += ' --occurrenceThreshold 1' # Use an absurd number here to make sure we get some suspicious detections
+    print('Running: {}'.format(cmd))
+    cmd_results = execute_and_print(cmd)    
+    
+    # Find the latest filtering folder
+    filtering_output_dir = os.listdir(rde_output_dir)
+    filtering_output_dir = [fn for fn in filtering_output_dir if fn.startswith('filtering_')]
+    filtering_output_dir = [os.path.join(rde_output_dir,fn) for fn in filtering_output_dir]
+    filtering_output_dir = [fn for fn in filtering_output_dir if os.path.isdir(fn)]
+    filtering_output_dir = sorted(filtering_output_dir)[-1]
+    
+    print('Using RDE filtering folder {}'.format(filtering_output_dir))
+    
+    filtered_output_file = inference_output_file.replace('.json','_filtered.json')
+    
+    if options.cli_working_dir is None:
+        cmd = 'python -m api.batch_processing.postprocessing.repeat_detection_elimination.remove_repeat_detections'
+    else:
+        cmd = 'python  api/batch_processing/postprocessing/repeat_detection_elimination/remove_repeat_detections.py'
+    cmd += ' {} {} {}'.format(inference_output_file,filtered_output_file,filtering_output_dir)
+    print('Running: {}'.format(cmd))
+    cmd_results = execute_and_print(cmd)
+    
+    assert os.path.isfile(filtered_output_file), \
+        'Could not find RDE output file {}'.format(filtered_output_file)
+    
+    
+    ## Run inference on a folder (tiled)
+    
+    image_folder = os.path.join(options.scratch_dir,'md-test-images')
+    tiling_folder = os.path.join(options.scratch_dir,'tiling-folder')
+    inference_output_file_tiled = os.path.join(options.scratch_dir,'folder_inference_output_tiled.json')
+    if options.cli_working_dir is None:
+        cmd = 'python -m detection.run_tiled_inference'
+    else:
+        cmd = 'python detection/run_tiled_inference.py'
+    cmd += ' {} {} {} {}'.format(
+        model_file,image_folder,tiling_folder,inference_output_file_tiled)
+    cmd += ' --overwrite_handling overwrite'
+    print('Running: {}'.format(cmd))
+    cmd_results = execute_and_print(cmd)
+    
+    with open(inference_output_file_tiled,'r') as f:
+        results_from_file = json.load(f) # noqa
+        
+    
+    ## Run inference on a folder (augmented)
+    
+    if options.yolo_working_folder is None:
+        
+        print('Bypassing YOLOv5 val tests, no yolo folder supplied')
+        
+    else:
+    
+        image_folder = os.path.join(options.scratch_dir,'md-test-images')
+        yolo_results_folder = os.path.join(options.scratch_dir,'yolo-output-folder')
+        yolo_symlink_folder = os.path.join(options.scratch_dir,'yolo-symlink_folder')
+        inference_output_file_yolo_val = os.path.join(options.scratch_dir,'folder_inference_output_yolo_val.json')
+        if options.cli_working_dir is None:
+            cmd = 'python -m detection.run_inference_with_yolov5_val'
+        else:
+            cmd = 'python detection/run_inference_with_yolov5_val.py'
+        cmd += ' {} {} {}'.format(
+            model_file,image_folder,inference_output_file_yolo_val)
+        cmd += ' --yolo_working_folder {}'.format(options.yolo_working_folder)
+        cmd += ' --yolo_results_folder {}'.format(yolo_results_folder)
+        cmd += ' --symlink_folder {}'.format(yolo_symlink_folder)
+        cmd += ' --augment_enabled 1'
+        # cmd += ' --no_use_symlinks'
+        cmd += ' --overwrite_handling overwrite'
+        print('Running: {}'.format(cmd))
+        cmd_results = execute_and_print(cmd)
+        
+        with open(inference_output_file_yolo_val,'r') as f:
+            results_from_file = json.load(f) # noqa
+        
+        
+    ## Video test
+    
+    model_file = 'MDV5A'
+    video_inference_output_file = os.path.join(options.scratch_dir,'video_inference_output.json')
+    output_video_file = os.path.join(options.scratch_dir,'video_scratch/cli_rendered_video.mp4')
+    frame_folder = os.path.join(options.scratch_dir,'video_scratch/frame_folder_cli')
+    frame_rendering_folder = os.path.join(options.scratch_dir,'video_scratch/rendered_frame_folder_cli')        
+    
+    video_fn = os.path.join(options.scratch_dir,options.test_videos[-1])
+    output_dir = os.path.join(options.scratch_dir,'single_video_test_cli')
+    if options.cli_working_dir is None:
+        cmd = 'python -m detection.process_video'
+    else:
+        cmd = 'python detection/process_video.py'
+    cmd += ' {} {}'.format(model_file,video_fn)
+    cmd += ' --frame_folder {} --frame_rendering_folder {} --output_json_file {} --output_video_file {}'.format(
+        frame_folder,frame_rendering_folder,video_inference_output_file,output_video_file)
+    cmd += ' --render_output_video --fourcc mp4v'
+    cmd += ' --force_extracted_frame_folder_deletion --force_rendered_frame_folder_deletion --n_cores 5 --frame_sample 3'
+    print('Running: {}'.format(cmd))
+    cmd_results = execute_and_print(cmd)
+    
+    
+    ## Run inference on a folder (again, so we can do a comparison)
+    
+    image_folder = os.path.join(options.scratch_dir,'md-test-images')
+    model_file = 'MDV5B'
+    inference_output_file_alt = os.path.join(options.scratch_dir,'folder_inference_output_alt.json')
+    if options.cli_working_dir is None:
+        cmd = 'python -m detection.run_detector_batch'
+    else:
+        cmd = 'python detection/run_detector_batch.py'
+    cmd += ' {} {} {} --recursive'.format(
+        model_file,image_folder,inference_output_file_alt)
+    cmd += ' --output_relative_filenames --quiet --include_image_size'
+    cmd += ' --include_image_timestamp --include_exif_data'
+    print('Running: {}'.format(cmd))
+    cmd_results = execute_and_print(cmd)
+    
+    with open(inference_output_file_alt,'r') as f:
+        results_from_file = json.load(f) # noqa
+    
+    
+    ## Compare the two files
+    
+    comparison_output_folder = os.path.join(options.scratch_dir,'results_comparison')
+    image_folder = os.path.join(options.scratch_dir,'md-test-images')
+    results_files_string = '"{}" "{}"'.format(
+        inference_output_file,inference_output_file_alt)
+    if options.cli_working_dir is None:
+        cmd = 'python -m api.batch_processing.postprocessing.compare_batch_results'
+    else:
+        cmd = 'python api/batch_processing/postprocessing/compare_batch_results.py'
+    cmd += ' {} {} {}'.format(comparison_output_folder,image_folder,results_files_string)
+    print('Running: {}'.format(cmd))
+    cmd_results = execute_and_print(cmd)
+    
+    assert cmd_results['status'] == 0, 'Error generating comparison HTML'
+    assert os.path.isfile(os.path.join(comparison_output_folder,'index.html')), \
+        'Failed to generate comparison HTML'
+    
+    print('\n*** Finished CLI tests ***\n')
+    
 # ...def run_cli_tests(...)
 
 
-
 #%% Main test wrapper
 
 def run_tests(options):
@@ -431,13 +723,15 @@ def run_tests(options):
     
     # If the GPU should be disabled, verify that it is
     if options.disable_gpu:
-        assert (not gpu_available), 'Oops, CPU execution specified, but the GPU appears to be available'
+        assert (not gpu_available), 'CPU execution specified, but the GPU appears to be available'
         
     # Run python tests
-    run_python_tests(options)
+    if not options.skip_python_tests:
+        run_python_tests(options)
     
     # Run CLI tests
-    run_cli_tests(options)
+    if not options.skip_cli_tests:
+        run_cli_tests(options)
 
 
 #%% Interactive driver
@@ -452,9 +746,19 @@ if False:
     
     options.disable_gpu = False
     options.cpu_execution_is_error = False
-    options.disable_video_tests = False
+    options.skip_video_tests = False
+    options.skip_python_tests = False
+    options.skip_cli_tests = False
     options.scratch_dir = None
+    options.test_data_url = 'https://lila.science/public/md-test-package.zip'
+    options.force_data_download = False
+    options.force_data_unzip = False
+    options.warning_mode = True
+    options.test_image_subdir = 'md-test-images'
+    options.max_coord_error = 0.001
+    options.max_conf_error = 0.005
     options.cli_working_dir = r'c:\git\MegaDetector'
+    options.yolo_working_folder = r'c:\git\yolov5'
 
 
     #%%
@@ -488,9 +792,19 @@ def main():
         help='Directory for temporary storage (defaults to system temp dir)')
     
     parser.add_argument(
-        '--disable_video_tests',
+        '--skip_video_tests',
         action='store_true',
-        help='Disable tests related to video (which can be slow)')
+        help='Skip tests related to video (which can be slow)')
+        
+    parser.add_argument(
+        '--skip_python_tests',
+        action='store_true',
+        help='Skip python tests')
+        
+    parser.add_argument(
+        '--skip_cli_tests',
+        action='store_true',
+        help='Skip CLI tests')
         
     parser.add_argument(
         '--force_data_download',
@@ -502,6 +816,11 @@ def main():
         action='store_true',
         help='Force extraction of all files in the test data file, even if they\'re already available')
     
+    parser.add_argument(
+        '--warning_mode',
+        action='store_true',
+        help='Turns numeric/content errors into warnings')
+    
     parser.add_argument(
         '--max_conf_error',
         type=float,
@@ -526,8 +845,11 @@ def main():
         
     options.disable_gpu = args.disable_gpu
     options.cpu_execution_is_error = args.cpu_execution_is_error
-    options.disable_video_tests = args.disable_video_tests
+    options.skip_video_tests = args.skip_video_tests
+    options.skip_python_tests = args.skip_python_tests
+    options.skip_cli_tests = args.skip_cli_tests
     options.scratch_dir = args.scratch_dir
+    options.warning_mode = args.warning_mode
     options.force_data_download = args.force_data_download
     options.max_conf_error = args.max_conf_error
     options.max_coord_error = args.max_coord_error
@@ -538,4 +860,3 @@ def main():
     
 if __name__ == '__main__':
     main()
-