megadetector 5.0.13__py3-none-any.whl → 5.0.15__py3-none-any.whl

megadetector/utils/md_tests.py

@@ -10,6 +10,8 @@ This module should not depend on anything else in this repo outside of the
 tests themselves, even if it means some duplicated code (e.g. for downloading files),
 since much of what it tries to test is, e.g., imports.
 
+"Correctness" is determined by agreement with a file that this script fetches from lila.science.
+
 """
 
 #%% Imports and constants
@@ -75,10 +77,12 @@ class MDTestOptions:
         self.max_coord_error = 0.001
         
         #: How much deviation from the expected confidence values should we allow before
-        #: a disrepancy becomes an error?    
+        #: a disrepancy becomes an error?
         self.max_conf_error = 0.005
         
         #: Current working directory when running CLI tests
+        #:
+        #: If this is None, we won't mess with the inherited working directory.
         self.cli_working_dir = None
         
         #: YOLOv5 installation, only relevant if we're testing run_inference_with_yolov5_val. 
@@ -92,7 +96,7 @@ class MDTestOptions:
         #: Default model to use for testing (filename, URL, or well-known model string)
         self.default_model = 'MDV5A'
 
-        #: For comparison tests, use a model that produces slightly different output        
+        #: For comparison tests, use a model that produces slightly different output
         self.alt_model = 'MDV5B'
         
         #: PYTHONPATH to set for CLI tests; if None, inherits from the parent process.  Only
@@ -104,7 +108,11 @@ class MDTestOptions:
 
 #%% Support functions
 
-def get_expected_results_filename(gpu_is_available):
+def get_expected_results_filename(gpu_is_available,
+                                  model_string='mdv5a',
+                                  test_type='image',
+                                  augment=False,
+                                  options=None):
     """
     Expected results vary just a little across inference environments, particularly
     between PT 1.x and 2.x, so when making sure things are working acceptably, we 
@@ -140,13 +148,27 @@ def get_expected_results_filename(gpu_is_available):
         import torch
         m1_inference = torch.backends.mps.is_built and torch.backends.mps.is_available()
         if m1_inference:
-            print('I appear to be running on M1/M2 hardware')
+            print('I appear to be running on M1/M2 hardware, using pt1/cpu as the reference results')
             hw_string = 'cpu'
             pt_string = 'pt1.10.1'
     except Exception:
         pass
     
-    return 'md-test-results-{}-{}.json'.format(hw_string,pt_string)
+    aug_string = ''
+    if augment:
+        aug_string = 'augment-'
+        
+    fn = '{}-{}{}-{}-{}.json'.format(model_string,aug_string,test_type,hw_string,pt_string)
+    
+    from megadetector.utils.path_utils import insert_before_extension
+    
+    if test_type == 'video':
+        fn = insert_before_extension(fn,'frames')
+    
+    if options is not None and options.scratch_dir is not None:
+        fn = os.path.join(options.scratch_dir,fn)
+        
+    return fn
     
     
 def download_test_data(options=None):
@@ -216,7 +238,12 @@ def download_test_data(options=None):
             
     # ...for each file in the zipfile
     
-    # Warn if file are present that aren't expected
+    try:
+        zipf.close()
+    except Exception as e:
+        print('Warning: error closing zipfile:\n{}'.format(str(e)))
+        
+    # Warn if files are present that aren't expected
     test_files = glob.glob(os.path.join(scratch_dir,'**/*'), recursive=True)
     test_files = [os.path.relpath(fn,scratch_dir).replace('\\','/') for fn in test_files]
     test_files_set = set(test_files)
@@ -312,18 +339,19 @@ def output_files_are_identical(fn1,fn2,verbose=False):
                 len(fn2_results['images']),fn2))
         return False
     
+    # i_image = 0; fn1_image = fn1_results['images'][i_image]
     for i_image,fn1_image in enumerate(fn1_results['images']):
         
         fn2_image = fn2_results['images'][i_image]
         
         if fn1_image['file'] != fn2_image['file']:
             if verbose:
-                print('Filename difference: {} vs {} '.format(fn1_image['file'],fn1_image['file']))
+                print('Filename difference at {}: {} vs {} '.format(i_image,fn1_image['file'],fn1_image['file']))
             return False
             
         if fn1_image != fn2_image:
             if verbose:
-                print('Image-level difference in image {}'.format(fn1_image['file']))
+                print('Image-level difference in image {}: {}'.format(i_image,fn1_image['file']))
             return False
         
     return True
@@ -331,6 +359,100 @@ def output_files_are_identical(fn1,fn2,verbose=False):
 # ...def output_files_are_identical(...)
    
 
+def compare_results(inference_output_file,expected_results_file,options):
+    """
+    Compare two MD-formatted output files that should be nearly identical, allowing small
+    changes (e.g. rounding differences).  Generally used to compare a new results file to 
+    an expected results file.
+    
+    Args:
+        inference_output_file (str): the first results file to compare
+        expected_results_file (str): the second results file to compare
+        options (MDTestOptions): options that determine tolerable differences between files
+    """
+    
+    # Read results
+    with open(inference_output_file,'r') as f:
+        results_from_file = json.load(f) # noqa
+    
+    with open(os.path.join(options.scratch_dir,expected_results_file),'r') as f:
+        expected_results = json.load(f)
+            
+    filename_to_results = {im['file'].replace('\\','/'):im for im in results_from_file['images']}
+    filename_to_results_expected = {im['file'].replace('\\','/'):im for im in expected_results['images']}
+    
+    assert len(filename_to_results) == len(filename_to_results_expected), \
+        'Error: expected {} files in results, found {}'.format(
+            len(filename_to_results_expected),
+            len(filename_to_results))
+    
+    max_coord_error = 0
+    max_conf_error = 0
+    
+    # fn = next(iter(filename_to_results.keys()))
+    for fn in filename_to_results.keys():
+                
+        actual_image_results = filename_to_results[fn]
+        expected_image_results = filename_to_results_expected[fn]
+        
+        if 'failure' in actual_image_results:
+            assert 'failure' in expected_image_results and \
+                'detections' not in actual_image_results and \
+                'detections' not in expected_image_results
+            continue
+        assert 'failure' not in expected_image_results
+        
+        actual_detections = actual_image_results['detections']
+        expected_detections = expected_image_results['detections']
+        
+        s = 'expected {} detections for file {}, found {}'.format(
+            len(expected_detections),fn,len(actual_detections))
+        s += '\nExpected results file: {}\nActual results file: {}'.format(
+            expected_results_file,inference_output_file)
+        
+        if options.warning_mode:
+            if len(actual_detections) != len(expected_detections):
+                print('Warning: {}'.format(s))
+            continue
+        assert len(actual_detections) == len(expected_detections), \
+            'Error: {}'.format(s)
+        
+        # i_det = 0
+        for i_det in range(0,len(actual_detections)):
+            actual_det = actual_detections[i_det]
+            expected_det = expected_detections[i_det]
+            assert actual_det['category'] == expected_det['category']
+            conf_err = abs(actual_det['conf'] - expected_det['conf'])
+            coord_differences = []
+            for i_coord in range(0,4):
+                coord_differences.append(abs(actual_det['bbox'][i_coord]-expected_det['bbox'][i_coord]))
+            coord_err = max(coord_differences)
+            
+            if conf_err > max_conf_error:
+                max_conf_error = conf_err
+            if coord_err > max_coord_error:
+                max_coord_error = coord_err
+        
+        # ...for each detection
+        
+    # ...for each image
+    
+    if not options.warning_mode:
+        
+        assert max_conf_error <= options.max_conf_error, \
+            'Confidence error {} is greater than allowable ({})'.format(
+                max_conf_error,options.max_conf_error)
+        
+        assert max_coord_error <= options.max_coord_error, \
+            'Coord error {} is greater than allowable ({})'.format(
+                max_coord_error,options.max_coord_error)
+        
+    print('Max conf error: {}'.format(max_conf_error))
+    print('Max coord error: {}'.format(max_coord_error))
+    
+# ...def compare_results(...)
+
+
 def _args_to_object(args, obj):
     """
     Copies all fields from a Namespace (typically the output from parse_args) to an
@@ -382,7 +504,7 @@ def execute(cmd):
     return return_code
 
 
-def execute_and_print(cmd,print_output=True,catch_exceptions=False):
+def execute_and_print(cmd,print_output=True,catch_exceptions=False,echo_command=True):
     """
     Runs [cmd] (a single string) in a shell, capturing (and optionally printing) output.
     
@@ -395,8 +517,11 @@ def execute_and_print(cmd,print_output=True,catch_exceptions=False):
         (the content of stdout)
     """
 
+    if echo_command:
+        print('Running command:\n{}\n'.format(cmd))
+        
     to_return = {'status':'unknown','output':''}
-    output=[]
+    output = []
     try:
         for s in execute(cmd):
             output.append(s)
@@ -441,6 +566,8 @@ def run_python_tests(options):
 
     
     ## Run inference on a folder
+
+    print('\n** Running MD on images **\n')
     
     from megadetector.detection.run_detector_batch import load_and_run_detector_batch,write_results_to_file
     from megadetector.utils import path_utils
@@ -450,95 +577,38 @@ def run_python_tests(options):
     inference_output_file = os.path.join(options.scratch_dir,'folder_inference_output.json')
     image_file_names = path_utils.find_images(image_folder,recursive=True)
     results = load_and_run_detector_batch(options.default_model, image_file_names, quiet=True)
-    _ = write_results_to_file(results,inference_output_file,
-                              relative_path_base=image_folder,detector_file=options.default_model)
+    _ = write_results_to_file(results,
+                              inference_output_file,
+                              relative_path_base=image_folder,
+                              detector_file=options.default_model)
 
-    # Read results
-    with open(inference_output_file,'r') as f:
-        results_from_file = json.load(f) # noqa
     
-
     ## Verify results
 
-    # Read expected results
-    expected_results_filename = get_expected_results_filename(is_gpu_available(verbose=False))
-    
-    with open(os.path.join(options.scratch_dir,expected_results_filename),'r') as f:
-        expected_results = json.load(f)
-            
-    filename_to_results = {im['file'].replace('\\','/'):im for im in results_from_file['images']}
-    filename_to_results_expected = {im['file'].replace('\\','/'):im for im in expected_results['images']}
+    expected_results_file = get_expected_results_filename(is_gpu_available(verbose=False),
+                                                          options=options)
+    compare_results(inference_output_file,expected_results_file,options)
+        
+
+    ## Run and verify again with augmentation enabled
     
-    assert len(filename_to_results) == len(filename_to_results_expected), \
-        'Error: expected {} files in results, found {}'.format(
-            len(filename_to_results_expected),
-            len(filename_to_results))
+    print('\n** Running MD on images with augmentation **\n')
     
-    max_coord_error = 0
-    max_conf_error = 0
+    from megadetector.utils.path_utils import insert_before_extension
     
-    # fn = next(iter(filename_to_results.keys()))
-    for fn in filename_to_results.keys():
-                
-        actual_image_results = filename_to_results[fn]
-        expected_image_results = filename_to_results_expected[fn]
-        
-        if 'failure' in actual_image_results:
-            assert 'failure' in expected_image_results and \
-                'detections' not in actual_image_results and \
-                'detections' not in expected_image_results
-            continue
-        assert 'failure' not in expected_image_results
-        
-        actual_detections = actual_image_results['detections']
-        expected_detections = expected_image_results['detections']
-        
-        s = 'expected {} detections for file {}, found {}'.format(
-            len(expected_detections),fn,len(actual_detections))
-        s += '\nExpected results file: {}\nActual results file: {}'.format(
-            expected_results_filename,inference_output_file)
-        
-        if options.warning_mode:
-            if len(actual_detections) != len(expected_detections):
-                print('Warning: {}'.format(s))
-            continue
-        assert len(actual_detections) == len(expected_detections), \
-            'Error: {}'.format(s)
-        
-        # i_det = 0
-        for i_det in range(0,len(actual_detections)):
-            actual_det = actual_detections[i_det]
-            expected_det = expected_detections[i_det]
-            assert actual_det['category'] == expected_det['category']
-            conf_err = abs(actual_det['conf'] - expected_det['conf'])
-            coord_differences = []
-            for i_coord in range(0,4):
-                coord_differences.append(abs(actual_det['bbox'][i_coord]-expected_det['bbox'][i_coord]))
-            coord_err = max(coord_differences)
-            
-            if conf_err > max_conf_error:
-                max_conf_error = conf_err
-            if coord_err > max_coord_error:
-                max_coord_error = coord_err
-        
-        # ...for each detection
+    inference_output_file_augmented = insert_before_extension(inference_output_file,'augmented')
+    results = load_and_run_detector_batch(options.default_model, image_file_names, quiet=True, augment=True)
+    _ = write_results_to_file(results,
+                              inference_output_file_augmented,
+                              relative_path_base=image_folder,
+                              detector_file=options.default_model)
+
+    expected_results_file_augmented = \
+        get_expected_results_filename(is_gpu_available(verbose=False),
+                                      augment=True,options=options)
+    compare_results(inference_output_file_augmented,expected_results_file_augmented,options)
         
-    # ...for each image
     
-    if not options.warning_mode:
-        
-        assert max_conf_error <= options.max_conf_error, \
-            'Confidence error {} is greater than allowable ({})'.format(
-                max_conf_error,options.max_conf_error)
-        
-        assert max_coord_error <= options.max_coord_error, \
-            'Coord error {} is greater than allowable ({})'.format(
-                max_coord_error,options.max_coord_error)
-        
-    print('Max conf error: {}'.format(max_conf_error))
-    print('Max coord error: {}'.format(max_coord_error))
-
-
     ## Postprocess results
     
     from megadetector.postprocessing.postprocess_batch_results import \
@@ -639,6 +709,8 @@ def run_python_tests(options):
         
         ## Video test (single video)
        
+        print('\n** Running MD on a single video **\n')
+        
         from megadetector.detection.process_video import ProcessVideoOptions, process_video
         
         video_options = ProcessVideoOptions()
@@ -660,7 +732,7 @@ def run_python_tests(options):
         video_options.fourcc = options.video_fourcc
         # video_options.rendering_confidence_threshold = None
         # video_options.json_confidence_threshold = 0.005
-        video_options.frame_sample = 5    
+        video_options.frame_sample = 10
         video_options.n_cores = 5
         # video_options.debug_max_frames = -1
         # video_options.class_mapping_filename = None
@@ -675,6 +747,8 @@ def run_python_tests(options):
         
         ## Video test (folder)
         
+        print('\n** Running MD on a folder of videos **\n')
+        
         from megadetector.detection.process_video import ProcessVideoOptions, process_video_folder
         
         video_options = ProcessVideoOptions()
@@ -697,15 +771,30 @@ def run_python_tests(options):
         video_options.fourcc = options.video_fourcc
         # video_options.rendering_confidence_threshold = None
         # video_options.json_confidence_threshold = 0.005
-        video_options.frame_sample = 5    
-        video_options.n_cores = 5
+        video_options.frame_sample = 10  
+        video_options.n_cores = 5        
         # video_options.debug_max_frames = -1
         # video_options.class_mapping_filename = None
         
+        # Use quality == None, because we can't control whether YOLOv5 has patched cm2.imread,
+        # and therefore can't rely on using the quality parameter
+        video_options.quality = None        
+        
         _ = process_video_folder(video_options)
     
         assert os.path.isfile(video_options.output_json_file), \
             'Python video test failed to render output .json file'
+            
+        frame_output_file = insert_before_extension(video_options.output_json_file,'frames')
+        assert os.path.isfile(frame_output_file)
+        
+        
+        ## Verify results
+        
+        expected_results_file = \
+            get_expected_results_filename(is_gpu_available(verbose=False),test_type='video',options=options)
+        assert os.path.isfile(expected_results_file)
+        compare_results(frame_output_file,expected_results_file,options)
         
     # ...if we're not skipping video tests
     
@@ -754,7 +843,6 @@ def run_cli_tests(options):
         cmd = 'python megadetector/detection/run_detector.py'
     cmd += ' "{}" --image_file "{}" --output_dir "{}"'.format(
         options.default_model,image_fn,output_dir)
-    print('Running: {}'.format(cmd))
     cmd_results = execute_and_print(cmd)
     
     if options.cpu_execution_is_error:
@@ -769,6 +857,7 @@ def run_cli_tests(options):
     
     ## Run inference on a folder
     
+    
     image_folder = os.path.join(options.scratch_dir,'md-test-images')
     assert os.path.isdir(image_folder), 'Test image folder {} is not available'.format(image_folder)
     inference_output_file = os.path.join(options.scratch_dir,'folder_inference_output.json')
@@ -780,24 +869,75 @@ def run_cli_tests(options):
         options.default_model,image_folder,inference_output_file)
     cmd += ' --output_relative_filenames --quiet --include_image_size'
     cmd += ' --include_image_timestamp --include_exif_data'
-    print('Running: {}'.format(cmd))
     cmd_results = execute_and_print(cmd)
     
+    base_cmd = cmd
+    
     
     ## Run again with checkpointing enabled, make sure the results are the same
     
-    cmd += ' --checkpoint_frequency 5'
     from megadetector.utils.path_utils import insert_before_extension
+        
+    checkpoint_string = ' --checkpoint_frequency 5'
+    cmd = base_cmd + checkpoint_string
     inference_output_file_checkpoint = insert_before_extension(inference_output_file,'_checkpoint')
-    assert inference_output_file_checkpoint != inference_output_file
     cmd = cmd.replace(inference_output_file,inference_output_file_checkpoint)
-    print('Running: {}'.format(cmd))
     cmd_results = execute_and_print(cmd)
     
     assert output_files_are_identical(fn1=inference_output_file, 
                                       fn2=inference_output_file_checkpoint,verbose=True)
     
     
+    ## Run again with the image queue enabled, make sure the results are the same
+    
+    cmd = base_cmd + ' --use_image_queue'
+    from megadetector.utils.path_utils import insert_before_extension
+    inference_output_file_queue = insert_before_extension(inference_output_file,'_queue')
+    cmd = cmd.replace(inference_output_file,inference_output_file_queue)
+    cmd_results = execute_and_print(cmd)
+    
+    assert output_files_are_identical(fn1=inference_output_file, 
+                                      fn2=inference_output_file_queue,verbose=True)
+    
+    
+    ## Run again on multiple cores, make sure the results are the same
+    
+    # First run again on the CPU on a single thread if necessary, so we get a file that 
+    # *should* be identical to the multicore version.
+    
+    gpu_available = is_gpu_available(verbose=False)
+    
+    cuda_visible_devices = None
+    if 'CUDA_VISIBLE_DEVICES' in os.environ:
+        cuda_visible_devices = os.environ['CUDA_VISIBLE_DEVICES']
+    os.environ['CUDA_VISIBLE_DEVICES'] = '-1'    
+    
+    # If we already ran on the CPU, no need to run again
+    if not gpu_available:
+        inference_output_file_cpu = inference_output_file
+    else:
+        inference_output_file_cpu = insert_before_extension(inference_output_file,'cpu')    
+        cmd = base_cmd
+        cmd = cmd.replace(inference_output_file,inference_output_file_cpu)    
+        cmd_results = execute_and_print(cmd)
+        
+    cpu_string = ' --ncores 4'
+    cmd = base_cmd + cpu_string
+    from megadetector.utils.path_utils import insert_before_extension
+    inference_output_file_cpu_multicore = insert_before_extension(inference_output_file,'multicore')
+    cmd = cmd.replace(inference_output_file,inference_output_file_cpu_multicore)
+    cmd_results = execute_and_print(cmd)
+    
+    if cuda_visible_devices is not None:
+        print('Restoring CUDA_VISIBLE_DEVICES')
+        os.environ['CUDA_VISIBLE_DEVICES'] = cuda_visible_devices
+    else:
+        del os.environ['CUDA_VISIBLE_DEVICES']
+        
+    assert output_files_are_identical(fn1=inference_output_file_cpu, 
+                                      fn2=inference_output_file_cpu_multicore,verbose=True)
+    
+    
     ## Postprocessing
     
     postprocessing_output_dir = os.path.join(options.scratch_dir,'postprocessing_output_cli')
@@ -809,7 +949,6 @@ def run_cli_tests(options):
     cmd += ' "{}" "{}"'.format(
         inference_output_file,postprocessing_output_dir)
     cmd += ' --image_base_dir "{}"'.format(image_folder)
-    print('Running: {}'.format(cmd))
     cmd_results = execute_and_print(cmd)
                 
     
@@ -825,7 +964,6 @@ def run_cli_tests(options):
     cmd += ' --imageBase "{}"'.format(image_folder)
     cmd += ' --outputBase "{}"'.format(rde_output_dir)
     cmd += ' --occurrenceThreshold 1' # Use an absurd number here to make sure we get some suspicious detections
-    print('Running: {}'.format(cmd))
     cmd_results = execute_and_print(cmd)    
     
     # Find the latest filtering folder
@@ -844,7 +982,6 @@ def run_cli_tests(options):
     else:
         cmd = 'python  megadetector/postprocessing/repeat_detection_elimination/remove_repeat_detections.py'
     cmd += ' "{}" "{}" "{}"'.format(inference_output_file,filtered_output_file,filtering_output_dir)
-    print('Running: {}'.format(cmd))
     cmd_results = execute_and_print(cmd)
     
     assert os.path.isfile(filtered_output_file), \
@@ -863,14 +1000,13 @@ def run_cli_tests(options):
     cmd += ' "{}" "{}" "{}" "{}"'.format(
         options.default_model,image_folder,tiling_folder,inference_output_file_tiled)
     cmd += ' --overwrite_handling overwrite'
-    print('Running: {}'.format(cmd))
     cmd_results = execute_and_print(cmd)
     
     with open(inference_output_file_tiled,'r') as f:
         results_from_file = json.load(f) # noqa
         
     
-    ## Run inference on a folder (augmented)
+    ## Run inference on a folder (augmented, w/YOLOv5 val script)
     
     if options.yolo_working_dir is None:
         
@@ -894,7 +1030,6 @@ def run_cli_tests(options):
         cmd += ' --augment_enabled 1'
         # cmd += ' --no_use_symlinks'
         cmd += ' --overwrite_handling overwrite'
-        print('Running: {}'.format(cmd))
         cmd_results = execute_and_print(cmd)
         
         # Run again with checkpointing, make sure the output are identical
@@ -931,7 +1066,6 @@ def run_cli_tests(options):
         cmd += ' --render_output_video --fourcc {}'.format(options.video_fourcc)
         cmd += ' --force_extracted_frame_folder_deletion --force_rendered_frame_folder_deletion --n_cores 5 --frame_sample 3'
         cmd += ' --verbose'
-        print('Running: {}'.format(cmd))
         cmd_results = execute_and_print(cmd)
 
     # ...if we're not skipping video tests
@@ -949,7 +1083,6 @@ def run_cli_tests(options):
         options.alt_model,image_folder,inference_output_file_alt)
     cmd += ' --output_relative_filenames --quiet --include_image_size'
     cmd += ' --include_image_timestamp --include_exif_data'
-    print('Running: {}'.format(cmd))
     cmd_results = execute_and_print(cmd)
     
     with open(inference_output_file_alt,'r') as f:
@@ -967,7 +1100,6 @@ def run_cli_tests(options):
     else:
         cmd = 'python megadetector/postprocessing/compare_batch_results.py'
     cmd += ' "{}" "{}" {}'.format(comparison_output_folder,image_folder,results_files_string)
-    print('Running: {}'.format(cmd))
     cmd_results = execute_and_print(cmd)
     
     assert cmd_results['status'] == 0, 'Error generating comparison HTML'
@@ -1040,6 +1172,10 @@ if False:
     options.cli_working_dir = r'c:\git\MegaDetector'
     options.yolo_working_dir = r'c:\git\yolov5-md'
 
+    import os
+    
+    if 'PYTHONPATH' not in os.environ or options.yolo_working_dir not in os.environ['PYTHONPATH']:
+        os.environ['PYTHONPATH'] += ';' + options.yolo_working_dir
 
     #%%
     
@@ -1150,15 +1286,50 @@ if __name__ == '__main__':
 
 #%% Sample invocations
 
-"""
+r"""
 # Windows
 set PYTHONPATH=c:\git\MegaDetector;c:\git\yolov5-md
+cd c:\git\MegaDetector\megadetector\utils
 python md_tests.py --cli_working_dir "c:\git\MegaDetector" --yolo_working_dir "c:\git\yolov5-md" --cli_test_pythonpath "c:\git\MegaDetector;c:\git\yolov5-md"
 
 # Linux
 export PYTHONPATH=/mnt/c/git/MegaDetector:/mnt/c/git/yolov5-md
+cd /mnt/c/git/MegaDetector/megadetector/utils
 python md_tests.py --cli_working_dir "/mnt/c/git/MegaDetector" --yolo_working_dir "/mnt/c/git/yolov5-md" --cli_test_pythonpath "/mnt/c/git/MegaDetector:/mnt/c/git/yolov5-md"
 
 python -c "import md_tests; print(md_tests.get_expected_results_filename(True))"
 """
 
+
+#%% Scrap
+
+if False:
+
+    pass
+
+    #%%
+    
+    import sys; sys.path.append(r'c:\git\yolov5-md')
+    
+    #%%
+    
+    fn1 = r"G:\temp\md-test-package\mdv5a-video-cpu-pt1.10.1.frames.json"
+    fn2 = r"G:\temp\md-test-package\mdv5a-video-gpu-pt1.10.1.frames.json"
+    fn3 = r"G:\temp\md-test-package\mdv5a-video-cpu-pt2.x.frames.json"
+    fn4 = r"G:\temp\md-test-package\mdv5a-video-gpu-pt2.x.frames.json"
+    
+    assert all([os.path.isfile(fn) for fn in [fn1,fn2,fn3,fn4]])
+    print(output_files_are_identical(fn1,fn1,verbose=False))
+    print(output_files_are_identical(fn1,fn2,verbose=False))
+    print(output_files_are_identical(fn1,fn3,verbose=False))
+    
+    #%%
+    
+    fn1 = r"G:\temp\md-test-package\mdv5a-image-gpu-pt1.10.1.json"
+    fn2 = r"G:\temp\md-test-package\mdv5a-augment-image-gpu-pt1.10.1.json"
+    print(output_files_are_identical(fn1,fn2,verbose=True))
+    
+    fn1 = r"G:\temp\md-test-package\mdv5a-image-cpu-pt1.10.1.json"
+    fn2 = r"G:\temp\md-test-package\mdv5a-augment-image-cpu-pt1.10.1.json"
+    print(output_files_are_identical(fn1,fn2,verbose=True))
+