megadetector 5.0.29__py3-none-any.whl → 10.0.1__py3-none-any.whl

megadetector/utils/md_tests.py

@@ -65,6 +65,9 @@ class MDTestOptions:
         #: Skip download tests
         self.skip_download_tests = False
 
+        #: Skip download tests for local URLs
+        self.skip_localhost_downloads = False
+
         #: Skip force-CPU tests
         self.skip_cpu_tests = False
 
@@ -163,6 +166,10 @@ def get_expected_results_filename(gpu_is_available,
 
     Args:
         gpu_is_available (bool): whether a GPU is available
+        model_string (str, optional): the model for which we're retrieving expected results
+        test_type (str, optional): the test type we're running ("image" or "video")
+        augment (bool, optional): whether we're running this test with image augmentation
+        options (MDTestOptiosn, optional): additional control flow options
 
     Returns:
         str: relative filename of the results file we should use (within the test
@@ -361,6 +368,7 @@ def output_files_are_identical(fn1,fn2,verbose=False):
     Args:
         fn1 (str): the first filename to compare
         fn2 (str): the second filename to compare
+        verbose (bool, optional): enable additional debug output
 
     Returns:
         bool: whether [fn1] and [fn2] are identical other than file sorting.
@@ -692,6 +700,9 @@ def execute_and_print(cmd,print_output=True,catch_exceptions=False,echo_command=
     Args:
         cmd (str): command to run
         print_output (bool, optional): whether to print output from [cmd]
+        catch_exceptions (bool, optional): whether to catch exceptions, rather than raising
+            them
+        echo_command (bool, optional): whether to print [cmd] to stdout prior to execution
 
     Returns:
         dict: a dictionary with fields "status" (the process return code) and "output"
@@ -762,7 +773,7 @@ def test_package_imports(package_name,exceptions=None,verbose=True):
             print(f"Failed to import module {modname}: {e}")
             raise
 
-    #%%
+
 def run_python_tests(options):
     """
     Runs Python-based (as opposed to CLI-based) package tests.
@@ -785,7 +796,7 @@ def run_python_tests(options):
     test_package_imports('megadetector.visualization')
     test_package_imports('megadetector.postprocessing')
     test_package_imports('megadetector.postprocessing.repeat_detection_elimination')
-    test_package_imports('megadetector.utils',exceptions=['azure_utils','sas_blob_utils','md_tests'])
+    test_package_imports('megadetector.utils',exceptions=['md_tests'])
     test_package_imports('megadetector.data_management',exceptions=['lila','ocr_tools'])
 
 
@@ -1523,6 +1534,8 @@ def run_cli_tests(options):
 
 def run_download_tests(options):
     """
+    Test automatic model downloads.
+
     Args:
         options (MDTestOptions): see MDTestOptions for details
     """
@@ -1539,14 +1552,15 @@ def run_download_tests(options):
     # e.g. "v5a.0.0"
     for model_name in known_models:
         url = known_models[model_name]['url']
-        if 'localhost' in url:
+        if ('localhost' in url) and options.skip_localhost_downloads:
             continue
         print('Testing download for known model {}'.format(model_name))
         fn = try_download_known_detector(model_name,
                                          force_download=False,
                                          verbose=False)
         version_string = get_detector_version_from_model_file(fn, verbose=False)
-        assert version_string == model_name
+        # Make sure this is the same version we asked for, modulo the MDv5 re-releases
+        assert (version_string.replace('.0.1','.0.0') == model_name.replace('.0.1','.0.0'))
 
     # Make sure we can download models based on short names, e.g. "MDV5A"
     for model_name in model_string_to_model_version:
@@ -1561,6 +1575,53 @@ def run_download_tests(options):
                                          verbose=False)
         assert fn != model_name
 
+    # Test corruption handling for .pt files
+    print('Testing corruption handling for MDV5B')
+
+    # First ensure MDV5B is downloaded
+    mdv5b_file = try_download_known_detector('MDV5B',
+                                             force_download=False,
+                                             verbose=False)
+    assert mdv5b_file is not None
+    assert os.path.exists(mdv5b_file)
+    assert mdv5b_file.endswith('.pt')
+
+    # Get the original file size and MD5 hash for comparison
+    original_size = os.path.getsize(mdv5b_file)
+    from megadetector.utils.path_utils import compute_file_hash
+    original_hash = compute_file_hash(mdv5b_file, algorithm='md5')
+
+    # Deliberately corrupt the file by overwriting the first few bytes
+    print('Corrupting model file: {}'.format(mdv5b_file))
+    with open(mdv5b_file, 'r+b') as f:
+        f.write(b'CORRUPTED_FILE_DATA_XXXXXX')
+
+    # Verify the file is now corrupted (different hash)
+    corrupted_hash = compute_file_hash(mdv5b_file, algorithm='md5')
+    assert corrupted_hash != original_hash, 'File corruption verification failed'
+
+    # Try to download again; this should detect corruption and re-download
+    print('Testing corruption detection and re-download')
+    mdv5b_file_redownloaded = try_download_known_detector('MDV5B',
+                                                          force_download=False,
+                                                          verbose=True)
+
+    # Verify that the file was re-downloaded and is now valid
+    assert mdv5b_file_redownloaded is not None
+    assert os.path.exists(mdv5b_file_redownloaded)
+    assert mdv5b_file_redownloaded == mdv5b_file
+
+    # Verify that the file is back to its original state
+    new_size = os.path.getsize(mdv5b_file_redownloaded)
+    new_hash = compute_file_hash(mdv5b_file_redownloaded, algorithm='md5')
+
+    assert new_size == original_size, \
+        'Re-downloaded file size ({}) does not match original ({})'.format(new_size, original_size)
+    assert new_hash == original_hash, \
+        'Re-downloaded file hash ({}) does not match original ({})'.format(new_hash, original_hash)
+
+    print('Corruption handling test passed')
+
 # ...def run_download_tests()
 
 
@@ -1630,6 +1691,9 @@ def run_tests(options):
 #%% Automated test entry point
 
 def test_suite_entry_point():
+    """
+    Main entry point for the numerical test suite.
+    """
 
     options = MDTestOptions()
     options.disable_gpu = False
@@ -1647,10 +1711,11 @@ def test_suite_entry_point():
     options.skip_video_rendering_tests = True
     options.cli_working_dir = None
     options.cli_test_pythonpath = None
-    
     options.skip_download_tests = True
-    
+    options.skip_localhost_downloads = True
+
     options = download_test_data(options)
+
     run_tests(options)
 
 
@@ -1660,7 +1725,7 @@ if False:
 
     pass
 
-    #%%
+    #%% Test Prep
 
     options = MDTestOptions()
 
@@ -1677,6 +1742,9 @@ if False:
     options.max_coord_error = 0.01 # 0.001
     options.max_conf_error = 0.01 # 0.005
     options.skip_video_rendering_tests = True
+    options.skip_download_tests = False
+    options.skip_localhost_downloads = False
+
     # options.iou_threshold_for_file_comparison = 0.7
 
     options.cli_working_dir = r'c:\git\MegaDetector'
@@ -1691,18 +1759,30 @@ if False:
     # options.yolo_working_dir = '/mnt/c/git/yolov5-md'
     options = download_test_data(options)
 
-    #%%
 
+    #%% Environment prep
+
+    # Add the YOLO working dir to the PYTHONPATH if necessary
     import os
-    if ('PYTHONPATH' not in os.environ) or \
-       (options.yolo_working_dir is not None and options.yolo_working_dir not in os.environ['PYTHONPATH']):
-        os.environ['PYTHONPATH'] += ';' + options.yolo_working_dir
+    if (options.yolo_working_dir is not None) and \
+        (('PYTHONPATH' not in os.environ) or (options.yolo_working_dir not in os.environ['PYTHONPATH'])):
+        if ('PYTHONPATH' not in os.environ):
+            os.environ['PYTHONPATH'] = options.yolo_working_dir
+        else:
+            os.environ['PYTHONPATH'] = os.environ['PYTHONPATH'] + ';' + options.yolo_working_dir
 
-    #%%
+
+    #%% Run download tests
+
+    run_download_tests(options=options)
+
+
+    #%% Run all tests
 
     run_tests(options)
 
-    #%%
+
+    #%% Run YOLO inference tests
 
     yolo_inference_options_dict = {'input_folder': '/tmp/md-tests/md-test-images',
                                    'image_filename_list': None,