megadetector 5.0.24__py3-none-any.whl → 5.0.25__py3-none-any.whl

megadetector/utils/ct_utils.py

@@ -517,6 +517,52 @@ def invert_dictionary(d):
     return {v: k for k, v in d.items()}
 
 
+def round_floats_in_nested_dict(obj, decimal_places=5):
+    """
+    Recursively rounds all floating point values in a nested structure to the 
+    specified number of decimal places. Handles dictionaries, lists, tuples, 
+    sets, and other iterables. Modifies mutable objects in place.
+    
+    Args:
+        obj: The object to process (can be a dict, list, set, tuple, or primitive value)
+        decimal_places: Number of decimal places to round to (default: 5)
+    
+    Returns:
+        The processed object (useful for recursive calls)
+    """
+    if isinstance(obj, dict):
+        for key in obj:
+            obj[key] = round_floats_in_nested_dict(obj[key], decimal_places)
+        return obj
+    
+    elif isinstance(obj, list):
+        for i in range(len(obj)):
+            obj[i] = round_floats_in_nested_dict(obj[i], decimal_places)
+        return obj
+    
+    elif isinstance(obj, tuple):
+        # Tuples are immutable, so we create a new one
+        return tuple(round_floats_in_nested_dict(item, decimal_places) for item in obj)
+    
+    elif isinstance(obj, set):
+        # Sets are mutable but we can't modify elements in-place
+        # Convert to list, process, and convert back to set
+        return set(round_floats_in_nested_dict(list(obj), decimal_places))
+    
+    elif hasattr(obj, '__iter__') and not isinstance(obj, (str, bytes, bytearray)):
+        # Handle other iterable types - convert to list, process, and convert back
+        return type(obj)(round_floats_in_nested_dict(item, decimal_places) for item in obj)
+    
+    elif isinstance(obj, float):
+        return round(obj, decimal_places)
+    
+    else:
+        # For other types (int, str, bool, None, etc.), return as is
+        return obj
+
+# ...def round_floats_in_nested_dict(...)    
+
+
 def image_file_to_camera_folder(image_fn):
     r"""
     Removes common overflow folders (e.g. RECNX101, RECNX102) from paths, i.e. turn:
@@ -780,7 +826,7 @@ def dict_to_kvp_list(d,
     if len(d) == 0:
         return ''
     
-    s = ''
+    s = None
     for k in d.keys():
         assert isinstance(k,str), 'Input {} is not a str <--> str dict'.format(str(d))
         v = d[k]
@@ -800,6 +846,9 @@ def dict_to_kvp_list(d,
             s += item_separator
         s += k + kv_separator + v
     
+    if s is None:
+        s = ''
+        
     return s
     
 
@@ -856,3 +905,25 @@ def __module_test__():
     L = [{'a':5},{'a':0},{'a':10}]
     k = 'a'
     sort_list_of_dicts_by_key(L, k, reverse=True)
+
+
+    ##%% Test float rounding
+    
+    # Example with mixed collection types
+    data = {
+        "name": "Project X",
+        "values": [1.23456789, 2.3456789],
+        "tuple_values": (3.45678901, 4.56789012),
+        "set_values": {5.67890123, 6.78901234},
+        "metrics": {
+            "score": 98.7654321,
+            "components": [5.6789012, 6.7890123]
+        }
+    }
+    
+    result = round_floats_in_nested_dict(data)
+    assert result['values'][0] == 1.23457
+    assert result['tuple_values'][0] == 3.45679
+    assert min(list(result['set_values'])) == 5.6789
+    
+    

megadetector/utils/directory_listing.py

@@ -17,9 +17,6 @@ import sys
 import argparse
 import re
 
-import azure.common
-from azure.storage.blob import BlobServiceClient, ContentSettings
-
 from megadetector.utils.path_utils import is_image_file
 
 
@@ -139,6 +136,8 @@ def traverse_and_create_index(dir, sas_url=None, overwrite_files=False,
     # If we want to set the content type in blob storage using a SAS URL
     if sas_url:
 
+        from azure.storage.blob import BlobServiceClient, ContentSettings
+
         # Example: sas_url = 'https://accname.blob.core.windows.net/bname/path/to/folder?st=...&se=...&sp=...&...'
         if '?' in sas_url:
             # 'https://accname.blob.core.windows.net/bname/path/to/folder' and 'st=...&se=...&sp=...&...'
@@ -196,6 +195,7 @@ def traverse_and_create_index(dir, sas_url=None, overwrite_files=False,
 
         # Set content type in blob storage
         if sas_url:
+            import azure.common
             if container_folder:
                 output_blob_path = container_folder + '/' + output_file[len(dir) + 1:]
             else:

megadetector/utils/gpu_test.py

@@ -7,6 +7,17 @@ for TF or PyTorch
 
 """
 
+# Minimize TF printouts
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1'
+
+try:
+    import logging
+    logging.getLogger('tensorflow').setLevel(logging.ERROR)
+except Exception:
+    pass
+
+       
 #%% Torch/TF test functions
 
 def torch_test():
@@ -45,7 +56,7 @@ def torch_test():
                 pass
             print('{}: {}'.format(device_id,device_name))
     else:
-        print('No CUDA GPUs reported by PyTorch')
+        print('No GPUs reported by PyTorch')
         
     try:
         if torch.backends.mps.is_built and torch.backends.mps.is_available():
@@ -72,8 +83,15 @@ def tf_test():
     
     from tensorflow.python.platform import build_info as build
     print(f"TF version: {tf.__version__}")
-    print(f"CUDA build version reported by TensorFlow: {build.build_info['cuda_version']}")
-    print(f"CuDNN build version reported by TensorFlow: {build.build_info['cudnn_version']}")
+    
+    if 'cuda_version' not in build.build_info:
+        print('TF does not appear to be built with CUDA')
+    else:
+        print(f"CUDA build version reported by TensorFlow: {build.build_info['cuda_version']}")
+    if 'cudnn_version' not in build.build_info:
+        print('TF does not appear to be built with CuDNN')
+    else:
+        print(f"CuDNN build version reported by TensorFlow: {build.build_info['cudnn_version']}")
 
     try:
         from tensorflow.python.compiler.tensorrt import trt_convert as trt

megadetector/utils/md_tests.py

@@ -131,6 +131,15 @@ class MDTestOptions:
         #: this is a range from 0-100.
         self.python_test_depth = 100
         
+        #: Currently should be 'all' or 'utils-only'
+        self.test_mode = 'all'
+        
+        #: Number of cores to use for multi-CPU inference tests
+        self.n_cores_for_multiprocessing_tests = 2
+        
+        #: Number of cores to use for multi-CPU video tests
+        self.n_cores_for_video_tests = 2
+                
     # ...def __init__()
     
 # ...class MDTestOptions()
@@ -302,18 +311,24 @@ def download_test_data(options=None):
 
 def is_gpu_available(verbose=True):
     """
-    Checks whether a GPU (including M1/M2 MPS) is available, according to PyTorch.
+    Checks whether a GPU (including M1/M2 MPS) is available, according to PyTorch.  Returns 
+    false if PT fails to import.
     
     Args:
-        verbose (bool, optional): enable additional debug console output
+        verbose (bool, optional): enable additional debug console output        
     
     Returns:
-        bool: whether a GPU is available
+        bool: whether a GPU is available        
     """
     
     # Import torch inside this function, so we have a chance to set CUDA_VISIBLE_DEVICES
     # before checking GPU availability.
-    import torch
+    try:
+        import torch
+    except Exception:
+        print('Warning: could not import torch')
+        return False
+    
     gpu_available = torch.cuda.is_available()
     
     if gpu_available:
@@ -705,6 +720,47 @@ def execute_and_print(cmd,print_output=True,catch_exceptions=False,echo_command=
 
 #%% Python tests
 
+def test_package_imports(package_name,exceptions=None,verbose=True):
+    """
+    Imports all modules in [package_name]
+    
+    Args:
+        package_name (str): the package name to test
+        exceptions (list, optional): exclude any modules that contain any of these strings
+        verbose (bool, optional): enable additional debug output
+    """
+    import importlib
+    import pkgutil
+    
+    package = importlib.import_module(package_name)
+    package_path = package.__path__
+    imported_modules = []
+    
+    if exceptions is None:
+        exceptions = []
+        
+    for _, modname, _ in pkgutil.walk_packages(package_path, package_name + '.'):
+        
+        skip_module = False
+        for s in exceptions:
+            if s in modname:
+                skip_module = True
+                break
+        if skip_module:
+            continue
+        
+        if verbose:
+            print('Testing import: {}'.format(modname))
+        
+        try:
+            # Attempt to import each module
+            _ = importlib.import_module(modname)
+            imported_modules.append(modname)            
+        except ImportError as e:
+            print(f"Failed to import module {modname}: {e}")
+            raise
+
+    #%%
 def run_python_tests(options):
     """
     Runs Python-based (as opposed to CLI-based) package tests.
@@ -716,11 +772,6 @@ def run_python_tests(options):
     print('\n*** Starting module tests ***\n')
         
     
-    ## Make sure our tests are doing what we think they're doing
-    
-    from megadetector.detection import pytorch_detector
-    pytorch_detector.require_non_default_compatibility_mode = True
-    
     ## Prepare data
     
     download_test_data(options)
@@ -734,12 +785,34 @@ def run_python_tests(options):
     ct_utils_test()
     
     
+    ## Import tests
+    
+    print('\n** Running package import tests **\n')
+    test_package_imports('megadetector.visualization')
+    test_package_imports('megadetector.postprocessing')
+    test_package_imports('megadetector.postprocessing.repeat_detection_elimination')
+    test_package_imports('megadetector.utils',exceptions=['azure_utils','sas_blob_utils','md_tests'])
+    test_package_imports('megadetector.data_management',exceptions=['lila','ocr_tools'])
+        
+
+    ## Return early if we're not running torch-related tests
+    
+    if options.test_mode == 'utils-only':
+        return
+    
+    
+    ## Make sure our tests are doing what we think they're doing
+    
+    from megadetector.detection import pytorch_detector
+    pytorch_detector.require_non_default_compatibility_mode = True
+    
+    
     ## Run inference on an image
         
     print('\n** Running MD on a single image (module) **\n')
     
     from megadetector.detection import run_detector
-    from megadetector.visualization import visualization_utils as vis_utils
+    from megadetector.visualization import visualization_utils as vis_utils # noqa
     image_fn = os.path.join(options.scratch_dir,options.test_images[0])
     model = run_detector.load_detector(options.default_model,
                                        detector_options=copy(options.detector_options))
@@ -755,7 +828,7 @@ def run_python_tests(options):
     print('\n** Running MD on a folder of images (module) **\n')
     
     from megadetector.detection.run_detector_batch import load_and_run_detector_batch,write_results_to_file
-    from megadetector.utils import path_utils
+    from megadetector.utils import path_utils # noqa
 
     image_folder = os.path.join(options.scratch_dir,'md-test-images')
     assert os.path.isdir(image_folder), 'Test image folder {} is not available'.format(image_folder)
@@ -773,7 +846,7 @@ def run_python_tests(options):
     ## Verify results
     
     # Verify format correctness
-    from megadetector.postprocessing.validate_batch_results import validate_batch_results 
+    from megadetector.postprocessing.validate_batch_results import validate_batch_results #noqa
     validate_batch_results(inference_output_file)
     
     # Verify value correctness
@@ -955,7 +1028,7 @@ def run_python_tests(options):
         # video_options.rendering_confidence_threshold = None
         # video_options.json_confidence_threshold = 0.005
         video_options.frame_sample = 10
-        video_options.n_cores = 5
+        video_options.n_cores = options.n_cores_for_video_tests
         # video_options.debug_max_frames = -1
         # video_options.class_mapping_filename = None
         video_options.detector_options = copy(options.detector_options)
@@ -996,7 +1069,7 @@ def run_python_tests(options):
         # video_options.rendering_confidence_threshold = None
         # video_options.json_confidence_threshold = 0.005
         video_options.frame_sample = 10  
-        video_options.n_cores = 5     
+        video_options.n_cores = options.n_cores_for_video_tests
         
         # Force frame extraction to disk, since that's how we generated our expected results file
         video_options.force_on_disk_frame_extraction = True
@@ -1092,6 +1165,17 @@ def run_cli_tests(options):
     from megadetector.utils.path_utils import insert_before_extension
     
     
+    ## Utility tests
+    
+    # TODO: move postprocessing tests up to this point, using pre-generated .json results files
+    
+
+    ## Return early if we're not running torch-related tests
+    
+    if options.test_mode == 'utils-only':
+        return
+
+    
     ## Run inference on an image
     
     print('\n** Running MD on a single image (CLI) **\n')
@@ -1218,7 +1302,7 @@ def run_cli_tests(options):
             
         print('\n** Running MD on a folder (multiple CPUs) (CLI) **\n')
         
-        cpu_string = ' --ncores 4'
+        cpu_string = ' --ncores {}'.format(options.n_cores_for_multiprocessing_tests)
         cmd = base_cmd + cpu_string
         inference_output_file_cpu_multicore = insert_before_extension(inference_output_file,'multicore')
         cmd = cmd.replace(inference_output_file,inference_output_file_cpu_multicore)
@@ -1385,7 +1469,9 @@ def run_cli_tests(options):
         cmd += ' --frame_folder "{}" --frame_rendering_folder "{}" --output_json_file "{}" --output_video_file "{}"'.format(
             frame_folder,frame_rendering_folder,video_inference_output_file,output_video_file)
         cmd += ' --fourcc {}'.format(options.video_fourcc)
-        cmd += ' --force_extracted_frame_folder_deletion --force_rendered_frame_folder_deletion --n_cores 5 --frame_sample 3'
+        cmd += ' --force_extracted_frame_folder_deletion --force_rendered_frame_folder_deletion'
+        cmd += ' --n_cores {}'.format(options.n_cores_for_video_tests)
+        cmd += ' --frame_sample 4'
         cmd += ' --verbose'
         cmd += ' --detector_options {}'.format(dict_to_kvp_list(options.detector_options))
         
@@ -1446,40 +1532,39 @@ def run_download_tests(options):
         options (MDTestOptions): see MDTestOptions for details    
     """
     
-    if not options.skip_download_tests:
+    if options.skip_download_tests or options.test_mode == 'utils-only':
+        return
 
-        from megadetector.detection.run_detector import known_models, \
-            try_download_known_detector, \
-            get_detector_version_from_model_file, \
-            model_string_to_model_version
+    from megadetector.detection.run_detector import known_models, \
+        try_download_known_detector, \
+        get_detector_version_from_model_file, \
+        model_string_to_model_version
 
-        # Make sure we can download models based on canonical version numbers, 
-        # e.g. "v5a.0.0"
-        for model_name in known_models:
-            url = known_models[model_name]['url']
-            if 'localhost' in url:
-                continue
-            print('Testing download for known model {}'.format(model_name))
-            fn = try_download_known_detector(model_name, 
-                                             force_download=False,
-                                             verbose=False)
-            version_string = get_detector_version_from_model_file(fn, verbose=False)
-            assert version_string == model_name
-            
-        # Make sure we can download models based on short names, e.g. "MDV5A"
-        for model_name in model_string_to_model_version:
-            model_version = model_string_to_model_version[model_name]
-            assert model_version in known_models
-            url = known_models[model_version]['url']
-            if 'localhost' in url:
-                continue
-            print('Testing download for model short name {}'.format(model_name))
-            fn = try_download_known_detector(model_name, 
-                                             force_download=False,
-                                             verbose=False)    
-            assert fn != model_name
-
-    # ...if we need to test model downloads
+    # Make sure we can download models based on canonical version numbers, 
+    # e.g. "v5a.0.0"
+    for model_name in known_models:
+        url = known_models[model_name]['url']
+        if 'localhost' in url:
+            continue
+        print('Testing download for known model {}'.format(model_name))
+        fn = try_download_known_detector(model_name, 
+                                         force_download=False,
+                                         verbose=False)
+        version_string = get_detector_version_from_model_file(fn, verbose=False)
+        assert version_string == model_name
+        
+    # Make sure we can download models based on short names, e.g. "MDV5A"
+    for model_name in model_string_to_model_version:
+        model_version = model_string_to_model_version[model_name]
+        assert model_version in known_models
+        url = known_models[model_version]['url']
+        if 'localhost' in url:
+            continue
+        print('Testing download for model short name {}'.format(model_name))
+        fn = try_download_known_detector(model_name, 
+                                         force_download=False,
+                                         verbose=False)    
+        assert fn != model_name    
     
 # ...def run_download_tests()
 
@@ -1497,7 +1582,7 @@ def run_tests(options):
     # Prepare data folder
     download_test_data(options)
     
-    # Run download tests if necessary
+    # Run model download tests if necessary
     run_download_tests(options)
     
     if options.disable_gpu:
@@ -1740,6 +1825,13 @@ def main():
         help='PYTHONPATH to set for CLI tests; if None, inherits from the parent process'
         )
     
+    parser.add_argument(
+        '--test_mode',
+        type=str,
+        default='utils-only',
+        help='Test mode: "all" or "utils-only"'
+        )
+    
     parser.add_argument(
         '--python_test_depth',
         type=int,

megadetector/utils/path_utils.py

@@ -547,8 +547,42 @@ def wsl_path_to_windows_path(filename):
     if result.returncode != 0:
         print('Could not convert path {} from WSL to Windows'.format(filename))
         return None
+    
     return result.stdout.strip()
     
+
+def windows_path_to_wsl_path(filename):
+    r"""
+    Converts a Windows path to a WSL path, or returns None if that's not possible.  E.g.
+    converts:
+        
+    e:\a\b\c
+    
+    ...to:
+        
+    /mnt/e/a/b/c
+    
+    Args:
+        filename (str): filename to convert
+    
+    Returns:
+        str: WSL equivalent to the Windows path [filename], or [filename] if the current
+        environment is neither Windows nor WSL.
+    """
+    
+    if (not environment_is_wsl()) and (os.name != 'nt'):
+        return filename
+    
+    if environment_is_wsl():
+        result = subprocess.run(['wslpath', '-u', filename], text=True, capture_output=True)
+    else:
+        result = subprocess.run(['wsl', 'wslpath', '-u', filename], text=True, capture_output=True)
+    if result.returncode != 0:
+        print('Could not convert path {} from Windows to WSL'.format(filename))
+        return None
+    
+    return result.stdout.strip()
+
     
 def open_file(filename, attempt_to_open_in_wsl_host=False, browser_name=None):
     """