megadetector 5.0.27__py3-none-any.whl → 5.0.28__py3-none-any.whl

megadetector/data_management/mewc_to_md.py

@@ -242,7 +242,7 @@ def mewc_to_md(mewc_input_folder,
         
         # ...for each image
         
-        ##%% Map MD reults to the global level
+        ##%% Map MD results to the global level
         
         if md_results_all['info'] is None:
             md_results_all['info'] = md_results['info']

megadetector/data_management/read_exif.py

@@ -31,6 +31,8 @@ from megadetector.data_management.cct_json_utils import write_object_with_serial
 
 debug_max_images = None
 
+minimal_exif_tags = ['DateTime','Model','Make','ExifImageWidth','ExifImageHeight','DateTimeOriginal','Orientation']    
+
 
 #%% Options
 

megadetector/detection/process_video.py

@@ -140,7 +140,7 @@ class ProcessVideoOptions:
         #: [frame_sample] and [time_sample].
         self.frames_to_extract = None
         
-        # Sample frames every N seconds.  Mutally exclusive with [frame_sample] and [frames_to_extract].
+        # Sample frames every N seconds.  Mutually exclusive with [frame_sample] and [frames_to_extract].
         self.time_sample = None
         
         #: Number of workers to use for parallelization; set to <= 1 to disable parallelization

megadetector/detection/pytorch_detector.py

@@ -107,7 +107,7 @@ def _get_model_type_for_model(model_file,
         if model_type_from_model_version == model_type_from_model_file_metadata:
             model_type = model_type_from_model_file_metadata
         else:
-            print('Waring: model type from model version is {}, from file metadata is {}'.format(
+            print('Warning: model type from model version is {}, from file metadata is {}'.format(
                 model_type_from_model_version,model_type_from_model_file_metadata))
             if prefer_model_type_source == 'table':
                 model_type = model_type_from_model_file_metadata
@@ -145,7 +145,7 @@ def _initialize_yolo_imports_for_model(model_file,
             appropriate metadata in the file or in the global table.        
         detector_options (dict, optional): dictionary of detector options that mean
             different things to different models
-        verbose (bool, optional): enable additonal debug output
+        verbose (bool, optional): enable additional debug output
             
     Returns:
         str: the model type for which we initialized support
@@ -238,7 +238,7 @@ def _initialize_yolo_imports(model_type='yolov5',
             typically used when the right support library is on the current PYTHONPATH.
         force_reimport (bool, optional): import the appropriate libraries even if the 
             requested model type matches the current initialization state
-        verbose (bool, optional): include additonal debug output
+        verbose (bool, optional): include additional debug output
             
     Returns:
         str: the model type for which we initialized support
@@ -527,7 +527,7 @@ def read_metadata_from_megadetector_model_file(model_file,
 default_compatibility_mode = 'classic'
 
 # This is a useful hack when I want to verify that my test driver (md_tests.py) is 
-# correctly forcing a specific compabitility mode (I use "classic-test" in that case)
+# correctly forcing a specific compatibility mode (I use "classic-test" in that case)
 require_non_default_compatibility_mode = False
 
 class PTDetector:

megadetector/detection/run_detector.py

@@ -29,6 +29,7 @@ import os
 import statistics
 import sys
 import time
+import json
 import warnings
 import tempfile
 
@@ -214,8 +215,8 @@ def get_detector_metadata_from_version_string(detector_version):
         print('Warning: no metadata for unknown detector version {}'.format(detector_version))
         default_detector_metadata = {
             'megadetector_version':'unknown',
-            'typical_detection_threshold':0.5,
-            'conservative_detection_threshold':0.25
+            'typical_detection_threshold':0.2,
+            'conservative_detection_threshold':0.1
         }
         return default_detector_metadata
     else:
@@ -423,12 +424,18 @@ def get_typical_confidence_threshold_from_results(results):
     threshold based on the detector version.
     
     Args:
-        results (dict): a dict of MD results, as it would be loaded from a MD results .json file
+        results (dict or str): a dict of MD results, as it would be loaded from a MD results .json 
+            file, or a .json filename
     
     Returns:
         float: a sensible default threshold for this model
     """
     
+    # Load results if necessary
+    if isinstance(results,str):
+        with open(results,'r') as f:
+            results = json.load(f)
+
     if 'detector_metadata' in results['info'] and \
         'typical_detection_threshold' in results['info']['detector_metadata']:
         default_threshold = results['info']['detector_metadata']['typical_detection_threshold']

megadetector/detection/run_detector_batch.py

@@ -130,7 +130,7 @@ def _producer_func(q,
         print('Producer starting: ID {}, preprocessor {}'.format(producer_id,preprocessor))
         sys.stdout.flush()
         
-    if preprocessor is not None:
+    if preprocessor is not None:        
         assert isinstance(preprocessor,str)
         detector_options = deepcopy(detector_options)
         detector_options['preprocess_only'] = True
@@ -161,8 +161,8 @@ def _producer_func(q,
                     
                 image = image_info
                 
-        except Exception:
-            print('Producer process: image {} cannot be loaded'.format(im_file))
+        except Exception as e:
+            print('Producer process: image {} cannot be loaded:\n{}'.format(im_file,str(e)))
             image = run_detector.FAILURE_IMAGE_OPEN            
         
         if verbose:
@@ -354,6 +354,7 @@ def run_detector_with_image_queue(image_files,
     preprocessor = None
     
     if preprocess_on_image_queue:
+        print('Enabling image queue preprocessing')
         preprocessor = model_file
     
     n_total_images = len(image_files)

megadetector/detection/run_tiled_inference.py

@@ -26,16 +26,22 @@ augmentation); the command-line interface only supports standard inference right
 
 import os
 import json
+import tempfile
+import uuid
 
 from tqdm import tqdm
 
 import torch
 from torchvision import ops
 
-from megadetector.detection.run_inference_with_yolov5_val import YoloInferenceOptions,run_inference_with_yolo_val
-from megadetector.detection.run_detector_batch import load_and_run_detector_batch,write_results_to_file
-from megadetector.detection.run_detector import try_download_known_detector
+from megadetector.detection.run_inference_with_yolov5_val import \
+    YoloInferenceOptions,run_inference_with_yolo_val
+from megadetector.detection.run_detector_batch import \
+    load_and_run_detector_batch,write_results_to_file
+from megadetector.detection.run_detector import \
+    try_download_known_detector, CONF_DIGITS, COORD_DIGITS
 from megadetector.utils import path_utils
+from megadetector.utils.ct_utils import round_float_array, round_float
 from megadetector.visualization import visualization_utils as vis_utils
 
 default_patch_overlap = 0.5
@@ -380,13 +386,25 @@ def _extract_tiles_for_image(fn_relative,image_folder,tiling_folder,patch_size,p
     
 #%% Main function
     
-def run_tiled_inference(model_file, image_folder, tiling_folder, output_file,
-                        tile_size_x=1280, tile_size_y=1280, tile_overlap=0.5,
-                        checkpoint_path=None, checkpoint_frequency=-1, remove_tiles=False, 
+def run_tiled_inference(model_file, 
+                        image_folder, 
+                        tiling_folder,
+                        output_file,
+                        tile_size_x=1280,
+                        tile_size_y=1280,
+                        tile_overlap=0.5,
+                        checkpoint_path=None,
+                        checkpoint_frequency=-1,
+                        remove_tiles=False, 
                         yolo_inference_options=None,
                         n_patch_extraction_workers=default_n_patch_extraction_workers,
                         overwrite_tiles=True,
-                        image_list=None):
+                        image_list=None,
+                        augment=False,
+                        detector_options=None,
+                        use_image_queue=True,
+                        preprocess_on_image_queue=True,
+                        inference_size=None):
     """
     Runs inference using [model_file] on the images in [image_folder], fist splitting each image up 
     into tiles of size [tile_size_x] x [tile_size_y], writing those tiles to [tiling_folder],
@@ -397,7 +415,8 @@ def run_tiled_inference(model_file, image_folder, tiling_folder, output_file,
     within that folder, including deleting everything, so it's best if it's a new folder.  
     Conceptually this folder is temporary, it's just helpful in this case to not actually
     use the system temp folder, because the tile cache may be very large, so the caller may 
-    want it to be on a specific drive.
+    want it to be on a specific drive.  If this is None, a new folder will be created in 
+    system temp space.
     
     tile_overlap is the fraction of overlap between tiles.
     
@@ -410,11 +429,12 @@ def run_tiled_inference(model_file, image_folder, tiling_folder, output_file,
     Args:
         model_file (str): model filename (ending in .pt), or a well-known model name (e.g. "MDV5A")
         image_folder (str): the folder of images to proess (always recursive)
-        tiling_folder (str): folder for temporary tile storage; see caveats above
+        tiling_folder (str): folder for temporary tile storage; see caveats above.  Can be None
+            to use system temp space.
         output_file (str): .json file to which we should write MD-formatted results
         tile_size_x (int, optional): tile width
         tile_size_y (int, optional): tile height
-        tile_overlap (float, optional): overlap between adjacenet tiles, as a fraction of the
+        tile_overlap (float, optional): overlap between adjacent tiles, as a fraction of the
             tile size
         checkpoint_path (str, optional): checkpoint path; passed directly to run_detector_batch; see
             run_detector_batch for details
@@ -427,7 +447,17 @@ def run_tiled_inference(model_file, image_folder, tiling_folder, output_file,
             set to <= 1 to disable parallelization
         image_list (list, optional): .json file containing a list of specific images to process.  If 
             this is supplied, and the paths are absolute, [image_folder] will be ignored. If this is supplied,
-            and the paths are relative, they should be relative to [image_folder].
+            and the paths are relative, they should be relative to [image_folder]
+        augment (bool, optional): apply test-time augmentation, only relevant if yolo_inference_options
+            is None
+        detector_options (dict, optional): parameters to pass to run_detector, only relevant if
+            yolo_inference_options is None
+        use_image_queue (bool, optional): whether to use a loader worker queue, only relevant if
+            yolo_inference_options is None 
+        preprocess_on_image_queue (bool, optional): whether the image queue should also be responsible
+            for preprocessing
+        inference_size (int, optional): override the default inference image size, only relevant if
+            yolo_inference_options is None
     
     Returns:
         dict: MD-formatted results dictionary, identical to what's written to [output_file]
@@ -447,6 +477,11 @@ def run_tiled_inference(model_file, image_folder, tiling_folder, output_file,
     patch_stride = (round(patch_size[0]*(1.0-tile_overlap)),
                     round(patch_size[1]*(1.0-tile_overlap)))
     
+    if tiling_folder is None:
+        tiling_folder = \
+            os.path.join(tempfile.gettempdir(), 'md-tiling', str(uuid.uuid1()))
+        print('Creating temporary tiling folder: {}'.format(tiling_folder))
+        
     os.makedirs(tiling_folder,exist_ok=True)
     
     ##%% List files
@@ -545,7 +580,7 @@ def run_tiled_inference(model_file, image_folder, tiling_folder, output_file,
             images_with_patch_errors[patch_info['image_fn']] = patch_info
     
     
-    ##%% Run inference on tiles
+    ##%% Run inference on the folder of tiles
     
     # When running with run_inference_with_yolov5_val, we'll pass the folder
     if yolo_inference_options is not None:
@@ -582,7 +617,12 @@ def run_tiled_inference(model_file, image_folder, tiling_folder, output_file,
                                                         patch_file_names, 
                                                         checkpoint_path=checkpoint_path,
                                                         checkpoint_frequency=checkpoint_frequency,
-                                                        quiet=True)
+                                                        quiet=True,
+                                                        augment=augment,
+                                                        detector_options=detector_options,
+                                                        use_image_queue=use_image_queue,
+                                                        preprocess_on_image_queue=preprocess_on_image_queue,
+                                                        image_size=inference_size)
         
         patch_level_output_file = os.path.join(tiling_folder,folder_name + '_patch_level_results.json')
         
@@ -591,6 +631,7 @@ def run_tiled_inference(model_file, image_folder, tiling_folder, output_file,
                                                     relative_path_base=tiling_folder, 
                                                     detector_file=model_file)
         
+    # ...if we are/aren't using run_inference_with_yolov5_val
     
     ##%% Map patch-level detections back to the original images    
     
@@ -708,6 +749,10 @@ def run_tiled_inference(model_file, image_folder, tiling_folder, output_file,
                                          w_image_normalized,
                                          h_image_normalized]
                 
+                bbox_image_normalized = round_float_array(bbox_image_normalized, 
+                                                          precision=COORD_DIGITS)
+                det['conf'] = round_float(det['conf'], precision=CONF_DIGITS)
+                
                 output_det = {}
                 output_det['bbox'] = bbox_image_normalized
                 output_det['conf'] = det['conf']
@@ -899,7 +944,14 @@ def main():
         type=str,
         default=None,
         help=('A .json list of relative filenames (or absolute paths contained within image_folder) to include'))
+    parser.add_argument(
+        '--detector_options',
+        type=str,
+        default=None,
+        help=('A list of detector options (key-value pairs) to '))
         
+    # detector_options = parse_kvp_list(args.detector_options)        
+
     if len(sys.argv[1:]) == 0:
         parser.print_help()
         parser.exit()

megadetector/detection/video_utils.py

@@ -714,8 +714,8 @@ def video_folder_to_frames(input_folder,
             parallelism
         every_n_frames (int, optional): sample every Nth frame starting from the first frame;
             if this is None or 1, every frame is extracted.  If this is a negative value, it's
-            interpreted as a sampling rate in seconds, which is rounded to the nearest frame sampling 
-            rate.  Mutually exclusive with frames_to_extract.
+            interpreted as a sampling rate in seconds, which is rounded to the nearest frame 
+            sampling rate.  Mutually exclusive with frames_to_extract.
         verbose (bool, optional): enable additional debug console output
         parallelization_uses_threads (bool, optional): whether to use threads (True) or
             processes (False) for parallelization; ignored if n_threads <= 1