megadetector 5.0.19__py3-none-any.whl → 5.0.21__py3-none-any.whl

megadetector/detection/run_inference_with_yolov5_val.py

@@ -788,38 +788,35 @@ def run_inference_with_yolo_val(options):
     yolo_read_failures = []
     
     for line in yolo_console_output:
-        # Lines look like:
+        
+        #
+        # Lines indicating read failures look like:
         #
         # For ultralytics val:
         #
-        # val: WARNING ⚠️ /a/b/c/d.jpg: ignoring corrupt image/label: [Errno 13] Permission denied: '/a/b/c/d.jpg'
         # line = "val: WARNING ⚠️ /a/b/c/d.jpg: ignoring corrupt image/label: [Errno 13] Permission denied: '/a/b/c/d.jpg'"
         #
         # For yolov5 val.py:
         #
-        # test: WARNING: a/b/c/d.jpg: ignoring corrupt image/label: cannot identify image file '/a/b/c/d.jpg'
         # line = "test: WARNING: a/b/c/d.jpg: ignoring corrupt image/label: cannot identify image file '/a/b/c/d.jpg'"
-        if 'cannot identify image file' in line:
-            tokens = line.split('cannot identify image file')
-            image_name = tokens[-1].strip()
-            assert image_name[0] == "'" and image_name [-1] == "'"
-            image_name = image_name[1:-1]
-            yolo_read_failures.append(image_name)            
-        elif 'ignoring corrupt image/label' in line:
-            assert 'WARNING' in line
-            if '⚠️' in line:
-                assert line.startswith('val'), \
-                    'Unrecognized line in YOLO output: {}'.format(line)
-                tokens = line.split('ignoring corrupt image/label')
-                image_name = tokens[0].split('⚠️')[-1].strip()
-            else:
-                assert line.startswith('test'), \
-                    'Unrecognized line in YOLO output: {}'.format(line)
-                tokens = line.split('ignoring corrupt image/label')
-                image_name = tokens[0].split('WARNING:')[-1].strip()
-            assert image_name.endswith(':')
-            image_name = image_name[0:-1]
+        #
+        # In both cases, when we are using symlinks, the first filename is the symlink name, the 
+        # second filename is the target, e.g.:
+        # 
+        # line = "test: WARNING: /tmp/md_to_yolo/md_to_yolo_xyz/symlinks/xyz/0000000004.jpg: ignoring corrupt image/label: cannot identify image file '/tmp/md-tests/md-test-images/corrupt-images/real-file.jpg'"
+        #
+        line = line.replace('⚠️',':')
+        if 'ignoring corrupt image/label' in line:
+                        
+            tokens = line.split('ignoring corrupt image/label')
+            assert len(tokens) == 2
+            tokens = tokens[0].split(':',maxsplit=3)
+            assert len(tokens) == 4
+            assert 'warning' in tokens[1].lower()
+            image_name = tokens[2].strip()
             yolo_read_failures.append(image_name)
+            
+    # ...for each line in the console output
                     
     # image_file = yolo_read_failures[0]
     for image_file in yolo_read_failures:

megadetector/detection/video_utils.py

@@ -228,9 +228,10 @@ def run_callback_on_frames(input_video_file,
         input_video_file (str): video file to process
         frame_callback (function): callback to run on frames, should take an np.array and a string and 
             return a single value.  callback should expect PIL-formatted (RGB) images.
-        every_n_frames (int, optional): sample every Nth frame starting from the first frame;
-            if this is None or 1, every frame is processed. Mutually exclusive with
-            frames_to_process.
+        every_n_frames (float, optional): sample every Nth frame starting from the first frame;
+            if this is None or 1, every frame is processed.  If this is a negative value, that's 
+            interpreted as a sampling rate in seconds, which is rounded to the nearest frame sampling 
+            rate. Mutually exclusive with frames_to_process.
         verbose (bool, optional): enable additional debug console output
         frames_to_process (list of int, optional): process this specific set of frames;
             mutually exclusive with every_n_frames.  If all values are beyond the length
@@ -263,6 +264,13 @@ def run_callback_on_frames(input_video_file,
     frame_filenames = []
     results = []
     
+    if every_n_frames is not None and every_n_frames < 0:
+        every_n_seconds = abs(every_n_frames)
+        every_n_frames = int(every_n_seconds * frame_rate)
+        if verbose:
+            print('Interpreting a time sampling rate of {} hz as a frame interval of {}'.format(
+                every_n_seconds,every_n_frames))        
+        
     # frame_number = 0
     for frame_number in range(0,n_frames):
 
@@ -670,12 +678,18 @@ def _video_to_frames_for_folder(relative_fn,input_folder,output_folder_base,
     return frame_filenames,fs
 
 
-def video_folder_to_frames(input_folder, output_folder_base, 
-                           recursive=True, overwrite=True,
-                           n_threads=1, every_n_frames=None,
-                           verbose=False, parallelization_uses_threads=True,
-                           quality=None, max_width=None, 
-                           frames_to_extract=None, allow_empty_videos=False):
+def video_folder_to_frames(input_folder,
+                           output_folder_base, 
+                           recursive=True, 
+                           overwrite=True,
+                           n_threads=1,
+                           every_n_frames=None,
+                           verbose=False,
+                           parallelization_uses_threads=True,
+                           quality=None,
+                           max_width=None, 
+                           frames_to_extract=None,
+                           allow_empty_videos=False):
     """
     For every video file in input_folder, creates a folder within output_folder_base, and 
     renders frame of that video to images in that folder.
@@ -701,6 +715,8 @@ def video_folder_to_frames(input_folder, output_folder_base,
             each video; mutually exclusive with every_n_frames.  If all values are beyond 
             the length of a video, no frames are extracted. Can also be a single int, 
             specifying a single frame number.
+        allow_empty_videos (bool, optional): Just print a warning if a video appears to have no
+            frames (by default, this is an error).
         
     Returns:
         tuple: a length-3 tuple containing:
@@ -711,8 +727,11 @@ def video_folder_to_frames(input_folder, output_folder_base,
     """
     
     # Recursively enumerate video files
+    if verbose:
+        print('Enumerating videos in {}'.format(input_folder))
     input_files_full_paths = find_videos(input_folder,recursive=recursive)
-    print('Found {} videos in folder {}'.format(len(input_files_full_paths),input_folder))
+    if verbose:
+        print('Found {} videos in folder {}'.format(len(input_files_full_paths),input_folder))
     if len(input_files_full_paths) == 0:
         return [],[],[]
     
@@ -776,6 +795,10 @@ class FrameToVideoOptions:
         #: for the whole video, i.e. "1" means "use the confidence value from the highest-confidence frame"
         self.nth_highest_confidence = 1
         
+        #: Should we include just a single representative frame result for each video (default), or
+        #: every frame that was processed?
+        self.include_all_processed_frames = False
+        
         #: What to do if a file referred to in a .json results file appears not to be a 
         #: video; can be 'error' or 'skip_with_warning'
         self.non_video_behavior = 'error'
@@ -803,7 +826,7 @@ def frame_results_to_video_results(input_file,
 
     if options is None:
         options = FrameToVideoOptions()
-        
+    
     # Load results
     with open(input_file,'r') as f:
         input_data = json.load(f)
@@ -856,47 +879,58 @@ def frame_results_to_video_results(input_file,
     # video_name = list(video_to_frame_info.keys())[0]
     for video_name in tqdm(video_to_frame_info):
         
-        frames = video_to_frame_info[video_name]
-        
-        all_detections_this_video = []
-        
-        # frame = frames[0]
-        for frame in frames:
-            if ('detections' in frame) and (frame['detections'] is not None):
-                all_detections_this_video.extend(frame['detections'])
-            
-        # At most one detection for each category for the whole video
-        canonical_detections = []
-            
-        # category_id = list(detection_categories.keys())[0]
-        for category_id in detection_categories:
-            
-            category_detections = [det for det in all_detections_this_video if \
-                                   det['category'] == category_id]
-            
-            # Find the nth-highest-confidence video to choose a confidence value
-            if len(category_detections) >= options.nth_highest_confidence:
-                
-                category_detections_by_confidence = sorted(category_detections, 
-                                                           key = lambda i: i['conf'],reverse=True)
-                canonical_detection = category_detections_by_confidence[options.nth_highest_confidence-1]
-                canonical_detections.append(canonical_detection)
-                                      
         # Prepare the output representation for this video
         im_out = {}
         im_out['file'] = video_name
-        im_out['detections'] = canonical_detections
         
         if (video_filename_to_frame_rate is not None) and \
             (video_name in video_filename_to_frame_rate):
             im_out['frame_rate'] = video_filename_to_frame_rate[video_name]
         
-        # 'max_detection_conf' is no longer included in output files by default
-        if False:
-            im_out['max_detection_conf'] = 0
-            if len(canonical_detections) > 0:
-                confidences = [d['conf'] for d in canonical_detections]
-                im_out['max_detection_conf'] = max(confidences)
+        # Find all detections for this video
+        all_detections_this_video = []
+        
+        frames = video_to_frame_info[video_name]
+                
+        # frame = frames[0]
+        for frame in frames:
+            if ('detections' in frame) and (frame['detections'] is not None):                
+                all_detections_this_video.extend(frame['detections'])
+        
+        # Should we keep detections for all frames?
+        if (options.include_all_processed_frames):
+        
+            im_out['detections'] = all_detections_this_video
+        
+        # ...or should we keep just a canonical detection for each category?
+        else:
+            
+            canonical_detections = []
+                
+            # category_id = list(detection_categories.keys())[0]
+            for category_id in detection_categories:
+                
+                category_detections = [det for det in all_detections_this_video if \
+                                       det['category'] == category_id]
+                
+                # Find the nth-highest-confidence video to choose a confidence value
+                if len(category_detections) >= options.nth_highest_confidence:
+                    
+                    category_detections_by_confidence = sorted(category_detections, 
+                                                               key = lambda i: i['conf'],reverse=True)
+                    canonical_detection = category_detections_by_confidence[options.nth_highest_confidence-1]
+                    canonical_detections.append(canonical_detection)
+                                          
+            im_out['detections'] = canonical_detections
+            
+            # 'max_detection_conf' is no longer included in output files by default
+            if False:
+                im_out['max_detection_conf'] = 0
+                if len(canonical_detections) > 0:
+                    confidences = [d['conf'] for d in canonical_detections]
+                    im_out['max_detection_conf'] = max(confidences)
+
+        # ...if we're keeping output for all frames / canonical frames
         
         output_images.append(im_out)
         
@@ -951,6 +985,7 @@ if False:
     results_file = r'results.json'
     confidence_threshold = 0.75
         
+    
     #%% Load detector output    
     
     with open(results_file,'r') as f:

megadetector/postprocessing/combine_api_outputs.py

@@ -192,7 +192,7 @@ def combine_api_shard_files(input_files, output_file=None):
     
     Args:
         input_files (list of str): files to merge
-        output_file (str, optiona): file to which we should write merged results
+        output_file (str, optional): file to which we should write merged results
         
     Returns:
         dict: merged results

megadetector/postprocessing/detector_calibration.py

@@ -0,0 +1,367 @@
+"""
+
+detector_calibration.py
+
+Tools for comparing/calibrating confidence values from detectors, particularly different
+versions of MegaDetector.
+
+"""
+
+#%% Constants and imports
+
+import random
+
+from tqdm import tqdm
+from enum import IntEnum
+from collections import defaultdict
+
+import numpy as np
+import matplotlib
+import matplotlib.pyplot as plt
+
+from megadetector.postprocessing.validate_batch_results import \
+    validate_batch_results, ValidateBatchResultsOptions
+from megadetector.utils.ct_utils import get_iou
+
+
+#%% Classes
+
+class CalibrationOptions:
+    """
+    Options controlling comparison/calibration behavior.
+    """
+    
+    def __init__(self):
+        
+        #: IoU threshold used for determining whether two detections are the same
+        #:
+        #: When multiple detections match, we will only use the highest-matching IoU.
+        self.iou_threshold = 0.75
+        
+        #: Minimum confidence threshold to consider for calibration (should be lower than
+        #: the lowest value you would use in realistic situations)
+        self.confidence_threshold = 0.025
+    
+        #: Should we populate the data_a and data_b fields in the return value?
+        self.return_data = False
+        
+        #: Model name to use in printouts and plots for result set A
+        self.model_name_a = 'model_a'
+        
+        #: Model name to use in printouts and plots for result set B
+        self.model_name_b = 'model_b'
+        
+        #: Maximum number of samples to use for plotting or calibration per category,
+        #: or None to use all paired values.
+        self.max_samples_per_category = None
+        
+        #: List of category IDs to use for plotting comparisons, or None to plot
+        #: all categories.
+        self.categories_to_plot = None
+        
+        #: Optionally map category ID to name in plot labels
+        self.category_id_to_name = None
+        
+# ...class CalibrationOptions
+
+class ConfidenceMatchColumns(IntEnum):
+    
+    COLUMN_CONF_A = 0
+    COLUMN_CONF_B = 1
+    COLUMN_CONF_IOU = 2
+    COLUMN_CONF_I_IMAGE = 3
+    COLUMN_CONF_CATEGORY_ID = 4
+    
+class CalibrationResults:
+    """
+    Results of a model-to-model comparison.
+    """
+        
+    def __init__(self):
+        
+        #: List of tuples: [conf_a, conf_b, iou, i_image, category_id]
+        self.confidence_matches = []
+        
+        #: Populated with the data loaded from json_filename_a if options.return_data is True
+        self.data_a = None
+        
+        #: Populated with the data loaded from json_filename_b if options.return_data is True
+        self.data_b = None
+
+# ...class CalibrationResults        
+
+
+#%% Calibration functions
+
+def compare_model_confidence_values(json_filename_a,json_filename_b,options=None):
+    """
+    Compare confidence values across two .json results files.  Compares only detections that
+    can be matched by IoU, i.e., does not do anything with detections that only appear in one file.
+    
+    Args: 
+        json_filename_a (str or dict): filename containing results from the first model to be compared; 
+            should refer to the same images as [json_filename_b].  Can also be a loaded results dict.
+        json_filename_b (str or dict): filename containing results from the second model to be compared; 
+            should refer to the same images as [json_filename_a].  Can also be a loaded results dict.
+        options (CalibrationOptions, optional): all the parameters used to control this process, see 
+            CalibrationOptions for details
+            
+    Returns:
+        CalibrationResults: description of the comparison results
+    """
+    
+    ## Option handling 
+    
+    if options is None:
+        options = CalibrationOptions()
+    
+    validation_options = ValidateBatchResultsOptions()
+    validation_options.return_data = True
+    
+    if isinstance(json_filename_a,str):
+        results_a = validate_batch_results(json_filename_a,options=validation_options)
+        assert len(results_a['validation_results']['errors']) == 0
+    else:
+        assert isinstance(json_filename_a,dict)
+        results_a = json_filename_a        
+        
+    if isinstance(json_filename_b,str):
+        results_b = validate_batch_results(json_filename_b,options=validation_options)
+        assert len(results_b['validation_results']['errors']) == 0
+    else:
+        assert isinstance(json_filename_b,dict)
+        results_b = json_filename_b
+        
+        
+    ## Make sure these results sets are comparable
+    
+    image_filenames_a = [im['file'] for im in results_a['images']]
+    image_filenames_b = [im['file'] for im in results_b['images']]
+    
+    assert set(image_filenames_a) == set(image_filenames_b), \
+        'Cannot calibrate non-matching image sets'
+    
+    categories_a = results_a['detection_categories']
+    categories_b = results_b['detection_categories']
+    assert set(categories_a.keys()) == set(categories_b.keys())
+    for k in categories_a.keys():
+        assert categories_a[k] == categories_b[k], 'Category mismatch'
+    
+    
+    ## Compare detections
+    
+    image_filename_b_to_im = {}
+    for im in results_b['images']:
+        image_filename_b_to_im[im['file']] = im
+    
+    n_detections_a = 0
+    n_detections_a_queried = 0
+    n_detections_a_matched = 0
+    
+    confidence_matches = []
+    
+    # For each image
+    # im_a = results_a['images'][0]
+    for i_image,im_a in tqdm(enumerate(results_a['images']),total=len(results_a['images'])):
+            
+        fn = im_a['file']
+        im_b = image_filename_b_to_im[fn]
+        
+        if 'detections' not in im_a or im_a['detections'] is None:
+            continue
+        if 'detections' not in im_b or im_b['detections'] is None:
+            continue
+    
+        # For each detection in result set A...
+        #
+        # det_a = im_a['detections'][0]
+        for det_a in im_a['detections']:
+            
+            n_detections_a += 1
+            
+            conf_a = det_a['conf']
+            category_id = det_a['category']
+            
+            # Is this above threshold?
+            if conf_a < options.confidence_threshold:
+                continue
+            
+            n_detections_a_queried += 1
+            
+            bbox_a = det_a['bbox']
+            
+            best_iou = None
+            best_iou_conf = None
+                        
+            # For each detection in result set B...
+            #
+            # det_b = im_b['detections'][0]
+            for det_b in im_b['detections']:
+            
+                # Is this the same category?
+                if det_b['category'] != category_id:
+                    continue
+                
+                conf_b = det_b['conf']
+                
+                # Is this above threshold?
+                if conf_b < options.confidence_threshold:
+                    continue
+                
+                bbox_b = det_b['bbox']
+                
+                iou = get_iou(bbox_a,bbox_b)
+                
+                # Is this an adequate IoU to consider?
+                if iou < options.iou_threshold:
+                    continue
+                
+                # Is this the best match so far?
+                if best_iou is None or iou > best_iou:
+                    best_iou = iou
+                    best_iou_conf = conf_b
+    
+            # ...for each detection in im_b
+        
+            if best_iou is not None:
+                n_detections_a_matched += 1
+                conf_result = [conf_a,best_iou_conf,best_iou,i_image,category_id]
+                confidence_matches.append(conf_result)
+        
+        # ...for each detection in im_a
+        
+    # ...for each image in result set A
+    
+    print('\nOf {} detections in result set A, queried {}, matched {}'.format(
+        n_detections_a,n_detections_a_queried,n_detections_a_matched))
+    assert len(confidence_matches) == n_detections_a_matched
+
+    calibration_results = CalibrationResults()        
+    calibration_results.confidence_matches = confidence_matches
+
+    if options.return_data:
+        calibration_results.data_a = results_a
+        calibration_results.data_b = results_b
+        
+    return calibration_results
+
+# ...def compare_model_confidence_values(...)
+
+
+#%% Plotting functions
+
+def plot_matched_confidence_values(calibration_results,output_filename,options=None):
+    """
+    Given a set of paired confidence values for matching detections (from 
+    compare_model_confidence_values), plot histograms of those pairs for each
+    detection category.
+    
+    Args: 
+        calibration_results (CalibrationResults): output from a call to 
+            compare_model_confidence_values, containing paired confidence
+            values for two sets of detection results.
+        output_filename (str): filename to write the plot (.png or .jpg)
+        options (CalibrationOptions, optional): plotting options, see
+            CalibrationOptions for details.
+    """
+    
+    fig_w = 12
+    fig_h = 8
+    n_hist_bins = 80
+    
+    if options is None:
+        options = CalibrationOptions()
+    
+    # Find matched confidence pairs for each category ID
+    category_to_matches = defaultdict(list)
+    
+    confidence_matches = calibration_results.confidence_matches
+    for m in confidence_matches:
+        category_id = m[ConfidenceMatchColumns.COLUMN_CONF_CATEGORY_ID]
+        category_to_matches[category_id].append(m)
+    
+    # Optionally sample matches
+    category_to_samples = defaultdict(list)
+           
+    for i_category,category_id in enumerate(category_to_matches.keys()):
+        
+        matches_this_category = category_to_matches[category_id]
+        
+        if (options.max_samples_per_category is None) or \
+            (len(matches_this_category) <= options.max_samples_per_category):
+            category_to_samples[category_id] = matches_this_category
+        else:
+            assert len(matches_this_category) > options.max_samples_per_category
+            category_to_samples[category_id] = random.sample(matches_this_category,options.max_samples_per_category)
+    
+    del category_to_matches
+    del confidence_matches
+    
+    categories_to_plot = list(category_to_samples.keys())
+    
+    if options.categories_to_plot is not None:
+        categories_to_plot = [category_id for category_id in categories_to_plot if\
+                              category_id in options.categories_to_plot]    
+    
+    n_subplots = len(categories_to_plot)
+    
+    plt.ioff()
+
+    fig = matplotlib.figure.Figure(figsize=(fig_w, fig_h), tight_layout=True)    
+    # fig,axes = plt.subplots(nrows=n_subplots,ncols=1)    
+    
+    axes = fig.subplots(n_subplots, 1)
+
+    # i_category = 0; category_id = categories_to_plot[i_category]
+    for i_category,category_id in enumerate(categories_to_plot):
+    
+        ax = axes[i_category]
+        
+        category_string = category_id
+        if options.category_id_to_name is not None and \
+            category_id in options.category_id_to_name:
+            category_string = options.category_id_to_name[category_id]
+            
+        samples_this_category = category_to_samples[category_id]
+        x = [m[0] for m in samples_this_category]
+        y = [m[1] for m in samples_this_category]
+        
+        weights_a = np.ones_like(x)/float(len(x))
+        weights_b = np.ones_like(y)/float(len(y))
+        ax.hist(x,histtype='step',bins=n_hist_bins,density=False,color='red',weights=weights_a)
+        ax.hist(y,histtype='step',bins=n_hist_bins,density=False,color='blue',weights=weights_b)
+        ax.legend([options.model_name_a,options.model_name_b])
+        ax.set_ylabel(category_string)
+        # plt.tight_layout()
+        
+        # I experimented with heat maps, but they weren't very informative.
+        # Leaving this code here in case I revisit.  Note to self: scatter plots
+        # were a disaster.
+        if False:        
+            heatmap, xedges, yedges = np.histogram2d(x, y, bins=30)
+            extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]]
+            plt.imshow(heatmap.T, extent=extent, origin='lower', norm='log')
+    
+    # ...for each category for which we need to generate a histogram
+    
+    plt.close(fig)
+    fig.savefig(output_filename,dpi=100)
+    
+# ...def plot_matched_confidence_values(...)
+
+
+#%% Interactive driver(s)
+
+if False:
+
+    #%%
+    
+    options = ValidateBatchResultsOptions()
+    # json_filename = r'g:\temp\format.json'
+    # json_filename = r'g:\temp\test-videos\video_results.json'
+    json_filename = r'g:\temp\test-videos\image_results.json'
+    options.check_image_existence = True
+    options.relative_path_base = r'g:\temp\test-videos'
+    validate_batch_results(json_filename,options)
+    
+

megadetector/postprocessing/md_to_coco.py

@@ -41,7 +41,8 @@ def md_to_coco(md_results_file,
     The default confidence threshold is not 0; the assumption is that by default, you are 
     going to treat the resulting COCO file as a set of labels.  If you are using the resulting COCO
     file to evaluate a detector, you likely want a default confidence threshold of 0.  Confidence
-    values will be written to the semi-standard "score" field for each image
+    values will be written to the semi-standard "score" field for each image if 
+    preserve_nonstandard_metadata is True.
     
     A folder of images is required if width and height information are not available 
     in the MD results file.