megadetector 5.0.7__py3-none-any.whl → 5.0.8__py3-none-any.whl

api/batch_processing/data_preparation/manage_local_batch.py

@@ -86,6 +86,8 @@ from api.batch_processing.postprocessing.postprocess_batch_results import (
 from detection.run_detector import get_detector_version_from_filename
 from md_utils.ct_utils import image_file_to_camera_folder
 
+## Inference options
+
 # To specify a non-default confidence threshold for including detections in the .json file
 json_threshold = None
 
@@ -109,6 +111,11 @@ quiet_mode = True
 # will use its default size, which is 1280 * 1.3, which is almost always what you want.
 image_size = None
 
+# Should we include image size, timestamp, and/or EXIF data in MD output?
+include_image_size = False
+include_image_timestamp = False
+include_exif_data = False
+
 # Only relevant when running on CPU
 ncores = 1
 
@@ -187,7 +194,7 @@ augment = False
 
 ## Constants related to tiled inference
 
-use_tiled_inference = True
+use_tiled_inference = False
 
 # Should we delete tiles after each job?  Only set this to False for debugging;
 # large jobs will take up a lot of space if you keep tiles around after each task.
@@ -234,7 +241,7 @@ checkpoint_frequency = 10000
 approx_images_per_second = estimate_md_images_per_second(model_file) 
     
 # Rough estimate for the inference time cost of augmentation    
-if augment:
+if augment and (approx_images_per_second is not None):
     approx_images_per_second = approx_images_per_second * 0.7
     
 base_task_name = organization_name_short + '-' + job_date + job_description_string + '-' + \
@@ -268,6 +275,10 @@ filename_base = os.path.join(base_output_folder_name, base_task_name)
 combined_api_output_folder = os.path.join(filename_base, 'combined_api_outputs')
 postprocessing_output_folder = os.path.join(filename_base, 'preview')
 
+combined_api_output_file = os.path.join(
+    combined_api_output_folder,
+    '{}_detections.json'.format(base_task_name))
+
 os.makedirs(filename_base, exist_ok=True)
 os.makedirs(combined_api_output_folder, exist_ok=True)
 os.makedirs(postprocessing_output_folder, exist_ok=True)
@@ -494,7 +505,14 @@ for i_task,task in enumerate(task_info):
         
         overwrite_handling_string = '--overwrite_handling {}'.format(overwrite_handling)        
         cmd = f'{cuda_string} python run_detector_batch.py "{model_file}" "{chunk_file}" "{output_fn}" {checkpoint_frequency_string} {checkpoint_path_string} {use_image_queue_string} {ncores_string} {quiet_string} {image_size_string} {confidence_threshold_string} {overwrite_handling_string}'
-                
+        
+        if include_image_size:
+            cmd += ' --include_image_size'
+        if include_image_timestamp:
+            cmd += ' --include_image_timestamp'
+        if include_exif_data:
+            cmd += ' --include_exif_data'
+        
     cmd_file = os.path.join(filename_base,'run_chunk_{}_gpu_{}{}'.format(str(i_task).zfill(3),
                             str(gpu_number).zfill(2),script_extension))
     
@@ -747,10 +765,6 @@ for im in combined_results['images']:
     else:
         im['file'] = im['file'].replace(input_path + '/','',1)
     
-combined_api_output_file = os.path.join(
-    combined_api_output_folder,
-    '{}_detections.json'.format(base_task_name))
-
 with open(combined_api_output_file,'w') as f:
     json.dump(combined_results,f,indent=1)
 
@@ -793,7 +807,7 @@ options.api_output_file = combined_api_output_file
 options.output_dir = output_base
 ppresults = process_batch_results(options)
 html_output_file = ppresults.output_html_file
-path_utils.open_file(html_output_file,attempt_to_open_in_wsl_host=True)
+path_utils.open_file(html_output_file,attempt_to_open_in_wsl_host=True,browser_name='chrome')
 # import clipboard; clipboard.copy(html_output_file)
 
 
@@ -823,7 +837,7 @@ options.otherDetectionsThreshold = options.confidenceMin
 
 options.bRenderDetectionTiles = True
 options.maxOutputImageWidth = 2000
-options.detectionTilesMaxCrops = 300
+options.detectionTilesMaxCrops = 250
 
 # options.lineThickness = 5
 # options.boxExpansion = 8
@@ -930,7 +944,7 @@ options.output_dir = output_base
 ppresults = process_batch_results(options)
 html_output_file = ppresults.output_html_file
 
-path_utils.open_file(html_output_file,attempt_to_open_in_wsl_host=True)
+path_utils.open_file(html_output_file,attempt_to_open_in_wsl_host=True,browser_name='chrome')
 # import clipboard; clipboard.copy(html_output_file)
 
 
@@ -2003,7 +2017,7 @@ print('Processing {} to {}'.format(base_task_name, output_base))
 options.api_output_file = sequence_smoothed_classification_file
 options.output_dir = output_base
 ppresults = process_batch_results(options)
-path_utils.open_file(ppresults.output_html_file,attempt_to_open_in_wsl_host=True)
+path_utils.open_file(ppresults.output_html_file,attempt_to_open_in_wsl_host=True,browser_name='chrome')
 # import clipboard; clipboard.copy(ppresults.output_html_file)
 
 #% Zip .json files
@@ -2071,7 +2085,7 @@ for i, j in itertools.combinations(list(range(0,len(filenames))),2):
 results = compare_batch_results(options)
 
 from md_utils.path_utils import open_file
-open_file(results.html_output_file,attempt_to_open_in_wsl_host=True)
+open_file(results.html_output_file,attempt_to_open_in_wsl_host=True,browser_name='chrome')
 
 
 #%% Merge in high-confidence detections from another results file
@@ -2125,7 +2139,7 @@ options.output_dir = output_base_large_boxes
 
 ppresults = process_batch_results(options)
 html_output_file = ppresults.output_html_file
-path_utils.open_file(html_output_file,attempt_to_open_in_wsl_host=True)
+path_utils.open_file(html_output_file,attempt_to_open_in_wsl_host=True,browser_name='chrome')
 
 
 #%% .json splitting
@@ -2280,7 +2294,7 @@ import nbformat as nbf
 if os.name == 'nt':
     git_base = r'c:\git'
 else:
-    git_base = os.path.expanduer('~/git')
+    git_base = os.path.expanduser('~/git')
     
 input_py_file = git_base + '/MegaDetector/api/batch_processing/data_preparation/manage_local_batch.py'
 assert os.path.isfile(input_py_file)

api/batch_processing/postprocessing/combine_api_outputs.py

@@ -48,7 +48,7 @@ def combine_api_output_files(input_files: List[str],
         input_files: list of str, paths to JSON detection files
         output_file: optional str, path to write merged JSON
         require_uniqueness: bool, whether to require that the images in
-            each input_dict be unique
+            each list of images be unique
     """
     
     def print_if_verbose(s):
@@ -84,7 +84,7 @@ def combine_api_output_dictionaries(input_dicts: Iterable[Mapping[str, Any]],
         input_dicts: list of dicts, each dict is the JSON of the detections
             output file from the Batch Processing API
         require_uniqueness: bool, whether to require that the images in
-            each input_dict be unique
+            each input dict be unique
 
     Returns: dict, represents the merged JSON
     """

api/batch_processing/postprocessing/compare_batch_results.py

@@ -291,7 +291,7 @@ def pairwise_compare_batch_results(options,output_index,pairwise_options):
     filenames_b_set = set([im['file'] for im in images_b])
     
     if len(images_a) != len(images_b):
-        s = 'set A has {} iamges, set B has {}'.format(len(images_a),len(images_b))
+        s = 'set A has {} images, set B has {}'.format(len(images_a),len(images_b))
         if options.error_on_non_matching_lists:
             raise ValueError(s)
         else:

api/batch_processing/postprocessing/convert_output_format.py

@@ -4,8 +4,8 @@
 #
 # Converts between file formats output by our batch processing API.  Currently
 # supports json <--> csv conversion, but this should be the landing place for any
-# conversion - including between future .json versions - that we support in the 
-# future.
+# conversion - including between hypothetical alternative .json versions - that we support 
+# in the future.
 #
 ########
 
@@ -30,10 +30,13 @@ CONF_DIGITS = 3
 #%% Conversion functions
 
 def convert_json_to_csv(input_path,output_path=None,min_confidence=None,
-                        omit_bounding_boxes=False,output_encoding=None):
+                        omit_bounding_boxes=False,output_encoding=None,
+                        overwrite=True):
     """
     Convert .json to .csv
     
+    If output_path is None, will convert x.json to x.csv.
+    
     TODO: this function should obviously be using Pandas or some other sensible structured
     representation of tabular data.  Even a list of dicts.  This implementation is quite
     brittle and depends on adding fields to every row in exactly the right order.
@@ -42,6 +45,10 @@ def convert_json_to_csv(input_path,output_path=None,min_confidence=None,
     if output_path is None:
         output_path = os.path.splitext(input_path)[0]+'.csv'
         
+    if os.path.isfile(output_path) and (not overwrite):
+        print('File {} exists, skipping json --> csv conversion'.format(output_path))
+        return
+    
     print('Loading json results from {}...'.format(input_path))
     json_output = json.load(open(input_path))
 
@@ -73,7 +80,7 @@ def convert_json_to_csv(input_path,output_path=None,min_confidence=None,
 
         n_classification_categories = len(classification_category_ids)
     
-    # There are several fields for which we add columns, other random bespoke fields
+    # There are several .json fields for which we add .csv columns; other random bespoke fields
     # will be ignored.
     optional_fields = ['width','height','datetime','exif_metadata']
     optional_fields_present = set()
@@ -104,7 +111,7 @@ def convert_json_to_csv(input_path,output_path=None,min_confidence=None,
         if 'failure' in im and im['failure'] is not None:
             row = [image_id, 'failure', im['failure']]
             rows.append(row)
-            print('Skipping failed image {} ({})'.format(im['file'],im['failure']))
+            # print('Skipping failed image {} ({})'.format(im['file'],im['failure']))
             continue
 
         max_conf = ct_utils.get_max_conf(im)
@@ -193,12 +200,21 @@ def convert_json_to_csv(input_path,output_path=None,min_confidence=None,
         writer.writerow(header)
         writer.writerows(rows)
 
+# ...def convert_json_to_csv(...)
+
     
-def convert_csv_to_json(input_path,output_path=None):
+def convert_csv_to_json(input_path,output_path=None,overwrite=True):
+    """
+    Convert .csv to .json.  If output_path is None, will convert x.csv to x.json.
+    """
     
     if output_path is None:
         output_path = os.path.splitext(input_path)[0]+'.json'
         
+    if os.path.isfile(output_path) and (not overwrite):
+        print('File {} exists, skipping csv --> json conversion'.format(output_path))
+        return
+            
     # Format spec:
     #
     # https://github.com/agentmorris/MegaDetector/tree/master/api/batch_processing
@@ -259,6 +275,8 @@ def convert_csv_to_json(input_path,output_path=None):
     json_out['images'] = images
     
     json.dump(json_out,open(output_path,'w'),indent=1)
+    
+# ...def convert_csv_to_json(...)
 
 
 #%% Interactive driver

api/batch_processing/postprocessing/load_api_results.py

@@ -64,11 +64,9 @@ def load_api_results(api_output_path: str, normalize_paths: bool = True,
         if k != 'images':
             other_fields[k] = v
 
-    # Normalize paths to simplify comparisons later
     if normalize_paths:
         for image in detection_results['images']:
-            image['file'] = os.path.normpath(image['file'])
-            # image['file'] = image['file'].replace('\\','/')
+            image['file'] = os.path.normpath(image['file'])            
 
     if force_forward_slashes:
         for image in detection_results['images']:

api/batch_processing/postprocessing/md_to_labelme.py

@@ -20,6 +20,10 @@ import json
 
 from tqdm import tqdm
 
+from multiprocessing.pool import Pool
+from multiprocessing.pool import ThreadPool
+from functools import partial
+
 from md_visualization.visualization_utils import open_image
 from md_utils.ct_utils import truncate_float
 
@@ -29,15 +33,21 @@ default_confidence_threshold = 0.15
 
 #%% Functions
 
-def get_labelme_dict_for_image(im,image_base_name,category_id_to_name,info=None,confidence_threshold=None):
+def get_labelme_dict_for_image(im,image_base_name,category_id_to_name,
+                               info=None,confidence_threshold=None):
     """
     For the given image struct in MD results format, reformat the detections into
     labelme format.  Returns a dict.
+    
+    'height' and 'width' are required in [im].
+    
+    image_base_name is written directly to the 'imagePath' field in the output; it should generally be
+    os.path.basename(your_image_file).
     """
     
     if confidence_threshold is None:        
         confidence_threshold = -1.0
-        
+     
     output_dict = {}
     if info is not None:
         output_dict['detector_info'] = info
@@ -50,6 +60,7 @@ def get_labelme_dict_for_image(im,image_base_name,category_id_to_name,info=None,
     output_dict['imageData'] = None
     output_dict['detections'] = im['detections']
     
+    # det = im['detections'][1]
     for det in im['detections']:
         
         if det['conf'] < confidence_threshold:
@@ -79,69 +90,125 @@ def get_labelme_dict_for_image(im,image_base_name,category_id_to_name,info=None,
 # ...def get_labelme_dict_for_image()
 
 
+def _write_output_for_image(im,image_base,extension_prefix,info,
+                            confidence_threshold,category_id_to_name,overwrite,
+                            verbose=False):
+    
+    if 'failure' in im and im['failure'] is not None:
+        assert 'detections' not in im or im['detections'] is None
+        if verbose:
+            print('Skipping labelme file generation for failed image {}'.format(
+                im['file']))
+        return
+        
+    im_full_path = os.path.join(image_base,im['file'])
+    json_path = os.path.splitext(im_full_path)[0] + extension_prefix + '.json'
+    
+    if (not overwrite) and (os.path.isfile(json_path)):
+        if verbose:
+            print('Skipping existing file {}'.format(json_path))
+        return
+
+    output_dict = get_labelme_dict_for_image(im,
+                                             image_base_name=os.path.basename(im_full_path),
+                                             category_id_to_name=category_id_to_name,
+                                             info=info,
+                                             confidence_threshold=confidence_threshold)
+            
+    with open(json_path,'w') as f:
+        json.dump(output_dict,f,indent=1)
+    
+# ...def write_output_for_image(...)
+
+
+
 def md_to_labelme(results_file,image_base,confidence_threshold=None,
-                  overwrite=False):
+                  overwrite=False,extension_prefix='',n_workers=1,
+                  use_threads=False,bypass_image_size_read=False,
+                  verbose=False):
     """
     For all the images in [results_file], write a .json file in labelme format alongside the
     corresponding relative path within image_base.
+    
+    If non-empty, "extension_prefix" will be inserted before the .json extension.
     """
     
-    # Load MD results
-    with open(results_file,'r') as f:
-        md_results = json.load(f)
+    if extension_prefix is None:
+        extension_prefix = ''
         
-    # Read image sizes
-    #
-    # TODO: parallelize this loop
-    #
-    # im = md_results['images'][0]
-    for im in tqdm(md_results['images']):
+    # Load MD results if necessary
+    if isinstance(results_file,dict):
+        md_results = results_file
+    else:
+        print('Loading MD results...')
+        with open(results_file,'r') as f:
+            md_results = json.load(f)
         
-        # Make sure this file exists
-        im_full_path = os.path.join(image_base,im['file'])
-        assert os.path.isfile(im_full_path), 'Image file {} does not exist'.format(im_full_path)
+    # Read image sizes if necessary            
+    if bypass_image_size_read:     
         
-        # Load w/h information if necessary
-        if 'height' not in im or 'width' not in im:
-            
-            try:
-                pil_im = open_image(im_full_path)
-                im['width'] = pil_im.width
-                im['height'] = pil_im.height
-            except Exception:
-                print('Warning: cannot open image {}, treating as a failure during inference'.format(
-                    im_full_path))
-                if 'failure' not in im:
-                    im['failure'] = 'Failure image access'        
-
-        # ...if we need to read w/h information
+        print('Bypassing image size read')
         
-    # ...for each image
+    else:
     
-    # Write output
-    for im in tqdm(md_results['images']):
-        
-        if 'failure' in im and im['failure'] is not None:
-            assert 'detections' not in im
-            print('Warning: skipping labelme file generation for failed image {}'.format(
-                im['file']))
-            continue
+        # TODO: parallelize this loop
+    
+        print('Reading image sizes...')
+                
+        # im = md_results['images'][0]
+        for im in tqdm(md_results['images']):
+            
+            # Make sure this file exists
+            im_full_path = os.path.join(image_base,im['file'])
+            assert os.path.isfile(im_full_path), 'Image file {} does not exist'.format(im_full_path)
+            
+            json_path = os.path.splitext(im_full_path)[0] + extension_prefix + '.json'
+            
+            # Don't even bother reading sizes for files we're not going to generate
+            if (not overwrite) and (os.path.isfile(json_path)):
+                continue
             
-        im_full_path = os.path.join(image_base,im['file'])
-        json_path = os.path.splitext(im_full_path)[0] + '.json'
+            # Load w/h information if necessary
+            if 'height' not in im or 'width' not in im:
+                
+                try:
+                    pil_im = open_image(im_full_path)
+                    im['width'] = pil_im.width
+                    im['height'] = pil_im.height
+                except Exception:
+                    print('Warning: cannot open image {}, treating as a failure during inference'.format(
+                        im_full_path))
+                    if 'failure' not in im:
+                        im['failure'] = 'Failure image access'        
+    
+            # ...if we need to read w/h information
+            
+        # ...for each image
         
-        if (not overwrite) and (os.path.isfile(json_path)):
-            print('Skipping existing file {}'.format(json_path))
-            continue
+    # ...if we're not bypassing image size read        
     
-        output_dict = get_labelme_dict_for_image(im,
-                                                 image_base_name=os.path.basename(im_full_path),
-                                                 category_id_to_name=md_results['detection_categories'],
-                                                 info=md_results['info'],
-                                                 confidence_threshold=confidence_threshold)
-                
-        with open(json_path,'w') as f:
-            json.dump(output_dict,f,indent=1)
+    print('\nGenerating labelme files...')
+        
+    # Write output
+    if n_workers <= 1:
+        for im in tqdm(md_results['images']):    
+            _write_output_for_image(im,image_base,extension_prefix,md_results['info'],confidence_threshold,
+                                   md_results['detection_categories'],overwrite,verbose)
+    else:
+        if use_threads:
+            print('Starting parallel thread pool with {} workers'.format(n_workers))
+            pool = ThreadPool(n_workers)
+        else:
+            print('Starting parallel process pool with {} workers'.format(n_workers))
+            pool = Pool(n_workers)
+        _ = list(tqdm(pool.imap(
+                partial(_write_output_for_image,
+                        image_base=image_base,extension_prefix=extension_prefix,
+                        info=md_results['info'],confidence_threshold=confidence_threshold,
+                        category_id_to_name=md_results['detection_categories'],
+                        overwrite=overwrite,verbose=verbose),
+                 md_results['images']),
+                 total=len(md_results['images'])))
             
     # ...for each image
     

api/batch_processing/postprocessing/merge_detections.py

@@ -3,9 +3,12 @@
 # merge_detections.py
 #
 # Merge high-confidence detections from one or more results files into another 
-# file.   Typically used to combine results from MDv5b and/or MDv4 into a "primary"
+# file.  Typically used to combine results from MDv5b and/or MDv4 into a "primary"
 # results file from MDv5a.
 #
+# Detection categories must be the same in both files; if you want to first remap
+# one file's category mapping to be the same as another's, see remap_detection_categories.
+#
 # If you want to literally merge two .json files, see combine_api_outputs.py.
 #
 ########
@@ -30,7 +33,7 @@ class MergeDetectionsOptions:
         
         self.max_detection_size = 1.01
         self.min_detection_size = 0
-        self.source_confidence_thresholds = [0.2]
+        self.source_confidence_thresholds = [0.05]
         
         # Don't bother merging into target images if there is a similar detection
         # above this threshold (or if there is *any* detection above this threshold,
@@ -38,7 +41,7 @@ class MergeDetectionsOptions:
         self.target_confidence_threshold = 0.2
         
         # If you want to merge only certain categories, specify one
-        # (but not both) of these.
+        # (but not both) of these.  These are category IDs, not names.
         self.categories_to_include = None
         self.categories_to_exclude = None
 
@@ -47,11 +50,28 @@ class MergeDetectionsOptions:
         self.merge_empty_only = False
         
         self.iou_threshold = 0.65
+        
+        self.overwrite = False
 
 
 #%% Main function
 
 def merge_detections(source_files,target_file,output_file,options=None):
+    """
+    Merge high-confidence detections from one or more results files into another 
+    file.   Typically used to combine results from MDv5b and/or MDv4 into a "primary"
+    results file from MDv5a.
+    
+    [source_files] (a list of files or a single filename) specifies the set of 
+    results files that will be merged into [target_file].  The difference between a 
+    "source file" and the "target file" is that if no merging is necessary, either because
+    two boxes are nearly identical or because merge_only_empty is True and the target
+    file already has above-threshold detection for an image+category, the output file gets
+    the results of the "target" file.  I.e., the "target" file wins all ties.
+    
+    The results are written to [output_file].
+
+    """
     
     if isinstance(source_files,str):
         source_files = [source_files]    
@@ -59,6 +79,10 @@ def merge_detections(source_files,target_file,output_file,options=None):
     if options is None:
         options = MergeDetectionsOptions()    
         
+    if (not options.overwrite) and (os.path.isfile(output_file)):
+        print('File {} exists, bypassing merge'.format(output_file))
+        return
+    
     assert not ((options.categories_to_exclude is not None) and \
                 (options.categories_to_include is not None)), \
                 'categories_to_include and categories_to_exclude are mutually exclusive'
@@ -133,7 +157,8 @@ def merge_detections(source_files,target_file,output_file,options=None):
         output_data['info']['detections_transferred_from'].append(os.path.basename(source_file))
         output_data['info']['detector'] = output_data['info']['detector'] + ' + ' + source_detector_name
         
-        assert source_data['detection_categories'] == output_data['detection_categories']
+        assert source_data['detection_categories'] == output_data['detection_categories'], \
+            'Cannot merge files with different detection category maps'
         
         source_confidence_threshold = options.source_confidence_thresholds[i_source_file]
         
@@ -246,7 +271,7 @@ def merge_detections(source_files,target_file,output_file,options=None):
     # ...for each source file        
     
     with open(output_file,'w') as f:
-        json.dump(output_data,f,indent=2)
+        json.dump(output_data,f,indent=1)
     
     print('Saved merged results to {}'.format(output_file))