megadetector 5.0.28__py3-none-any.whl → 10.0.0__py3-none-any.whl

megadetector/postprocessing/top_folders_to_bottom.py

@@ -44,55 +44,55 @@ class TopFoldersToBottomOptions:
     """
     Options used to parameterize top_folders_to_bottom()
     """
-    
+
     def __init__(self,input_folder,output_folder,copy=True,n_threads=1):
-        
+
         #: Whether to copy (True) vs. move (False) false when re-organizing
         self.copy = copy
-        
+
         #: Number of worker threads to use, or <1 to disable parallelization
         self.n_threads = n_threads
-        
+
         #: Input folder
         self.input_folder = input_folder
-        
+
         #: Output folder
         self.output_folder = output_folder
-        
+
         #: If this is False and an output file exists, throw an error
         self.overwrite = False
-        
+
 
 #%% Main functions
 
 def _process_file(relative_filename,options,execute=True):
-    
+
     assert ('/' in relative_filename) and \
         ('\\' not in relative_filename) and \
         (not path_is_abs(relative_filename))
-    
+
     # Find top-level folder
     tokens = relative_filename.split('/')
     topmost_folder = tokens.pop(0)
     tokens.insert(len(tokens)-1,topmost_folder)
-    
+
     # Find file/folder names
     output_relative_path = '/'.join(tokens)
     output_relative_folder = '/'.join(tokens[0:-1])
-    
+
     output_absolute_folder = os.path.join(options.output_folder,output_relative_folder)
     output_absolute_path = os.path.join(options.output_folder,output_relative_path)
 
     if execute:
-        
+
         os.makedirs(output_absolute_folder,exist_ok=True)
-        
+
         input_absolute_path = os.path.join(options.input_folder,relative_filename)
-        
+
         if not options.overwrite:
             assert not os.path.isfile(output_absolute_path), \
                 'Error: output file {} exists'.format(output_absolute_path)
-            
+
         # Move or copy
         if options.copy:
             shutil.copy(input_absolute_path, output_absolute_path)
@@ -100,7 +100,7 @@ def _process_file(relative_filename,options,execute=True):
             shutil.move(input_absolute_path, output_absolute_path)
 
     return output_absolute_path
-    
+
 # ...def _process_file()
 
 
@@ -131,93 +131,93 @@ def top_folders_to_bottom(options):
 
     """
     os.makedirs(options.output_folder,exist_ok=True)
-    
+
     # Enumerate input folder
     print('Enumerating files...')
     files = list(Path(options.input_folder).rglob('*'))
     files = [p for p in files if not p.is_dir()]
     files = [str(s) for s in files]
     print('Enumerated {} files'.format(len(files)))
-    
+
     # Convert absolute paths to relative paths
     relative_files = [os.path.relpath(s,options.input_folder) for s in files]
-    
+
     # Standardize delimiters
     relative_files = [s.replace('\\','/') for s in relative_files]
-    
+
     base_files = [s for s in relative_files if '/' not in s]
     if len(base_files) > 0:
         print('Warning: ignoring {} files in the base folder'.format(len(base_files)))
         relative_files = [s for s in relative_files if '/' in s]
-    
+
     # Make sure each input file maps to a unique output file
     absolute_output_files = [_process_file(s, options, execute=False) for s in relative_files]
     assert len(absolute_output_files) == len(set(absolute_output_files)),\
         "Error: input filenames don't map to unique output filenames"
-        
+
     # relative_filename = relative_files[0]
-    
+
     # Loop
     if options.n_threads <= 1:
-        
+
         for relative_filename in tqdm(relative_files):
             _process_file(relative_filename,options)
-    
+
     else:
-        
+
         print('Starting a pool with {} threads'.format(options.n_threads))
         pool = ThreadPool(options.n_threads)
         process_file_with_options = partial(_process_file, options=options)
         _ = list(tqdm(pool.imap(process_file_with_options, relative_files), total=len(relative_files)))
 
-# ...def top_folders_to_bottom()        
-        
+# ...def top_folders_to_bottom()
+
 
 #%% Interactive driver
-        
+
 if False:
 
     pass
 
     #%%
-    
+
     input_folder = r"G:\temp\output"
-    output_folder = r"G:\temp\output-inverted"    
+    output_folder = r"G:\temp\output-inverted"
     options = TopFoldersToBottomOptions(input_folder,output_folder,copy=True,n_threads=10)
-    
+
     #%%
-    
+
     top_folders_to_bottom(options)
 
 
-#%% Command-line driver   
+#%% Command-line driver
 
 # python top_folders_to_bottom.py "g:\temp\separated_images" "g:\temp\separated_images_inverted" --n_threads 100
 
-def main():
-    
+def main(): # noqa
+
     parser = argparse.ArgumentParser()
     parser.add_argument('input_folder', type=str, help='Input image folder')
     parser.add_argument('output_folder', type=str, help='Output image folder')
 
-    parser.add_argument('--copy', action='store_true', 
+    parser.add_argument('--copy', action='store_true',
                         help='Copy images, instead of moving (moving is the default)')
-    parser.add_argument('--overwrite', action='store_true', 
+    parser.add_argument('--overwrite', action='store_true',
                         help='Allow image overwrite (default=False)')
     parser.add_argument('--n_threads', type=int, default=1,
                         help='Number of threads to use for parallel operation (default=1)')
-    
+
     if len(sys.argv[1:])==0:
         parser.print_help()
         parser.exit()
-        
-    args = parser.parse_args()    
-    
+
+    args = parser.parse_args()
+
     # Convert to an options object
     options = TopFoldersToBottomOptions(
         args.input_folder,args.output_folder,copy=args.copy,n_threads=args.n_threads)
-    
+
     top_folders_to_bottom(options)
-    
-if __name__ == '__main__':    
+
+if __name__ == '__main__':
     main()

megadetector/postprocessing/validate_batch_results.py

@@ -41,27 +41,27 @@ class ValidateBatchResultsOptions:
     """
     Options controlling the behavior of validate_bach_results()
     """
-    
+
     def __init__(self):
-        
+
         #: Should we verify that images exist?  If this is True, and the .json
         #: file contains relative paths, relative_path_base needs to be specified.
         self.check_image_existence = False
-        
+
         #: If check_image_existence is True, where do the images live?
         #:
         #: If None, assumes absolute paths.
         self.relative_path_base = None
-        
+
         #: Should we return the loaded data, or just the validation results?
         self.return_data = False
-        
+
         #: Enable additional debug output
         self.verbose = False
-        
+
         #: Should we raise errors immediately (vs. just catching and reporting)?
         self.raise_errors = False
-    
+
 # ...class ValidateBatchResultsOptions
 
 
@@ -70,67 +70,67 @@ class ValidateBatchResultsOptions:
 def validate_batch_results(json_filename,options=None):
     """
     Verify that [json_filename] is a valid MD output file.  Currently errors on invalid files.
-    
-    Args: 
+
+    Args:
         json_filename (str): the filename to validate
-        options (ValidateBatchResultsOptions, optional): all the parameters used to control this 
+        options (ValidateBatchResultsOptions, optional): all the parameters used to control this
             process, see ValidateBatchResultsOptions for details
-            
+
     Returns:
         dict: a dict with a field called "validation_results", which is itself a dict.  The reason
-        it's a dict inside a dict is that if return_data is True, the outer dict also contains all 
+        it's a dict inside a dict is that if return_data is True, the outer dict also contains all
         the loaded data.  The "validation_results" dict contains fields called "errors", "warnings",
         and "filename".  "errors" and "warnings" are lists of strings, although "errors" will never
         be longer than N=1, since validation fails at the first error.
-            
+
     """
-    
+
     if options is None:
         options = ValidateBatchResultsOptions()
-    
+
     if options.verbose:
         print('Loading results from {}'.format(json_filename))
-        
+
     with open(json_filename,'r') as f:
         d = json.load(f)
-    
+
     validation_results = {}
     validation_results['filename'] = json_filename
     validation_results['warnings'] = []
     validation_results['errors'] = []
-    
+
     if not isinstance(d,dict):
-        
+
         validation_results['errors'].append('Input data is not a dict')
         to_return = {}
         to_return['validation_results'] = validation_results
         return to_return
-    
+
     try:
-        
+
         ## Info validation
-        
-        if not 'info' in d:
+
+        if 'info' not in d:
             raise ValueError('Input does not contain info field')
-            
+
         info = d['info']
-        
+
         if not isinstance(info,dict):
             raise ValueError('Input contains invalid info field')
-        
+
         if 'format_version' not in info :
             raise ValueError('Input does not specify format version')
-            
+
         format_version = float(info['format_version'])
         if format_version < 1.3:
             raise ValueError('This validator can only be used with format version 1.3 or later')
-        
-        
+
+
         ## Category validation
-        
+
         if 'detection_categories' not in d:
             raise ValueError('Input does not contain detection_categories field')
-            
+
         for k in d['detection_categories'].keys():
             # Category ID should be string-formatted ints
             if not isinstance(k,str):
@@ -139,7 +139,7 @@ def validate_batch_results(json_filename,options=None):
             if not isinstance(d['detection_categories'][k],str):
                 raise ValueError('Invalid detection category name: {}'.format(
                     d['detection_categories'][k]))
-            
+
         if 'classification_categories' in d:
             for k in d['classification_categories'].keys():
                 # Categories should be string-formatted ints
@@ -149,28 +149,28 @@ def validate_batch_results(json_filename,options=None):
                 if not isinstance(d['classification_categories'][k],str):
                     raise ValueError('Invalid classification category name: {}'.format(
                         d['classification_categories'][k]))
-        
-        
+
+
         ## Image validation
-        
+
         if 'images' not in d:
             raise ValueError('images field not present')
         if not isinstance(d['images'],list):
             raise ValueError('Invalid images field')
-        
+
         if options.verbose:
             print('Validating images')
-            
+
         # im = d['images'][0]
         for i_im,im in tqdm(enumerate(d['images']),total=len(d['images']),disable=(not options.verbose)):
-            
+
             if not isinstance(im,dict):
                 raise ValueError('Invalid image at index {}'.format(i_im))
             if 'file' not in im:
                 raise ValueError('Image without filename at index {}'.format(i_im))
-            
+
             file = im['file']
-            
+
             if 'detections' in im and im['detections'] is not None:
                 for det in im['detections']:
                     assert 'category' in det, 'Image {} has a detection with no category'.format(file)
@@ -180,17 +180,17 @@ def validate_batch_results(json_filename,options=None):
                     assert 'bbox' in det, 'Image {} has a detection with no box'.format(file)
                     assert det['category'] in d['detection_categories'], \
                         'Image {} has a detection with an unmapped category {}'.format(
-                            file,det['category'])                    
-                
+                            file,det['category'])
+
             if options.check_image_existence:
-                
+
                 if options.relative_path_base is None:
                     file_abs = file
                 else:
                     file_abs = os.path.join(options.relative_path_base,file)
                 if not os.path.isfile(file_abs):
                     raise ValueError('Cannot find file {}'.format(file_abs))
-            
+
             if 'failure' in im:
                 if im['failure'] is not None:
                     if not isinstance(im['failure'],str):
@@ -206,9 +206,9 @@ def validate_batch_results(json_filename,options=None):
             else:
                 if not isinstance(im['detections'],list):
                     raise ValueError('Invalid detections list for image {}'.format(im['file']))
-              
+
             if is_video_file(im['file']) and (format_version >= 1.4):
-                
+
                 if 'frame_rate' not in im:
                     raise ValueError('Video without frame rate: {}'.format(im['file']))
                 if im['frame_rate'] < 0:
@@ -220,34 +220,34 @@ def validate_batch_results(json_filename,options=None):
                             raise ValueError('Frame without frame number in video {}'.format(
                                 im['file']))
                     frame_numbers = [det['frame_number'] for det in im['detections']] # noqa
-                    # assert is_list_sorted(frame_numbers)                    
-                
+                    # assert is_list_sorted(frame_numbers)
+
         # ...for each image
-            
-        
+
+
         ## Validation of other keys
-        
-        for k in d.keys():                        
+
+        for k in d.keys():
             if (k not in typical_keys) and (k not in required_keys):
                 validation_results['warnings'].append(
-                    'Warning: non-standard key {} present at file level'.format(k))                
-        
+                    'Warning: non-standard key {} present at file level'.format(k))
+
     except Exception as e:
-                
+
         if options.raise_errors:
             raise
         else:
             validation_results['errors'].append(str(e))
-        
+
     # ...try/except
-    
+
     if options.return_data:
         to_return = d
     else:
         to_return = {}
-    
+
     to_return['validation_results'] = validation_results
-    
+
     return to_return
 
 # ...def validate_batch_results(...)
@@ -258,16 +258,16 @@ def validate_batch_results(json_filename,options=None):
 if False:
 
     #%% Validate all .json files in the MD test suite
-    
+
     from megadetector.utils.path_utils import recursive_file_list
     filenames = recursive_file_list(os.path.expanduser('~/AppData/Local/Temp/md-tests'))
     filenames = [fn for fn in filenames if fn.endswith('.json')]
     filenames = [fn for fn in filenames if 'detectionIndex' not in fn]
-    
+
     options = ValidateBatchResultsOptions()
     options.check_image_existence = False
     options.relative_path_base = None # r'g:\temp\test-videos'
-    
+
     for json_filename in filenames:
         results = validate_batch_results(json_filename,options)
         if len(results['validation_results']['warnings']) > 0:
@@ -275,13 +275,13 @@ if False:
             for s in results['validation_results']['warnings']:
                 print(s)
             print('')
-        assert len(results['validation_results']['errors']) == 0    
-    
+        assert len(results['validation_results']['errors']) == 0
+
 
 #%% Command-line driver
 
-def main():
-    
+def main(): # noqa
+
     options = ValidateBatchResultsOptions()
 
     parser = argparse.ArgumentParser()
@@ -299,11 +299,11 @@ def main():
         parser.exit()
 
     args = parser.parse_args()
-    
-    args_to_object(args, options)    
+
+    args_to_object(args, options)
 
     validate_batch_results(args.json_filename,options)
-    
-    
+
+
 if __name__ == '__main__':
     main()