megadetector 5.0.6__py3-none-any.whl → 5.0.8__py3-none-any.whl

data_management/databases/integrity_check_json_db.py

@@ -24,9 +24,9 @@ import sys
 
 from multiprocessing.pool import ThreadPool
 from operator import itemgetter
-from PIL import Image
 from tqdm import tqdm
 
+from md_visualization.visualization_utils import open_image
 from md_utils import ct_utils
 
 
@@ -41,6 +41,8 @@ class IntegrityCheckOptions:
     bRequireLocation = True
     iMaxNumImages = -1
     nThreads = 10
+    verbose = True
+    
     
 # This is used in a medium-hacky way to share modified options across threads
 defaultOptions = IntegrityCheckOptions()
@@ -65,7 +67,9 @@ def check_image_existence_and_size(image,options=None):
             print('Missing image size in {}'.format(filePath))
             return False
 
-        width, height = Image.open(filePath).size
+        # width, height = Image.open(filePath).size
+        pil_im = open_image(filePath)
+        width,height = pil_im.size
         if (not (width == image['width'] and height == image['height'])):
             print('Size mismatch for image {}: {} (reported {},{}, actual {},{})'.format(
                     image['id'], filePath, image['width'], image['height'], width, height))
@@ -86,9 +90,13 @@ def integrity_check_json_db(jsonFile, options=None):
     
     if options.bCheckImageSizes:        
         options.bCheckImageExistence = True
-        
-    print(options.__dict__)
+     
+    if options.verbose:
+        print(options.__dict__)
     
+    if options.baseDir is None:
+        options.baseDir = ''
+        
     baseDir = options.baseDir
     
     
@@ -102,8 +110,9 @@ def integrity_check_json_db(jsonFile, options=None):
         
         assert os.path.isfile(jsonFile), '.json file {} does not exist'.format(jsonFile)
     
-        print('Reading .json {} with base dir [{}]...'.format(
-                jsonFile,baseDir))
+        if options.verbose:
+            print('Reading .json {} with base dir [{}]...'.format(
+                    jsonFile,baseDir))
         
         with open(jsonFile,'r') as f:
             data = json.load(f) 
@@ -116,10 +125,9 @@ def integrity_check_json_db(jsonFile, options=None):
     annotations = data['annotations']
     categories = data['categories']
     # info = data['info']
-    assert 'info' in data
+    assert 'info' in data, 'No info struct in database'
     
-    if len(baseDir) > 0:
-        
+    if len(baseDir) > 0:        
         assert os.path.isdir(baseDir), 'Base directory {} does not exist'.format(baseDir)
         
         
@@ -131,7 +139,8 @@ def integrity_check_json_db(jsonFile, options=None):
     catNameToCat = {}
     imageLocationSet = set()
     
-    print('Checking categories...')
+    if options.verbose:
+        print('Checking categories...')
     
     for cat in tqdm(categories):
         
@@ -139,8 +148,8 @@ def integrity_check_json_db(jsonFile, options=None):
         assert 'name' in cat
         assert 'id' in cat
         
-        assert isinstance(cat['id'],int), 'Illegal category ID type'
-        assert isinstance(cat['name'],str), 'Illegal category name type'
+        assert isinstance(cat['id'],int), 'Illegal category ID type: [{}]'.format(str(cat['id']))
+        assert isinstance(cat['name'],str), 'Illegal category name type [{}]'.format(str(cat['name']))
         
         catId = cat['id']
         catName = cat['name']
@@ -155,11 +164,13 @@ def integrity_check_json_db(jsonFile, options=None):
         
     # ...for each category
         
-    print('\nChecking images...')
+    if options.verbose:
+        print('\nChecking images...')
     
     if options.iMaxNumImages > 0 and len(images) > options.iMaxNumImages:
         
-        print('Trimming image list to {}'.format(options.iMaxNumImages))
+        if options.verbose:
+            print('Trimming image list to {}'.format(options.iMaxNumImages))
         images = images[0:options.iMaxNumImages]
         
     imagePathsInJson = set()
@@ -215,7 +226,8 @@ def integrity_check_json_db(jsonFile, options=None):
     # Are we checking for unused images?
     if (len(baseDir) > 0) and options.bFindUnusedImages:    
         
-        print('\nEnumerating images...')
+        if options.verbose:
+            print('\nEnumerating images...')
         
         # Recursively enumerate images
         imagePaths = []
@@ -242,8 +254,9 @@ def integrity_check_json_db(jsonFile, options=None):
         
         if len(baseDir) == 0:
             print('Warning: checking image sizes without a base directory, assuming "."')
-            
-        print('Checking image existence and/or image sizes...')
+         
+        if options.verbose:
+            print('Checking image existence and/or image sizes...')
         
         if options.nThreads is not None and options.nThreads > 1:
             pool = ThreadPool(options.nThreads)
@@ -263,9 +276,9 @@ def integrity_check_json_db(jsonFile, options=None):
                             
     # ...for each image
     
-    print('{} validation errors (of {})'.format(len(validationErrors),len(images)))
-                                                
-    print('Checking annotations...')
+    if options.verbose:
+        print('{} validation errors (of {})'.format(len(validationErrors),len(images)))
+        print('Checking annotations...')
     
     nBoxes = 0
     
@@ -300,58 +313,56 @@ def integrity_check_json_db(jsonFile, options=None):
         catIdToCat[ann['category_id']]['_count'] +=1 
         
     # ...for each annotation
-        
-        
-    ##%% Print statistics
     
-    # Find un-annotated images and multi-annotation images
-    nUnannotated = 0
-    nMultiAnnotated = 0
+    sortedCategories = sorted(categories, key=itemgetter('_count'), reverse=True)
     
-    for image in images:
-        if image['_count'] == 0:
-            nUnannotated += 1
-        elif image['_count'] > 1:
-            nMultiAnnotated += 1
-            
-    print('Found {} unannotated images, {} images with multiple annotations'.format(
-            nUnannotated,nMultiAnnotated))
     
-    if (len(baseDir) > 0) and options.bFindUnusedImages:
-        print('Found {} unused image files'.format(len(unusedFiles)))
-        
-    nUnusedCategories = 0
+    ##%% Print statistics
     
-    # Find unused categories
-    for cat in categories:
-        if cat['_count'] == 0:
-            print('Unused category: {}'.format(cat['name']))
-            nUnusedCategories += 1
+    if options.verbose:
     
-    print('Found {} unused categories'.format(nUnusedCategories))
+        # Find un-annotated images and multi-annotation images
+        nUnannotated = 0
+        nMultiAnnotated = 0
+        
+        for image in images:
+            if image['_count'] == 0:
+                nUnannotated += 1
+            elif image['_count'] > 1:
+                nMultiAnnotated += 1
+                
+        print('Found {} unannotated images, {} images with multiple annotations'.format(
+                nUnannotated,nMultiAnnotated))
+        
+        if (len(baseDir) > 0) and options.bFindUnusedImages:
+            print('Found {} unused image files'.format(len(unusedFiles)))
             
-    sequenceString = 'no sequence info'
-    if len(sequences) > 0:
-        sequenceString = '{} sequences'.format(len(sequences))
+        nUnusedCategories = 0
         
-    print('\nDB contains {} images, {} annotations, {} bboxes, {} categories, {}\n'.format(
-            len(images),len(annotations),nBoxes,len(categories),sequenceString))
-
-    if len(imageLocationSet) > 0:
-        print('DB contains images from {} locations\n'.format(len(imageLocationSet)))
-    
-    # Prints a list of categories sorted by count
-    #
-    # https://stackoverflow.com/questions/72899/how-do-i-sort-a-list-of-dictionaries-by-a-value-of-the-dictionary
-    
-    sortedCategories = sorted(categories, key=itemgetter('_count'), reverse=True)
-    
-    print('Categories and annotation (not image) counts:\n')
-    
-    for cat in sortedCategories:
-        print('{:6} {}'.format(cat['_count'],cat['name']))
+        # Find unused categories
+        for cat in categories:
+            if cat['_count'] == 0:
+                print('Unused category: {}'.format(cat['name']))
+                nUnusedCategories += 1
+        
+        print('Found {} unused categories'.format(nUnusedCategories))
+                
+        sequenceString = 'no sequence info'
+        if len(sequences) > 0:
+            sequenceString = '{} sequences'.format(len(sequences))
+            
+        print('\nDB contains {} images, {} annotations, {} bboxes, {} categories, {}\n'.format(
+                len(images),len(annotations),nBoxes,len(categories),sequenceString))
     
-    print('')
+        if len(imageLocationSet) > 0:
+            print('DB contains images from {} locations\n'.format(len(imageLocationSet)))
+                
+        print('Categories and annotation (not image) counts:\n')
+        
+        for cat in sortedCategories:
+            print('{:6} {}'.format(cat['_count'],cat['name']))
+        
+        print('')
     
     errorInfo = {}
     errorInfo['unusedFiles'] = unusedFiles

data_management/databases/subset_json_db.py

@@ -65,7 +65,7 @@ def subset_json_db(input_json, query, output_json=None, ignore_case=False):
     # Write the output file if requested
     if output_json is not None:
         print('Writing output .json...')
-        json.dump(output_data,open(output_json,'w'),indent=4)
+        json.dump(output_data,open(output_json,'w'),indent=1)
         
     return output_data
 

data_management/generate_crops_from_cct.py

@@ -164,4 +164,4 @@ if False:
     options.box_expansion = 25
     
     htmlOutputFile,db = visualize_db(db_path,output_dir,image_base_dir,options)
-    
+    

data_management/get_image_sizes.py

@@ -15,6 +15,8 @@ import os
 from PIL import Image
 import sys
 
+from md_utils.path_utils import find_images
+
 from multiprocessing.pool import ThreadPool
 from multiprocessing.pool import Pool
 from functools import partial
@@ -27,7 +29,11 @@ use_threads = False
 
 #%% Processing functions
 
-def process_image(image_path,image_prefix=None):
+def _get_image_size(image_path,image_prefix=None):
+    """
+    Support function to get the size of a single image.  Returns a (path,w,h) tuple.
+    w and h will be -1 if the image fails to load.
+    """
     
     if image_prefix is not None:
         full_path = os.path.join(image_prefix,image_path)
@@ -49,43 +55,56 @@ def process_image(image_path,image_prefix=None):
         return (image_path,-1,-1)
     
     
-def process_images(filenames,image_prefix=None,n_threads=default_n_threads):
+def get_image_sizes(filenames,image_prefix=None,output_file=None,
+                    n_workers=default_n_threads,use_threads=True,
+                    recursive=True):
+    """
+    Get the width and height of all images in [filenames], which can be:
+        
+    * A .json-formatted file 
+    * A folder
+    * A list of files
+
+    ...returning a list of (path,w,h) tuples, and optionally writing the results to [output_file].
+    """        
+    
+    if output_file is not None:
+        assert os.path.isdir(os.path.dirname(output_file)), \
+            'Illegal output file {}, parent folder does not exist'.format(output_file)
+        
+    if isinstance(filenames,str) and os.path.isfile(filenames):
+        with open(filenames,'r') as f:        
+            filenames = json.load(f)
+        filenames = [s.strip() for s in filenames]
+    elif isinstance(filenames,str) and os.path.isdir(filenames):
+        filenames = find_images(filenames,recursive=recursive,
+                                return_relative_paths=False,convert_slashes=True)
+    else:
+        assert isinstance(filenames,list)        
     
-    if n_threads <= 1:
+    if n_workers <= 1:
         
         all_results = []
         for i_file,fn in tqdm(enumerate(filenames),total=len(filenames)):
-            all_results.append(process_image(fn,image_prefix=image_prefix))
+            all_results.append(_get_image_size(fn,image_prefix=image_prefix))
     
     else:
         
-        print('Creating a pool with {} threads'.format(n_threads))
+        print('Creating a pool with {} workers'.format(n_workers))
         if use_threads:
-            pool = ThreadPool(n_threads)        
+            pool = ThreadPool(n_workers)        
         else:
-            pool = Pool(n_threads)
+            pool = Pool(n_workers)
         # all_results = list(tqdm(pool.imap(process_image, filenames), total=len(filenames)))
         all_results = list(tqdm(pool.imap(
-            partial(process_image,image_prefix=image_prefix), filenames), total=len(filenames)))
-                
-    return all_results
-
-
-def process_list_file(input_file,output_file=None,image_prefix=None,n_threads=default_n_threads):
-    
-    assert os.path.isdir(os.path.dirname(output_file))
-    assert os.path.isfile(input_file)
-    
-    with open(input_file,'r') as f:        
-        filenames = json.load(f)
-    filenames = [s.strip() for s in filenames]
-    
-    all_results = process_images(filenames,image_prefix=image_prefix,n_threads=n_threads)
+            partial(_get_image_size,image_prefix=image_prefix), filenames), total=len(filenames)))
     
     if output_file is not None:
         with open(output_file,'w') as f:
             json.dump(all_results,f,indent=1)
-    
+            
+    return all_results
+
     
 #%% Interactive driver
 
@@ -116,8 +135,7 @@ if False:
     
     #%%
     
-    # process_list_file(image_list_file,image_size_file,image_prefix=base_dir)
-    process_list_file(relative_image_list_file,image_size_file,image_prefix=base_dir,n_threads=4)
+    get_image_sizes(relative_image_list_file,image_size_file,image_prefix=base_dir,n_threads=4)
     
     
 #%% Command-line driver
@@ -136,7 +154,7 @@ def main():
         
     args = parser.parse_args()
     
-    process_list_file(args.input_file,args.output_file,args.image_prefix,args.n_threads)
+    _ = get_image_sizes(args.input_file,args.output_file,args.image_prefix,args.n_threads)
     
 
 if __name__ == '__main__':

data_management/importers/animl_results_to_md_results.py

@@ -39,8 +39,6 @@ def animl_results_to_md_results(input_file,output_file=None):
     If [output_file] is None, '.json' will be appended to the input file.
     """
     
-    #%%
-    
     if output_file is None:
         output_file = input_file + '.json'
 
@@ -111,7 +109,7 @@ def animl_results_to_md_results(input_file,output_file=None):
     with open(output_file,'w') as f:
         json.dump(results,f,indent=1)
         
-# ...zamba_results_to_md_results(...)
+# ...animl_results_to_md_results(...)
         
 
 #%% Interactive driver
@@ -145,7 +143,7 @@ def main():
         '--output_file',
         type=str,
         default=None,
-        help='output .json file (defaults to input file appened with ".json")')
+        help='output .json file (defaults to input file appended with ".json")')
     
     if len(sys.argv[1:]) == 0:
         parser.print_help()
@@ -157,4 +155,4 @@ def main():
     
 if __name__ == '__main__':
     main()
-    
+    

data_management/importers/noaa_seals_2019.py

@@ -2,7 +2,7 @@
 #
 # noaa_seals_2019.py
 #
-# Prepare the NOAA Artic Seals 2019 metadata for LILA.
+# Prepare the NOAA Arctic Seals 2019 metadata for LILA.
 #
 ########
 
@@ -165,7 +165,7 @@ for b in ir_boxes:
                                                    use_normalized_coordinates=False,
                                                    thickness=3)
 
-visualization_utils.show_images_in_a_row([img_rgb,img_ir])
+# visualization_utils.show_images_in_a_row([img_rgb,img_ir])
 
 
 #%% Save images

data_management/importers/zamba_results_to_md_results.py

@@ -166,7 +166,7 @@ def main():
         '--output_file',
         type=str,
         default=None,
-        help='output .json file (defaults to input file appened with ".json")')
+        help='output .json file (defaults to input file appended with ".json")')
     
     if len(sys.argv[1:]) == 0:
         parser.print_help()
@@ -178,4 +178,4 @@ def main():
     
 if __name__ == '__main__':
     main()
-    
+