megadetector 5.0.7__py3-none-any.whl → 5.0.9__py3-none-any.whl

data_management/databases/integrity_check_json_db.py

@@ -1,19 +1,19 @@
-########
-#
-# integrity_check_json_db.py
-#
-# Does some integrity-checking and computes basic statistics on a db, specifically:
-#
-# * Verifies that required fields are present and have the right types
-# * Verifies that annotations refer to valid images
-# * Verifies that annotations refer to valid categories
-# * Verifies that image, category, and annotation IDs are unique 
-# * Optionally checks file existence
-# * Finds un-annotated images
-# * Finds unused categories
-# * Prints a list of categories sorted by count
-#
-########
+"""
+
+integrity_check_json_db.py
+
+Does some integrity-checking and computes basic statistics on a COCO Camera Traps .json file, specifically:
+
+* Verifies that required fields are present and have the right types
+* Verifies that annotations refer to valid images
+* Verifies that annotations refer to valid categories
+* Verifies that image, category, and annotation IDs are unique 
+* Optionally checks file existence
+* Finds un-annotated images
+* Finds unused categories
+* Prints a list of categories sorted by count
+
+"""
 
 #%% Constants and environment
 
@@ -24,31 +24,65 @@ import sys
 
 from multiprocessing.pool import ThreadPool
 from operator import itemgetter
-from PIL import Image
 from tqdm import tqdm
 
+from md_visualization.visualization_utils import open_image
 from md_utils import ct_utils
 
 
 #%% Classes and environment
 
 class IntegrityCheckOptions:
+    """
+    Options for integrity_check_json_db()
+    """
     
+    #: Image path; the filenames in the .json file should be relative to this folder
     baseDir = ''
+    
+    #: Should we validate the image sizes?
     bCheckImageSizes = False
+    
+    #: Should we check that all the images in the .json file exist on disk?
     bCheckImageExistence = False
+    
+    #: Should we search [baseDir] for images that are not used in the .json file?
     bFindUnusedImages = False
+    
+    #: Should we require that all images in the .json file have a 'location' field?
     bRequireLocation = True
+    
+    #: For debugging, limit the number of images we'll process
     iMaxNumImages = -1
+    
+    #: Number of threads to use for parallelization, set to <= 1 to disable parallelization
     nThreads = 10
     
+    #: Enable additional debug output
+    verbose = True
+    
+    
 # This is used in a medium-hacky way to share modified options across threads
 defaultOptions = IntegrityCheckOptions()
 
 
 #%% Functions
 
-def check_image_existence_and_size(image,options=None):
+def _check_image_existence_and_size(image,options=None):
+    """
+    Validate the image represented in the CCT image dict [image], which should have fields:
+    
+    * file_name
+    * width
+    * height
+
+    Args:
+        image (dict): image to validate
+        options (IntegrityCheckOptions): parameters impacting validation
+    
+    Returns:
+        bool: whether this image passes validation
+    """
 
     if options is None:        
         options = defaultOptions
@@ -65,7 +99,9 @@ def check_image_existence_and_size(image,options=None):
             print('Missing image size in {}'.format(filePath))
             return False
 
-        width, height = Image.open(filePath).size
+        # width, height = Image.open(filePath).size
+        pil_im = open_image(filePath)
+        width,height = pil_im.size
         if (not (width == image['width'] and height == image['height'])):
             print('Size mismatch for image {}: {} (reported {},{}, actual {},{})'.format(
                     image['id'], filePath, image['width'], image['height'], width, height))
@@ -76,9 +112,17 @@ def check_image_existence_and_size(image,options=None):
   
 def integrity_check_json_db(jsonFile, options=None):
     """
-    jsonFile can be a filename or an already-loaded json database
+    Does some integrity-checking and computes basic statistics on a COCO Camera Traps .json file; see
+    module header comment for a list of the validation steps.
     
-    return sortedCategories, data, errorInfo
+    Args:
+        jsonFile (str): filename to validate, or an already-loaded dict
+    
+    Returns:
+        tuple: tuple containing:
+            - sortedCategories (dict): list of categories used in [jsonFile], sorted by frequency
+            - data (dict): the data loaded from [jsonFile]
+            - errorInfo (dict): specific validation errors
     """
     
     if options is None:       
@@ -86,8 +130,9 @@ def integrity_check_json_db(jsonFile, options=None):
     
     if options.bCheckImageSizes:        
         options.bCheckImageExistence = True
-        
-    print(options.__dict__)
+     
+    if options.verbose:
+        print(options.__dict__)
     
     if options.baseDir is None:
         options.baseDir = ''
@@ -105,8 +150,9 @@ def integrity_check_json_db(jsonFile, options=None):
         
         assert os.path.isfile(jsonFile), '.json file {} does not exist'.format(jsonFile)
     
-        print('Reading .json {} with base dir [{}]...'.format(
-                jsonFile,baseDir))
+        if options.verbose:
+            print('Reading .json {} with base dir [{}]...'.format(
+                    jsonFile,baseDir))
         
         with open(jsonFile,'r') as f:
             data = json.load(f) 
@@ -133,7 +179,8 @@ def integrity_check_json_db(jsonFile, options=None):
     catNameToCat = {}
     imageLocationSet = set()
     
-    print('Checking categories...')
+    if options.verbose:
+        print('Checking categories...')
     
     for cat in tqdm(categories):
         
@@ -157,11 +204,13 @@ def integrity_check_json_db(jsonFile, options=None):
         
     # ...for each category
         
-    print('\nChecking images...')
+    if options.verbose:
+        print('\nChecking images...')
     
     if options.iMaxNumImages > 0 and len(images) > options.iMaxNumImages:
         
-        print('Trimming image list to {}'.format(options.iMaxNumImages))
+        if options.verbose:
+            print('Trimming image list to {}'.format(options.iMaxNumImages))
         images = images[0:options.iMaxNumImages]
         
     imagePathsInJson = set()
@@ -217,7 +266,8 @@ def integrity_check_json_db(jsonFile, options=None):
     # Are we checking for unused images?
     if (len(baseDir) > 0) and options.bFindUnusedImages:    
         
-        print('\nEnumerating images...')
+        if options.verbose:
+            print('\nEnumerating images...')
         
         # Recursively enumerate images
         imagePaths = []
@@ -244,8 +294,9 @@ def integrity_check_json_db(jsonFile, options=None):
         
         if len(baseDir) == 0:
             print('Warning: checking image sizes without a base directory, assuming "."')
-            
-        print('Checking image existence and/or image sizes...')
+         
+        if options.verbose:
+            print('Checking image existence and/or image sizes...')
         
         if options.nThreads is not None and options.nThreads > 1:
             pool = ThreadPool(options.nThreads)
@@ -253,11 +304,11 @@ def integrity_check_json_db(jsonFile, options=None):
             defaultOptions.baseDir = options.baseDir
             defaultOptions.bCheckImageSizes = options.bCheckImageSizes
             defaultOptions.bCheckImageExistence = options.bCheckImageExistence
-            results = tqdm(pool.imap(check_image_existence_and_size, images), total=len(images))
+            results = tqdm(pool.imap(_check_image_existence_and_size, images), total=len(images))
         else:
             results = []
             for im in tqdm(images):
-                results.append(check_image_existence_and_size(im,options))
+                results.append(_check_image_existence_and_size(im,options))
                 
         for iImage,r in enumerate(results):
             if not r:            
@@ -265,9 +316,9 @@ def integrity_check_json_db(jsonFile, options=None):
                             
     # ...for each image
     
-    print('{} validation errors (of {})'.format(len(validationErrors),len(images)))
-                                                
-    print('Checking annotations...')
+    if options.verbose:
+        print('{} validation errors (of {})'.format(len(validationErrors),len(images)))
+        print('Checking annotations...')
     
     nBoxes = 0
     
@@ -302,58 +353,56 @@ def integrity_check_json_db(jsonFile, options=None):
         catIdToCat[ann['category_id']]['_count'] +=1 
         
     # ...for each annotation
-        
-        
-    ##%% Print statistics
     
-    # Find un-annotated images and multi-annotation images
-    nUnannotated = 0
-    nMultiAnnotated = 0
+    sortedCategories = sorted(categories, key=itemgetter('_count'), reverse=True)
     
-    for image in images:
-        if image['_count'] == 0:
-            nUnannotated += 1
-        elif image['_count'] > 1:
-            nMultiAnnotated += 1
-            
-    print('Found {} unannotated images, {} images with multiple annotations'.format(
-            nUnannotated,nMultiAnnotated))
     
-    if (len(baseDir) > 0) and options.bFindUnusedImages:
-        print('Found {} unused image files'.format(len(unusedFiles)))
-        
-    nUnusedCategories = 0
+    ##%% Print statistics
     
-    # Find unused categories
-    for cat in categories:
-        if cat['_count'] == 0:
-            print('Unused category: {}'.format(cat['name']))
-            nUnusedCategories += 1
+    if options.verbose:
     
-    print('Found {} unused categories'.format(nUnusedCategories))
+        # Find un-annotated images and multi-annotation images
+        nUnannotated = 0
+        nMultiAnnotated = 0
+        
+        for image in images:
+            if image['_count'] == 0:
+                nUnannotated += 1
+            elif image['_count'] > 1:
+                nMultiAnnotated += 1
+                
+        print('Found {} unannotated images, {} images with multiple annotations'.format(
+                nUnannotated,nMultiAnnotated))
+        
+        if (len(baseDir) > 0) and options.bFindUnusedImages:
+            print('Found {} unused image files'.format(len(unusedFiles)))
             
-    sequenceString = 'no sequence info'
-    if len(sequences) > 0:
-        sequenceString = '{} sequences'.format(len(sequences))
+        nUnusedCategories = 0
         
-    print('\nDB contains {} images, {} annotations, {} bboxes, {} categories, {}\n'.format(
-            len(images),len(annotations),nBoxes,len(categories),sequenceString))
-
-    if len(imageLocationSet) > 0:
-        print('DB contains images from {} locations\n'.format(len(imageLocationSet)))
-    
-    # Prints a list of categories sorted by count
-    #
-    # https://stackoverflow.com/questions/72899/how-do-i-sort-a-list-of-dictionaries-by-a-value-of-the-dictionary
-    
-    sortedCategories = sorted(categories, key=itemgetter('_count'), reverse=True)
-    
-    print('Categories and annotation (not image) counts:\n')
-    
-    for cat in sortedCategories:
-        print('{:6} {}'.format(cat['_count'],cat['name']))
+        # Find unused categories
+        for cat in categories:
+            if cat['_count'] == 0:
+                print('Unused category: {}'.format(cat['name']))
+                nUnusedCategories += 1
+        
+        print('Found {} unused categories'.format(nUnusedCategories))
+                
+        sequenceString = 'no sequence info'
+        if len(sequences) > 0:
+            sequenceString = '{} sequences'.format(len(sequences))
+            
+        print('\nDB contains {} images, {} annotations, {} bboxes, {} categories, {}\n'.format(
+                len(images),len(annotations),nBoxes,len(categories),sequenceString))
     
-    print('')
+        if len(imageLocationSet) > 0:
+            print('DB contains images from {} locations\n'.format(len(imageLocationSet)))
+                
+        print('Categories and annotation (not image) counts:\n')
+        
+        for cat in sortedCategories:
+            print('{:6} {}'.format(cat['_count'],cat['name']))
+        
+        print('')
     
     errorInfo = {}
     errorInfo['unusedFiles'] = unusedFiles
@@ -398,9 +447,7 @@ def main():
     ct_utils.args_to_object(args, options)
     integrity_check_json_db(args.jsonFile,options)
 
-
-if __name__ == '__main__':
-    
+if __name__ == '__main__':    
     main()
 
 

data_management/databases/subset_json_db.py

@@ -1,14 +1,14 @@
-########
-#
-# subset_json_db.py
-#
-# Select a subset of images (and associated annotations) from a .json file
-# in COCO Camera Traps format.
-#
-# To subset the .json files in the MegaDetector output format, see 
-# subset_json_detector_output.py
-#
-########
+"""
+
+subset_json_db.py
+
+Select a subset of images (and associated annotations) from a .json file in COCO 
+Camera Traps format based on a string query.
+
+To subset .json files in the MegaDetector output format, see
+subset_json_detector_output.py.
+
+"""
     
 #%% Constants and imports
 
@@ -26,6 +26,16 @@ def subset_json_db(input_json, query, output_json=None, ignore_case=False):
     Given a json file (or dictionary already loaded from a json file), produce a new 
     database containing only the images whose filenames contain the string 'query', 
     optionally writing that DB output to a new json file.
+    
+    Args:
+        input_json (str): COCO Camera Traps .json file to load, or an already-loaded dict
+        query (str): string to query for, only include images in the output whose filenames 
+            contain this string.
+        output_json (str, optional): file to write the resulting .json file to
+        ignore_case (bool, optional): whether to perform a case-insensitive search for [query]
+        
+    Returns:
+        dict: possibly-modified CCT dictionary
     """
     
     if ignore_case:
@@ -65,7 +75,7 @@ def subset_json_db(input_json, query, output_json=None, ignore_case=False):
     # Write the output file if requested
     if output_json is not None:
         print('Writing output .json...')
-        json.dump(output_data,open(output_json,'w'),indent=4)
+        json.dump(output_data,open(output_json,'w'),indent=1)
         
     return output_data
 
@@ -76,8 +86,8 @@ if False:
     
     #%%
     
-    input_json = r"E:\Statewide_wolf_container\idfg_20190409.json"
-    output_json = r"E:\Statewide_wolf_container\idfg_20190409_clearcreek.json"
+    input_json = r"e:\Statewide_wolf_container\idfg_20190409.json"
+    output_json = r"e:\Statewide_wolf_container\idfg_20190409_clearcreek.json"
     query = 'clearcreek'
     ignore_case = True
     db = subset_json_db(input_json, query, output_json, ignore_case)
@@ -101,6 +111,5 @@ def main():
     
     subset_json_db(args.input_json,args.query,args.output_json,args.ignore_case)
 
-if __name__ == '__main__':
-    
+if __name__ == '__main__':    
     main()

data_management/generate_crops_from_cct.py

@@ -1,11 +1,11 @@
-########
-#
-# generate_crops_from_cct.py
-#
-# Given a .json file in COCO Camera Traps format, create a cropped image for
-# each bounding box.
-#
-########
+"""
+
+generate_crops_from_cct.py
+
+Given a .json file in COCO Camera Traps format, creates a cropped image for
+each bounding box.
+
+"""
 
 #%% Imports and constants
 
@@ -19,6 +19,23 @@ from PIL import Image
 #%% Functions
 
 def generate_crops_from_cct(cct_file,image_dir,output_dir,padding=0,flat_output=True):
+    """
+    Given a .json file in COCO Camera Traps format, creates a cropped image for
+    each bounding box.
+    
+    Args:
+        cct_file (str): the COCO .json file from which we should load data
+        image_dir (str): the folder where the images live; filenames in the .json
+            file should be relative to this folder
+        output_dir (str): the folder where we should write cropped images
+        padding (float, optional): number of pixels we should expand each box before
+            cropping
+        flat_output (bool, optional): if False, folder structure will be preserved
+            in the output, e.g. the image a/b/c/d.jpg will result in image files 
+            in the output folder called, e.g., a/b/c/d_crop_000_id_12345.jpg.  If
+            [flat_output] is True, the corresponding output image will be 
+            a_b_c_d_crop_000_id_12345.jpg.            
+    """
     
     ## Read and validate input
     
@@ -123,45 +140,10 @@ if False:
     flat_output = True
     output_dir = '/home/user/tmp/noaa-fish-crops'
     
-    #%%
-    
     generate_crops_from_cct(cct_file,image_dir,output_dir,padding,flat_output=True)
     files = os.listdir(output_dir)
     
-    #%%
-    
-    import random
-    fn = os.path.join(output_dir,random.choice(files))
-    
-    from md_utils.path_utils import open_file
-    open_file(fn)
-
-
-#%% Scrap
-
-if False:
     
-    pass
+#%% Command-line driver
 
-    #%%
-    
-    from md_visualization.visualize_db import DbVizOptions,visualize_db
-    
-    db_path = cct_file
-    output_dir = os.path.expanduser('~/tmp/noaa-fish-preview')
-    image_base_dir = image_dir
-    
-    options = DbVizOptions()
-    options.num_to_visualize = None
-    
-    options.parallelize_rendering_n_cores = 5
-    options.parallelize_rendering = True    
-    
-    options.viz_size = (-1, -1)
-    options.trim_to_images_with_bboxes = True
-    
-    options.box_thickness = 4
-    options.box_expansion = 25
-    
-    htmlOutputFile,db = visualize_db(db_path,output_dir,image_base_dir,options)
-    
+# TODO