megadetector 5.0.8__py3-none-any.whl → 5.0.10__py3-none-any.whl

data_management/coco_to_yolo.py

@@ -1,20 +1,20 @@
-########
-#
-# coco_to_yolo.py
-#
-# Converts a COCO-formatted dataset to a YOLO-formatted dataset, flattening
-# the dataset (to a single folder) in the process.
-#
-# If the input and output folders are the same, writes .txt files to the input folder,
-# and neither moves nor modifies images.
-#
-# Currently ignores segmentation masks, and errors if an annotation has a 
-# segmentation polygon but no bbox.
-# 
-# Has only been tested on a handful of COCO Camera Traps data sets; if you
-# use it for more general COCO conversion, YMMV.
-#
-########
+"""
+
+coco_to_yolo.py
+
+Converts a COCO-formatted dataset to a YOLO-formatted dataset, flattening
+the dataset (to a single folder) in the process.
+
+If the input and output folders are the same, writes .txt files to the input folder,
+and neither moves nor modifies images.
+
+Currently ignores segmentation masks, and errors if an annotation has a 
+segmentation polygon but no bbox.
+
+Has only been tested on a handful of COCO Camera Traps data sets; if you
+use it for more general COCO conversion, YMMV.
+
+"""
 
 #%% Imports and constants
 
@@ -37,16 +37,16 @@ def write_yolo_dataset_file(yolo_dataset_file,
                             val_folder_relative=None,
                             test_folder_relative=None):
     """
-    Write a YOLOv5 dataset.yaml file to the absolute path yolo_dataset_file (should
+    Write a YOLOv5 dataset.yaml file to the absolute path [yolo_dataset_file] (should
     have a .yaml extension, though it's only a warning if it doesn't).  
-
-    [dataset_base_dir] should be the absolute path of the dataset root.
     
-    yolo_dataset_file does not have to be within dataset_base_dir.
-
-    [class_list] can be an ordered list of class names (the first item will be class 0, 
-    etc.), or the name of a text file containing an ordered list of class names (one per 
-    line, starting from class zero).
+    Args:
+        yolo_dataset_file (str): the file, typically ending in .yaml or .yml, to write.  
+            Does not have to be within dataset_base_dir.
+        dataset_base_dir (str): the absolute base path of the YOLO dataset
+        class_list (list or str): an ordered list of class names (the first item will be class 0, 
+            etc.), or the name of a text file containing an ordered list of class names (one per 
+            line, starting from class zero).
     """
     
     # Read class names
@@ -82,7 +82,9 @@ def write_yolo_dataset_file(yolo_dataset_file,
 # ...def write_yolo_dataset_file(...)
 
             
-def coco_to_yolo(input_image_folder,output_folder,input_file,
+def coco_to_yolo(input_image_folder,
+                 output_folder,
+                 input_file,
                  source_format='coco',
                  overwrite_images=False,
                  create_image_and_label_folders=False,
@@ -97,7 +99,7 @@ def coco_to_yolo(input_image_folder,output_folder,input_file,
                  write_output=True,
                  flatten_paths=True):
     """
-    Convert a COCO-formatted dataset to a YOLO-formatted dataset, optionally flattening the 
+    Converts a COCO-formatted dataset to a YOLO-formatted dataset, optionally flattening the 
     dataset to a single folder in the process.
     
     If the input and output folders are the same, writes .txt files to the input folder,
@@ -106,32 +108,51 @@ def coco_to_yolo(input_image_folder,output_folder,input_file,
     Currently ignores segmentation masks, and errors if an annotation has a 
     segmentation polygon but no bbox.
     
-    source_format can be 'coco' (default) or 'coco_camera_traps'.  The only difference
-    is that when source_format is 'coco_camera_traps', we treat an image with a non-bbox
-    annotation with a category id of 0 as a special case, i.e. that's how an empty image
-    is indicated.  The original COCO standard is a little ambiguous on this issue.  If
-    source_format is 'coco', we either treat images as empty or error, depending on the value
-    of allow_empty_annotations.  allow_empty_annotations has no effect if source_format is
-    'coco_camera_traps'.
-    
-    If create_image_and_label_folders is false, a/b/c/image001.jpg will become a#b#c#image001.jpg, 
-    and the corresponding text file will be a#b#c#image001.txt.  
-    
-    If create_image_and_label_folders is true, a/b/c/image001.jpg will become 
-    images/a#b#c#image001.jpg, and the corresponding text file will be 
-    labels/a#b#c#image001.txt.  Some tools still use this variant of the YOLO standard.
-    
-    If clip_boxes is True, bounding boxes coordinates will be clipped to [0,1].
-    
-    image_id_to_output_image_json_file is an optional *output* file, to which we will write
-    a mapping from image IDs to output file names.
-    
-    images_to_exclude is a list of image files (relative paths in the input folder) that we 
-    should ignore.
-    
-    write_output determines whether we actually copy images and write annotations;
-    setting this to False basically puts this function in "test mode".  The class list
-    file is written regardless of the value of write_output.
+    Args:
+        input_image_folder (str): the folder where images live; filenames in the COCO .json
+            file [input_file] should be relative to this folder
+        output_folder (str): the base folder for the YOLO dataset
+        input_file (str): a .json file in COCO format; can be the same as [input_image_folder], in which case
+            images are left alone.
+        source_format (str, optional): can be 'coco' (default) or 'coco_camera_traps'.  The only difference
+            is that when source_format is 'coco_camera_traps', we treat an image with a non-bbox
+            annotation with a category id of 0 as a special case, i.e. that's how an empty image
+            is indicated.  The original COCO standard is a little ambiguous on this issue.  If
+            source_format is 'coco', we either treat images as empty or error, depending on the value
+            of [allow_empty_annotations].  [allow_empty_annotations] has no effect if source_format is
+            'coco_camera_traps'.
+        create_image_and_label_folder (bool, optional): whether to create separate folders called 'images' and
+            'labels' in the YOLO output folder.  If create_image_and_label_folders is False, 
+            a/b/c/image001.jpg will become a#b#c#image001.jpg, and the corresponding text file will 
+            be a#b#c#image001.txt.  If create_image_and_label_folders is True, a/b/c/image001.jpg will become 
+            images/a#b#c#image001.jpg, and the corresponding text file will be 
+            labels/a#b#c#image001.txt.    
+        clip_boxes (bool, optional): whether to clip bounding box coordinates to the range [0,1] before
+            converting to YOLO xywh format
+        image_id_to_output_image_json_file (str, optional): an optional *output* file, to which we will write
+            a mapping from image IDs to output file names
+        images_to_exclude (list, optional): a list of image files (relative paths in the input folder) that we 
+            should ignore
+        path_replacement_char (str, optional): only relevant if [flatten_paths] is True; this is used to replace
+            path separators, e.g. if [path_replacement_char] is '#' and [flatten_paths] is True, a/b/c/d.jpg
+            becomes a#b#c#d.jpg
+        category_names_to_exclude (str, optional): category names that should not be represented in the
+            YOLO output; only impacts annotations, does not prevent copying images.  There's almost no reason
+            you would want to specify this and [category_names_to_include]. 
+        category_names_to_include (str, optional): allow-list of category names that should be represented in the
+            YOLO output; only impacts annotations, does not prevent copying images.  There's almost no reason
+            you would want to specify this and [category_names_to_exclude]. 
+        write_output (bool, optional): determines whether we actually copy images and write annotations;
+            setting this to False mostly puts this function in "dry run" "mode.  The class list
+            file is written regardless of the value of write_output.
+    
+    Returns:
+        dict: information about the coco --> yolo mapping, containing at least the fields:
+        
+        - class_list_filename: the filename to which we wrote the flat list of class names required 
+          by the YOLO format.
+        - source_image_to_dest_image: a dict mapping source images to destination images
+        - coco_id_to_yolo_id: a dict mapping COCO category IDs to YOLO category IDs        
     """
         
     ## Validate input
@@ -500,12 +521,12 @@ def create_yolo_symlinks(source_folder,images_folder,labels_folder,
                          class_list_output_name='object.data',
                          force_lowercase_image_extension=False):
     """
-    Given a YOLO-formatted folder of images and .txt files, create a folder
+    Given a YOLO-formatted folder of images and .txt files, creates a folder
     of symlinks to all the images, and a folder of symlinks to all the labels. 
-    Used to support preview/editing tools (like BoundingBoxEditor) that assume
-    images and labels are in separate folders.
+    Used to support preview/editing tools that assume images and labels are in separate 
+    folders.
     
-    images_folder and labels_folder are absolute paths.
+    :meta private:
     """    
     
     assert source_folder != images_folder and source_folder != labels_folder
@@ -619,7 +640,7 @@ def main():
     parser.add_argument(
         '--create_bounding_box_editor_symlinks',
         action='store_true',
-        help='Prepare symlinks so the whole folder is BoundingBoxEditor-friendly')        
+        help='Prepare symlinks so the whole folder appears to contain "images" and "labels" folderss')        
     
     if len(sys.argv[1:]) == 0:
         parser.print_help()

data_management/databases/add_width_and_height_to_db.py

@@ -1,10 +1,12 @@
-########
-#
-# add_width_and_height_to_db.py
-#
-# Grabs width and height from actual image files for a .json database that is missing w/h.
-#
-########
+"""
+
+add_width_and_height_to_db.py
+
+Grabs width and height from actual image files for a .json database that is missing w/h.
+
+TODO: this is a one-off script waiting to be cleaned up for more general use.
+
+"""
 
 #%% Imports and constants
 
@@ -14,16 +16,18 @@ from PIL import Image
 datafile = '/datadrive/snapshotserengeti/databases/snapshotserengeti.json'
 image_base = '/datadrive/snapshotserengeti/images/'
 
+def main():
 
-#%% Execution
+    with open(datafile,'r') as f:
+        data = json.load(f)
 
-with open(datafile,'r') as f:
-    data = json.load(f)
+    for im in data['images']:
+        if 'height' not in im:
+            im_w, im_h = Image.open(image_base+im['file_name']).size
+            im['height'] = im_h
+            im['width'] = im_w
 
-for im in data['images']:
-    if 'height' not in im:
-        im_w, im_h = Image.open(image_base+im['file_name']).size
-        im['height'] = im_h
-        im['width'] = im_w
+    json.dump(data, open(datafile,'w'))
 
-json.dump(data, open(datafile,'w'))
+if __name__ == '__main__':
+    main()

data_management/databases/combine_coco_camera_traps_files.py

@@ -1,17 +1,19 @@
-########
-#
-# combine_coco_camera_traps_files.py
-# 
-# Merges two or more .json files in COCO Camera Traps format, optionally
-# writing the results to another .json file.
-#
-# - Concatenates image lists, erroring if images are not unique.
-# - Errors on unrecognized fields.
-# - Checks compatibility in info structs, within reason.
-# 
-# combine_coco_camera_traps_files input1.json input2.json ... inputN.json output.json
-#
-########
+"""
+
+combine_coco_camera_traps_files.py
+
+Merges two or more .json files in COCO Camera Traps format, optionally
+writing the results to another .json file.
+
+- Concatenates image lists, erroring if images are not unique.
+- Errors on unrecognized fields.
+- Checks compatibility in info structs, within reason.
+
+*Example command-line invocation*
+    
+combine_coco_camera_traps_files input1.json input2.json ... inputN.json output.json
+
+"""
 
 #%% Constants and imports
 
@@ -19,26 +21,25 @@ import argparse
 import json
 import sys
 
-from typing import Any, Dict, Iterable, Mapping, List, Optional
-
 
 #%% Merge functions
 
-def combine_cct_files(input_files: List[str],
-                             output_file: Optional[str] = None,
-                             require_uniqueness: Optional[bool] = True,
-                             filename_prefixes: Optional[dict] = None
-                             ) -> Dict[str, Any]:
+def combine_cct_files(input_files, output_file=None, require_uniqueness=True,
+                      filename_prefixes=None):                             
     """
-    Merges list of COCO Camera Traps files *input_files* into a single
-    dictionary, optionally writing the result to *output_file*.
+    Merges the list of COCO Camera Traps files [input_files] into a single
+    dictionary, optionally writing the result to [output_file].
 
     Args:
-        input_files: list of str, paths to JSON detection files
-        output_file: optional str, path to write merged JSON
-        require_uniqueness: bool, whether to require that the images in
+        input_files (list): paths to CCT .json files
+        output_file (str, optional): path to write merged .json file
+        require_uniqueness (bool): whether to require that the images in
             each input_dict be unique
+    
+    Returns:
+        dict: the merged COCO-formatted .json dict
     """
+    
     input_dicts = []
     print('Loading input files')
     for fn in input_files:
@@ -61,11 +62,9 @@ def combine_cct_files(input_files: List[str],
     return merged_dict
 
 
-def combine_cct_dictionaries(input_dicts: Iterable[Mapping[str, Any]],
-                                    require_uniqueness: Optional[bool] = True                                    
-                                    ) -> Dict[str, Any]:
+def combine_cct_dictionaries(input_dicts, require_uniqueness=True):                                    
     """
-    Merges the list of COCO Camera Traps dictionaries *input_dicts*.  See header
+    Merges the list of COCO Camera Traps dictionaries [input_dicts].  See module header
     comment for details on merge rules.
 
     Args:
@@ -73,7 +72,8 @@ def combine_cct_dictionaries(input_dicts: Iterable[Mapping[str, Any]],
         require_uniqueness: bool, whether to require that the images in
             each input_dict be unique
 
-    Returns: dict, represents the merged JSON
+    Returns: 
+        dict: the merged COCO-formatted .json dict
     """
     
     filename_to_image = {}
@@ -177,12 +177,16 @@ def combine_cct_dictionaries(input_dicts: Iterable[Mapping[str, Any]],
                    'categories': all_categories,
                    'images': sorted_images,
                    'annotations': all_annotations}
+    
     return merged_dict
 
+# ...combine_cct_dictionaries(...)
+
 
 #%% Command-line driver
 
 def main():
+    
     parser = argparse.ArgumentParser()
     parser.add_argument(
         'input_paths', nargs='+',

data_management/databases/integrity_check_json_db.py

@@ -1,19 +1,19 @@
-########
-#
-# integrity_check_json_db.py
-#
-# Does some integrity-checking and computes basic statistics on a db, specifically:
-#
-# * Verifies that required fields are present and have the right types
-# * Verifies that annotations refer to valid images
-# * Verifies that annotations refer to valid categories
-# * Verifies that image, category, and annotation IDs are unique 
-# * Optionally checks file existence
-# * Finds un-annotated images
-# * Finds unused categories
-# * Prints a list of categories sorted by count
-#
-########
+"""
+
+integrity_check_json_db.py
+
+Does some integrity-checking and computes basic statistics on a COCO Camera Traps .json file, specifically:
+
+* Verifies that required fields are present and have the right types
+* Verifies that annotations refer to valid images
+* Verifies that annotations refer to valid categories
+* Verifies that image, category, and annotation IDs are unique 
+* Optionally checks file existence
+* Finds un-annotated images
+* Finds unused categories
+* Prints a list of categories sorted by count
+
+"""
 
 #%% Constants and environment
 
@@ -33,14 +33,32 @@ from md_utils import ct_utils
 #%% Classes and environment
 
 class IntegrityCheckOptions:
+    """
+    Options for integrity_check_json_db()
+    """
     
+    #: Image path; the filenames in the .json file should be relative to this folder
     baseDir = ''
+    
+    #: Should we validate the image sizes?
     bCheckImageSizes = False
+    
+    #: Should we check that all the images in the .json file exist on disk?
     bCheckImageExistence = False
+    
+    #: Should we search [baseDir] for images that are not used in the .json file?
     bFindUnusedImages = False
+    
+    #: Should we require that all images in the .json file have a 'location' field?
     bRequireLocation = True
+    
+    #: For debugging, limit the number of images we'll process
     iMaxNumImages = -1
+    
+    #: Number of threads to use for parallelization, set to <= 1 to disable parallelization
     nThreads = 10
+    
+    #: Enable additional debug output
     verbose = True
     
     
@@ -50,7 +68,21 @@ defaultOptions = IntegrityCheckOptions()
 
 #%% Functions
 
-def check_image_existence_and_size(image,options=None):
+def _check_image_existence_and_size(image,options=None):
+    """
+    Validate the image represented in the CCT image dict [image], which should have fields:
+    
+    * file_name
+    * width
+    * height
+
+    Args:
+        image (dict): image to validate
+        options (IntegrityCheckOptions): parameters impacting validation
+    
+    Returns:
+        bool: whether this image passes validation
+    """
 
     if options is None:        
         options = defaultOptions
@@ -80,9 +112,17 @@ def check_image_existence_and_size(image,options=None):
   
 def integrity_check_json_db(jsonFile, options=None):
     """
-    jsonFile can be a filename or an already-loaded json database
+    Does some integrity-checking and computes basic statistics on a COCO Camera Traps .json file; see
+    module header comment for a list of the validation steps.
     
-    return sortedCategories, data, errorInfo
+    Args:
+        jsonFile (str): filename to validate, or an already-loaded dict
+    
+    Returns:
+        tuple: tuple containing:
+            - sortedCategories (dict): list of categories used in [jsonFile], sorted by frequency
+            - data (dict): the data loaded from [jsonFile]
+            - errorInfo (dict): specific validation errors
     """
     
     if options is None:       
@@ -264,11 +304,11 @@ def integrity_check_json_db(jsonFile, options=None):
             defaultOptions.baseDir = options.baseDir
             defaultOptions.bCheckImageSizes = options.bCheckImageSizes
             defaultOptions.bCheckImageExistence = options.bCheckImageExistence
-            results = tqdm(pool.imap(check_image_existence_and_size, images), total=len(images))
+            results = tqdm(pool.imap(_check_image_existence_and_size, images), total=len(images))
         else:
             results = []
             for im in tqdm(images):
-                results.append(check_image_existence_and_size(im,options))
+                results.append(_check_image_existence_and_size(im,options))
                 
         for iImage,r in enumerate(results):
             if not r:            
@@ -407,9 +447,7 @@ def main():
     ct_utils.args_to_object(args, options)
     integrity_check_json_db(args.jsonFile,options)
 
-
-if __name__ == '__main__':
-    
+if __name__ == '__main__':    
     main()
 
 

data_management/databases/subset_json_db.py

@@ -1,14 +1,14 @@
-########
-#
-# subset_json_db.py
-#
-# Select a subset of images (and associated annotations) from a .json file
-# in COCO Camera Traps format.
-#
-# To subset the .json files in the MegaDetector output format, see 
-# subset_json_detector_output.py
-#
-########
+"""
+
+subset_json_db.py
+
+Select a subset of images (and associated annotations) from a .json file in COCO 
+Camera Traps format based on a string query.
+
+To subset .json files in the MegaDetector output format, see
+subset_json_detector_output.py.
+
+"""
     
 #%% Constants and imports
 
@@ -26,6 +26,16 @@ def subset_json_db(input_json, query, output_json=None, ignore_case=False):
     Given a json file (or dictionary already loaded from a json file), produce a new 
     database containing only the images whose filenames contain the string 'query', 
     optionally writing that DB output to a new json file.
+    
+    Args:
+        input_json (str): COCO Camera Traps .json file to load, or an already-loaded dict
+        query (str): string to query for, only include images in the output whose filenames 
+            contain this string.
+        output_json (str, optional): file to write the resulting .json file to
+        ignore_case (bool, optional): whether to perform a case-insensitive search for [query]
+        
+    Returns:
+        dict: possibly-modified CCT dictionary
     """
     
     if ignore_case:
@@ -76,8 +86,8 @@ if False:
     
     #%%
     
-    input_json = r"E:\Statewide_wolf_container\idfg_20190409.json"
-    output_json = r"E:\Statewide_wolf_container\idfg_20190409_clearcreek.json"
+    input_json = r"e:\Statewide_wolf_container\idfg_20190409.json"
+    output_json = r"e:\Statewide_wolf_container\idfg_20190409_clearcreek.json"
     query = 'clearcreek'
     ignore_case = True
     db = subset_json_db(input_json, query, output_json, ignore_case)
@@ -101,6 +111,5 @@ def main():
     
     subset_json_db(args.input_json,args.query,args.output_json,args.ignore_case)
 
-if __name__ == '__main__':
-    
+if __name__ == '__main__':    
     main()

data_management/generate_crops_from_cct.py

@@ -1,11 +1,11 @@
-########
-#
-# generate_crops_from_cct.py
-#
-# Given a .json file in COCO Camera Traps format, create a cropped image for
-# each bounding box.
-#
-########
+"""
+
+generate_crops_from_cct.py
+
+Given a .json file in COCO Camera Traps format, creates a cropped image for
+each bounding box.
+
+"""
 
 #%% Imports and constants
 
@@ -19,6 +19,23 @@ from PIL import Image
 #%% Functions
 
 def generate_crops_from_cct(cct_file,image_dir,output_dir,padding=0,flat_output=True):
+    """
+    Given a .json file in COCO Camera Traps format, creates a cropped image for
+    each bounding box.
+    
+    Args:
+        cct_file (str): the COCO .json file from which we should load data
+        image_dir (str): the folder where the images live; filenames in the .json
+            file should be relative to this folder
+        output_dir (str): the folder where we should write cropped images
+        padding (float, optional): number of pixels we should expand each box before
+            cropping
+        flat_output (bool, optional): if False, folder structure will be preserved
+            in the output, e.g. the image a/b/c/d.jpg will result in image files 
+            in the output folder called, e.g., a/b/c/d_crop_000_id_12345.jpg.  If
+            [flat_output] is True, the corresponding output image will be 
+            a_b_c_d_crop_000_id_12345.jpg.            
+    """
     
     ## Read and validate input
     
@@ -123,45 +140,10 @@ if False:
     flat_output = True
     output_dir = '/home/user/tmp/noaa-fish-crops'
     
-    #%%
-    
     generate_crops_from_cct(cct_file,image_dir,output_dir,padding,flat_output=True)
     files = os.listdir(output_dir)
     
-    #%%
-    
-    import random
-    fn = os.path.join(output_dir,random.choice(files))
-    
-    from md_utils.path_utils import open_file
-    open_file(fn)
-
-
-#%% Scrap
-
-if False:
     
-    pass
+#%% Command-line driver
 
-    #%%
-    
-    from md_visualization.visualize_db import DbVizOptions,visualize_db
-    
-    db_path = cct_file
-    output_dir = os.path.expanduser('~/tmp/noaa-fish-preview')
-    image_base_dir = image_dir
-    
-    options = DbVizOptions()
-    options.num_to_visualize = None
-    
-    options.parallelize_rendering_n_cores = 5
-    options.parallelize_rendering = True    
-    
-    options.viz_size = (-1, -1)
-    options.trim_to_images_with_bboxes = True
-    
-    options.box_thickness = 4
-    options.box_expansion = 25
-    
-    htmlOutputFile,db = visualize_db(db_path,output_dir,image_base_dir,options)
-    
+# TODO