megadetector 5.0.7__py3-none-any.whl → 5.0.9__py3-none-any.whl

data_management/coco_to_labelme.py

@@ -0,0 +1,272 @@
+"""
+
+coco_to_labelme.py
+
+Converts a COCO dataset to labelme format (one .json per image file).
+
+If you want to convert YOLO-formatted data to labelme format, use yolo_to_coco, then 
+coco_to_labelme.
+
+"""
+
+#%% Imports and constants
+
+import os
+import json
+
+from tqdm import tqdm
+from collections import defaultdict
+
+from md_visualization.visualization_utils import open_image
+
+
+#%% Functions
+
+def get_labelme_dict_for_image_from_coco_record(im,annotations,categories,info=None):
+    """
+    For the given image struct in COCO format and associated list of annotations, reformats the 
+    detections into labelme format.  
+    
+    Args:
+        im (dict): image dict, as loaded from a COCO .json file; 'height' and 'width' are required
+        annotations (list): a list of annotations that refer to this image (this function errors if 
+            that's not the case)
+        categories (list): a list of category in dicts in COCO format ({'id':x,'name':'s'})
+        info (dict, optional): a dict to store in a non-standard "custom_info"  field in the output
+        
+    Returns:
+        dict: a dict in labelme format, suitable for writing to a labelme .json file
+    """
+    
+    image_base_name = os.path.basename(im['file_name'])
+    
+    output_dict = {}
+    if info is not None:
+        output_dict['custom_info'] = info
+    output_dict['version'] = '5.3.0a0'
+    output_dict['flags'] = {}
+    output_dict['shapes'] = []
+    output_dict['imagePath'] = image_base_name
+    output_dict['imageHeight'] = im['height']
+    output_dict['imageWidth'] = im['width']
+    output_dict['imageData'] = None
+    
+    # Store COCO categories in case we want to reconstruct the original IDs later
+    output_dict['coco_categories'] = categories
+    
+    category_id_to_name = {c['id']:c['name'] for c in categories}
+    
+    if 'flags' in im:
+        output_dict['flags'] = im['flags']
+        
+    # ann = annotations[0]
+    for ann in annotations:
+        
+        assert ann['image_id'] == im['id'], 'Annotation {} does not refer to image {}'.format(
+            ann['id'],im['id'])
+        
+        if 'bbox' not in ann:
+            continue
+        
+        shape = {}
+        shape['label'] = category_id_to_name[ann['category_id']] 
+        shape['shape_type'] = 'rectangle'
+        shape['description'] = ''
+        shape['group_id'] = None
+        
+        # COCO boxes are [x_min, y_min, width_of_box, height_of_box] (absolute)
+        # 
+        # labelme boxes are [[x0,y0],[x1,y1]] (absolute)
+        x0 = ann['bbox'][0]
+        y0 = ann['bbox'][1]
+        x1 = ann['bbox'][0] + ann['bbox'][2]
+        y1 = ann['bbox'][1] + ann['bbox'][3]
+        
+        shape['points'] = [[x0,y0],[x1,y1]]
+        output_dict['shapes'].append(shape)
+    
+    # ...for each detection
+    
+    return output_dict
+
+# ...def get_labelme_dict_for_image()
+
+
+def coco_to_labelme(coco_data,image_base,overwrite=False,bypass_image_size_check=False,verbose=False):
+    """
+    For all the images in [coco_data] (a dict or a filename), write a .json file in 
+    labelme format alongside the corresponding relative path within image_base.    
+    """
+    
+    # Load COCO data if necessary
+    if isinstance(coco_data,str):
+        with open(coco_data,'r') as f:
+            coco_data = json.load(f)
+    assert isinstance(coco_data,dict)
+     
+    
+    ## Read image sizes if necessary
+    
+    if bypass_image_size_check:
+        
+        print('Bypassing size check')
+        
+    else:
+    
+        # TODO: parallelize this loop
+        
+        print('Reading/validating image sizes...')
+        
+        # im = coco_data['images'][0]
+        for im in tqdm(coco_data['images']):
+            
+            # Make sure this file exists
+            im_full_path = os.path.join(image_base,im['file_name'])
+            assert os.path.isfile(im_full_path), 'Image file {} does not exist'.format(im_full_path)
+            
+            # Load w/h information if necessary
+            if 'height' not in im or 'width' not in im:
+                
+                try:
+                    pil_im = open_image(im_full_path)
+                    im['width'] = pil_im.width
+                    im['height'] = pil_im.height
+                except Exception:
+                    print('Warning: cannot open image {}'.format(im_full_path))
+                    if 'failure' not in im:
+                        im['failure'] = 'Failure image access'
+    
+            # ...if we need to read w/h information
+            
+        # ...for each image
+    
+    # ...if we need to load image sizes
+    
+    
+    ## Generate labelme files
+    
+    print('Generating .json files...')
+    
+    image_id_to_annotations = defaultdict(list)
+    for ann in coco_data['annotations']:
+        image_id_to_annotations[ann['image_id']].append(ann)
+        
+    n_json_files_written = 0
+    n_json_files_error = 0
+    n_json_files_exist = 0
+    
+    # Write output
+    for im in tqdm(coco_data['images']):
+        
+        # Skip this image if it failed to load in whatever system generated this COCO file
+        skip_image = False
+        
+        # Errors are represented differently depending on the source
+        for error_string in ('failure','error'):
+            if (error_string in im) and (im[error_string] is not None):
+                if verbose:
+                    print('Warning: skipping labelme file generation for failed image {}'.format(
+                        im['file_name']))
+                skip_image = True
+                n_json_files_error += 1
+                break
+        if skip_image:
+            continue
+                    
+        im_full_path = os.path.join(image_base,im['file_name'])
+        json_path = os.path.splitext(im_full_path)[0] + '.json'
+        
+        if (not overwrite) and (os.path.isfile(json_path)):
+            if verbose:
+                print('Skipping existing file {}'.format(json_path))
+            n_json_files_exist += 1
+            continue
+    
+        annotations_this_image = image_id_to_annotations[im['id']]
+        output_dict = get_labelme_dict_for_image_from_coco_record(im,
+                                                                  annotations_this_image,
+                                                                  coco_data['categories'],
+                                                                  info=None)
+        
+        n_json_files_written += 1
+        with open(json_path,'w') as f:
+            json.dump(output_dict,f,indent=1)
+            
+    # ...for each image
+    
+    print('\nWrote {} .json files (skipped {} for errors, {} because they exist)'.format(
+        n_json_files_written,n_json_files_error,n_json_files_exist))
+    
+# ...def coco_to_labelme()
+
+
+#%% Interactive driver
+
+if False:
+    
+    pass
+
+    #%% Configure options
+    
+    coco_file = \
+        r'C:\\temp\\snapshot-exploration\\images\\training-images-good\\training-images-good_from_yolo.json'
+    image_folder = os.path.dirname(coco_file)    
+    overwrite = True    
+    
+    
+    #%% Programmatic execution
+    
+    coco_to_labelme(coco_data=coco_file,image_base=image_folder,overwrite=overwrite)
+
+    
+    #%% Command-line execution
+    
+    s = 'python coco_to_labelme.py "{}" "{}"'.format(coco_file,image_folder)
+    if overwrite:
+        s += ' --overwrite'
+        
+    print(s)
+    import clipboard; clipboard.copy(s)
+
+
+    #%% Opening labelme
+    
+    s = 'python labelme {}'.format(image_folder)
+    print(s)
+    import clipboard; clipboard.copy(s)
+    
+
+#%% Command-line driver
+
+import sys,argparse
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        description='Convert a COCO database to labelme annotation format')
+    
+    parser.add_argument(
+        'coco_file',
+        type=str,
+        help='Path to COCO data file (.json)')
+    
+    parser.add_argument(
+        'image_base',
+        type=str,
+        help='Path to images (also the output folder)')
+    
+    parser.add_argument(
+        '--overwrite',
+        action='store_true',
+        help='Overwrite existing labelme .json files')
+
+    if len(sys.argv[1:]) == 0:
+        parser.print_help()
+        parser.exit()
+
+    args = parser.parse_args()
+
+    coco_to_labelme(coco_data=args.coco_file,image_base=args.image_base,overwrite=args.overwrite)    
+
+if __name__ == '__main__':
+    main()

data_management/coco_to_yolo.py

@@ -1,20 +1,20 @@
-########
-#
-# coco_to_yolo.py
-#
-# Converts a COCO-formatted dataset to a YOLO-formatted dataset, flattening
-# the dataset (to a single folder) in the process.
-#
-# If the input and output folders are the same, writes .txt files to the input folder,
-# and neither moves nor modifies images.
-#
-# Currently ignores segmentation masks, and errors if an annotation has a 
-# segmentation polygon but no bbox.
-# 
-# Has only been tested on a handful of COCO Camera Traps data sets; if you
-# use it for more general COCO conversion, YMMV.
-#
-########
+"""
+
+coco_to_yolo.py
+
+Converts a COCO-formatted dataset to a YOLO-formatted dataset, flattening
+the dataset (to a single folder) in the process.
+
+If the input and output folders are the same, writes .txt files to the input folder,
+and neither moves nor modifies images.
+
+Currently ignores segmentation masks, and errors if an annotation has a 
+segmentation polygon but no bbox.
+
+Has only been tested on a handful of COCO Camera Traps data sets; if you
+use it for more general COCO conversion, YMMV.
+
+"""
 
 #%% Imports and constants
 
@@ -37,16 +37,16 @@ def write_yolo_dataset_file(yolo_dataset_file,
                             val_folder_relative=None,
                             test_folder_relative=None):
     """
-    Write a YOLOv5 dataset.yaml file to the absolute path yolo_dataset_file (should
+    Write a YOLOv5 dataset.yaml file to the absolute path [yolo_dataset_file] (should
     have a .yaml extension, though it's only a warning if it doesn't).  
-
-    [dataset_base_dir] should be the absolute path of the dataset root.
     
-    yolo_dataset_file does not have to be within dataset_base_dir.
-
-    [class_list] can be an ordered list of class names (the first item will be class 0, 
-    etc.), or the name of a text file containing an ordered list of class names (one per 
-    line, starting from class zero).
+    Args:
+        yolo_dataset_file (str): the file, typically ending in .yaml or .yml, to write.  
+            Does not have to be within dataset_base_dir.
+        dataset_base_dir (str): the absolute base path of the YOLO dataset
+        class_list (list or str): an ordered list of class names (the first item will be class 0, 
+            etc.), or the name of a text file containing an ordered list of class names (one per 
+            line, starting from class zero).
     """
     
     # Read class names
@@ -56,6 +56,10 @@ def write_yolo_dataset_file(yolo_dataset_file,
         class_lines = [s.strip() for s in class_lines]    
         class_list = [s for s in class_lines if len(s) > 0]
 
+    if not (yolo_dataset_file.endswith('.yml') or yolo_dataset_file.endswith('.yaml')):
+        print('Warning: writing dataset file to a non-yml/yaml extension:\n{}'.format(
+            yolo_dataset_file))
+        
     # Write dataset.yaml
     with open(yolo_dataset_file,'w') as f:
         
@@ -78,7 +82,9 @@ def write_yolo_dataset_file(yolo_dataset_file,
 # ...def write_yolo_dataset_file(...)
 
             
-def coco_to_yolo(input_image_folder,output_folder,input_file,
+def coco_to_yolo(input_image_folder,
+                 output_folder,
+                 input_file,
                  source_format='coco',
                  overwrite_images=False,
                  create_image_and_label_folders=False,
@@ -93,7 +99,7 @@ def coco_to_yolo(input_image_folder,output_folder,input_file,
                  write_output=True,
                  flatten_paths=True):
     """
-    Convert a COCO-formatted dataset to a YOLO-formatted dataset, optionally flattening the 
+    Converts a COCO-formatted dataset to a YOLO-formatted dataset, optionally flattening the 
     dataset to a single folder in the process.
     
     If the input and output folders are the same, writes .txt files to the input folder,
@@ -102,32 +108,51 @@ def coco_to_yolo(input_image_folder,output_folder,input_file,
     Currently ignores segmentation masks, and errors if an annotation has a 
     segmentation polygon but no bbox.
     
-    source_format can be 'coco' (default) or 'coco_camera_traps'.  The only difference
-    is that when source_format is 'coco_camera_traps', we treat an image with a non-bbox
-    annotation with a category id of 0 as a special case, i.e. that's how an empty image
-    is indicated.  The original COCO standard is a little ambiguous on this issue.  If
-    source_format is 'coco', we either treat images as empty or error, depending on the value
-    of allow_empty_annotations.  allow_empty_annotations has no effect if source_format is
-    'coco_camera_traps'.
-    
-    If create_image_and_label_folders is false, a/b/c/image001.jpg will become a#b#c#image001.jpg, 
-    and the corresponding text file will be a#b#c#image001.txt.  
-    
-    If create_image_and_label_folders is true, a/b/c/image001.jpg will become 
-    images/a#b#c#image001.jpg, and the corresponding text file will be 
-    labels/a#b#c#image001.txt.  Some tools still use this variant of the YOLO standard.
-    
-    If clip_boxes is True, bounding boxes coordinates will be clipped to [0,1].
-    
-    image_id_to_output_image_json_file is an optional *output* file, to which we will write
-    a mapping from image IDs to output file names.
-    
-    images_to_exclude is a list of image files (relative paths in the input folder) that we 
-    should ignore.
-    
-    write_output determines whether we actually copy images and write annotations;
-    setting this to False basically puts this function in "test mode".  The class list
-    file is written regardless of the value of write_output.
+    Args:
+        input_image_folder (str): the folder where images live; filenames in the COCO .json
+            file [input_file] should be relative to this folder
+        output_folder (str): the base folder for the YOLO dataset
+        input_file (str): a .json file in COCO format; can be the same as [input_image_folder], in which case
+            images are left alone.
+        source_format (str, optional): can be 'coco' (default) or 'coco_camera_traps'.  The only difference
+            is that when source_format is 'coco_camera_traps', we treat an image with a non-bbox
+            annotation with a category id of 0 as a special case, i.e. that's how an empty image
+            is indicated.  The original COCO standard is a little ambiguous on this issue.  If
+            source_format is 'coco', we either treat images as empty or error, depending on the value
+            of [allow_empty_annotations].  [allow_empty_annotations] has no effect if source_format is
+            'coco_camera_traps'.
+        create_image_and_label_folder (bool, optional): whether to create separate folders called 'images' and
+            'labels' in the YOLO output folder.  If create_image_and_label_folders is False, 
+            a/b/c/image001.jpg will become a#b#c#image001.jpg, and the corresponding text file will 
+            be a#b#c#image001.txt.  If create_image_and_label_folders is True, a/b/c/image001.jpg will become 
+            images/a#b#c#image001.jpg, and the corresponding text file will be 
+            labels/a#b#c#image001.txt.    
+        clip_boxes (bool, optional): whether to clip bounding box coordinates to the range [0,1] before
+            converting to YOLO xywh format
+        image_id_to_output_image_json_file (str, optional): an optional *output* file, to which we will write
+            a mapping from image IDs to output file names
+        images_to_exclude (list, optional): a list of image files (relative paths in the input folder) that we 
+            should ignore
+        path_replacement_char (str, optional): only relevant if [flatten_paths] is True; this is used to replace
+            path separators, e.g. if [path_replacement_char] is '#' and [flatten_paths] is True, a/b/c/d.jpg
+            becomes a#b#c#d.jpg
+        category_names_to_exclude (str, optional): category names that should not be represented in the
+            YOLO output; only impacts annotations, does not prevent copying images.  There's almost no reason
+            you would want to specify this and [category_names_to_include]. 
+        category_names_to_include (str, optional): allow-list of category names that should be represented in the
+            YOLO output; only impacts annotations, does not prevent copying images.  There's almost no reason
+            you would want to specify this and [category_names_to_exclude]. 
+        write_output (bool, optional): determines whether we actually copy images and write annotations;
+            setting this to False mostly puts this function in "dry run" "mode.  The class list
+            file is written regardless of the value of write_output.
+    
+    Returns:
+        dict: information about the coco --> yolo mapping, containing at least the fields:
+        
+        - class_list_filename: the filename to which we wrote the flat list of class names required 
+          by the YOLO format.
+        - source_image_to_dest_image: a dict mapping source images to destination images
+        - coco_id_to_yolo_id: a dict mapping COCO category IDs to YOLO category IDs        
     """
         
     ## Validate input
@@ -189,7 +214,6 @@ def coco_to_yolo(input_image_folder,output_folder,input_file,
     coco_id_to_name = {}
     yolo_id_to_name = {}
     coco_category_ids_to_exclude = set()
-    category_exclusion_warnings_printed = set()
     
     for category in data['categories']:
         coco_id_to_name[category['id']] = category['name']
@@ -465,9 +489,9 @@ def coco_to_yolo(input_image_folder,output_folder,input_file,
             #
             # https://github.com/ultralytics/yolov5/issues/3218
             #
-            # I think this is also true for images with empty annotation files, but 
-            # I'm using the convention suggested on that issue, i.e. hard negatives 
-            # are expressed as images without .txt files.
+            # I think this is also true for images with empty .txt files, but 
+            # I'm using the convention suggested on that issue, i.e. hard 
+            # negatives are expressed as images without .txt files.
             if len(bboxes) > 0:
                 
                 with open(dest_txt,'w') as f:
@@ -497,12 +521,12 @@ def create_yolo_symlinks(source_folder,images_folder,labels_folder,
                          class_list_output_name='object.data',
                          force_lowercase_image_extension=False):
     """
-    Given a YOLO-formatted folder of images and .txt files, create a folder
+    Given a YOLO-formatted folder of images and .txt files, creates a folder
     of symlinks to all the images, and a folder of symlinks to all the labels. 
-    Used to support preview/editing tools (like BoundingBoxEditor) that assume
-    images and labels are in separate folders.
+    Used to support preview/editing tools that assume images and labels are in separate 
+    folders.
     
-    images_folder and labels_folder are absolute paths.
+    :meta private:
     """    
     
     assert source_folder != images_folder and source_folder != labels_folder
@@ -616,7 +640,7 @@ def main():
     parser.add_argument(
         '--create_bounding_box_editor_symlinks',
         action='store_true',
-        help='Prepare symlinks so the whole folder is BoundingBoxEditor-friendly')        
+        help='Prepare symlinks so the whole folder appears to contain "images" and "labels" folderss')        
     
     if len(sys.argv[1:]) == 0:
         parser.print_help()

data_management/databases/add_width_and_height_to_db.py

@@ -1,10 +1,12 @@
-########
-#
-# add_width_and_height_to_db.py
-#
-# Grabs width and height from actual image files for a .json database that is missing w/h.
-#
-########
+"""
+
+add_width_and_height_to_db.py
+
+Grabs width and height from actual image files for a .json database that is missing w/h.
+
+TODO: this is a one-off script waiting to be cleaned up for more general use.
+
+"""
 
 #%% Imports and constants
 
@@ -14,16 +16,18 @@ from PIL import Image
 datafile = '/datadrive/snapshotserengeti/databases/snapshotserengeti.json'
 image_base = '/datadrive/snapshotserengeti/images/'
 
+def main():
 
-#%% Execution
+    with open(datafile,'r') as f:
+        data = json.load(f)
 
-with open(datafile,'r') as f:
-    data = json.load(f)
+    for im in data['images']:
+        if 'height' not in im:
+            im_w, im_h = Image.open(image_base+im['file_name']).size
+            im['height'] = im_h
+            im['width'] = im_w
 
-for im in data['images']:
-    if 'height' not in im:
-        im_w, im_h = Image.open(image_base+im['file_name']).size
-        im['height'] = im_h
-        im['width'] = im_w
+    json.dump(data, open(datafile,'w'))
 
-json.dump(data, open(datafile,'w'))
+if __name__ == '__main__':
+    main()

data_management/databases/combine_coco_camera_traps_files.py

@@ -1,17 +1,19 @@
-########
-#
-# combine_coco_camera_traps_files.py
-# 
-# Merges two or more .json files in COCO Camera Traps format, optionally
-# writing the results to another .json file.
-#
-# - Concatenates image lists, erroring if images are not unique.
-# - Errors on unrecognized fields.
-# - Checks compatibility in info structs, within reason.
-# 
-# combine_coco_camera_traps_files input1.json input2.json ... inputN.json output.json
-#
-########
+"""
+
+combine_coco_camera_traps_files.py
+
+Merges two or more .json files in COCO Camera Traps format, optionally
+writing the results to another .json file.
+
+- Concatenates image lists, erroring if images are not unique.
+- Errors on unrecognized fields.
+- Checks compatibility in info structs, within reason.
+
+*Example command-line invocation*
+    
+combine_coco_camera_traps_files input1.json input2.json ... inputN.json output.json
+
+"""
 
 #%% Constants and imports
 
@@ -19,26 +21,25 @@ import argparse
 import json
 import sys
 
-from typing import Any, Dict, Iterable, Mapping, List, Optional
-
 
 #%% Merge functions
 
-def combine_cct_files(input_files: List[str],
-                             output_file: Optional[str] = None,
-                             require_uniqueness: Optional[bool] = True,
-                             filename_prefixes: Optional[dict] = None
-                             ) -> Dict[str, Any]:
+def combine_cct_files(input_files, output_file=None, require_uniqueness=True,
+                      filename_prefixes=None):                             
     """
-    Merges list of COCO Camera Traps files *input_files* into a single
-    dictionary, optionally writing the result to *output_file*.
+    Merges the list of COCO Camera Traps files [input_files] into a single
+    dictionary, optionally writing the result to [output_file].
 
     Args:
-        input_files: list of str, paths to JSON detection files
-        output_file: optional str, path to write merged JSON
-        require_uniqueness: bool, whether to require that the images in
+        input_files (list): paths to CCT .json files
+        output_file (str, optional): path to write merged .json file
+        require_uniqueness (bool): whether to require that the images in
             each input_dict be unique
+    
+    Returns:
+        dict: the merged COCO-formatted .json dict
     """
+    
     input_dicts = []
     print('Loading input files')
     for fn in input_files:
@@ -61,11 +62,9 @@ def combine_cct_files(input_files: List[str],
     return merged_dict
 
 
-def combine_cct_dictionaries(input_dicts: Iterable[Mapping[str, Any]],
-                                    require_uniqueness: Optional[bool] = True                                    
-                                    ) -> Dict[str, Any]:
+def combine_cct_dictionaries(input_dicts, require_uniqueness=True):                                    
     """
-    Merges the list of COCO Camera Traps dictionaries *input_dicts*.  See header
+    Merges the list of COCO Camera Traps dictionaries [input_dicts].  See module header
     comment for details on merge rules.
 
     Args:
@@ -73,7 +72,8 @@ def combine_cct_dictionaries(input_dicts: Iterable[Mapping[str, Any]],
         require_uniqueness: bool, whether to require that the images in
             each input_dict be unique
 
-    Returns: dict, represents the merged JSON
+    Returns: 
+        dict: the merged COCO-formatted .json dict
     """
     
     filename_to_image = {}
@@ -177,12 +177,16 @@ def combine_cct_dictionaries(input_dicts: Iterable[Mapping[str, Any]],
                    'categories': all_categories,
                    'images': sorted_images,
                    'annotations': all_annotations}
+    
     return merged_dict
 
+# ...combine_cct_dictionaries(...)
+
 
 #%% Command-line driver
 
 def main():
+    
     parser = argparse.ArgumentParser()
     parser.add_argument(
         'input_paths', nargs='+',