megadetector 5.0.27__py3-none-any.whl → 5.0.29__py3-none-any.whl

megadetector/data_management/cct_to_wi.py

@@ -5,7 +5,7 @@ cct_to_wi.py
 Converts COCO Camera Traps .json files to the Wildlife Insights
 batch upload format.
 
-**This is very much just a demo script; all the relevant constants are hard-coded 
+**This is very much just a demo script; all the relevant constants are hard-coded
 at the top of main().**
 
 But given that caveat, it works.  You need to set up all the paths in the "paths" cell
@@ -23,42 +23,42 @@ Also see:
 import os
 import json
 import pandas as pd
-from collections import defaultdict 
+from collections import defaultdict
 
 
 #%% Main wrapper
 
-def main():
+def main(): # noqa
     """
     Converts COCO Camera Traps .json files to the Wildlife Insights
-    batch upload format; to use this, you need to modify all the paths in the "Paths"    
+    batch upload format; to use this, you need to modify all the paths in the "Paths"
     cell.
     """
-    
+
     #%% Paths
 
     # A COCO camera traps file with information about this dataset
     input_file = r'c:\temp\camera_trap_images_no_people\bellevue_camera_traps.2020-12-26.json'
-    
-    # A .json dictionary mapping common names in this dataset to dictionaries with the 
+
+    # A .json dictionary mapping common names in this dataset to dictionaries with the
     # WI taxonomy fields: common_name, wi_taxon_id, class, order, family, genus, species
     taxonomy_file = r'c:\temp\camera_trap_images_no_people\bellevue_camera_traps_to_wi.json'
 
     # The folder where the .csv template files live
     templates_dir = r'c:\temp\wi_batch_upload_templates'
-    
+
     # The folder to which you want to write WI-formatted .csv files
     output_base = r'c:\temp\wi_output'
-    
-    
+
+
     #%% Path validation
-    
+
     assert os.path.isfile(input_file)
     assert os.path.isfile(taxonomy_file)
     assert os.path.isdir(templates_dir)
     os.makedirs(output_base,exist_ok = True)
 
-        
+
     #%% Constants
 
     projects_file_name = 'Template Wildlife Insights Batch Upload - Projectv1.0.csv'
@@ -99,7 +99,7 @@ def main():
     project_info['project_sensor_cluster'] = 'No'
 
     camera_info = {}
-    camera_info['project_id'] = project_info['project_id'] 
+    camera_info['project_id'] = project_info['project_id']
     camera_info['camera_id'] = '0000'
     camera_info['make'] = ''
     camera_info['model'] = ''
@@ -108,7 +108,7 @@ def main():
 
     deployment_info = {}
 
-    deployment_info['project_id'] = project_info['project_id'] 
+    deployment_info['project_id'] = project_info['project_id']
     deployment_info['deployment_id'] = 'test_deployment'
     deployment_info['subproject_name'] = 'test_subproject'
     deployment_info['subproject_design'] = ''
@@ -140,7 +140,7 @@ def main():
     #%% Read templates
 
     def parse_fields(templates_dir,file_name):
-        
+
         with open(os.path.join(templates_dir,file_name),'r') as f:
             lines = f.readlines()
             lines = [s.strip() for s in lines if len(s.strip().replace(',','')) > 0]
@@ -158,7 +158,7 @@ def main():
     #%% Compare dictionary to template lists
 
     def compare_info_to_template(info,template_fields,name):
-        
+
         for s in info.keys():
             assert s in template_fields,'Field {} not specified in {}_fields'.format(s,name)
         for s in template_fields:
@@ -166,26 +166,26 @@ def main():
 
 
     def write_table(file_name,info,template_fields):
-        
+
         assert len(info) == len(template_fields)
-        
+
         project_output_file = os.path.join(output_base,file_name)
         with open(project_output_file,'w') as f:
-        
+
             # Write the header
             for i_field,s in enumerate(template_fields):
                 f.write(s)
                 if i_field != len(template_fields)-1:
                     f.write(',')
             f.write('\n')
-            
+
             # Write values
             for i_field,s in enumerate(template_fields):
                 f.write(info[s])
                 if i_field != len(template_fields)-1:
                     f.write(',')
             f.write('\n')
-        
+
 
     #%% Project file
 
@@ -214,7 +214,7 @@ def main():
     # Read taxonomy dictionary
     with open(taxonomy_file,'r') as f:
         taxonomy_mapping = json.load(f)
-        
+
     url_base = taxonomy_mapping['url_base']
     taxonomy_mapping = taxonomy_mapping['taxonomy']
 
@@ -226,7 +226,7 @@ def main():
     image_id_to_annotations = defaultdict(list)
 
     annotations = input_data['annotations']
-                            
+
     # annotation = annotations[0]
     for annotation in annotations:
         image_id_to_annotations[annotation['image_id']].append(
@@ -238,31 +238,31 @@ def main():
     for im in input_data['images']:
 
         row = {}
-        
+
         url = url_base + im['file_name'].replace('\\','/')
         row['project_id'] = project_info['project_id']
         row['deployment_id'] = deployment_info['deployment_id']
         row['image_id'] = im['id']
         row['location'] = url
         row['identified_by'] = image_info['identified_by']
-        
+
         category_names = image_id_to_annotations[im['id']]
         assert len(category_names) == 1
         category_name = category_names[0]
-        
+
         taxon_info = taxonomy_mapping[category_name]
-        
+
         assert len(taxon_info.keys()) == 7
-        
+
         for s in taxon_info.keys():
-            row[s] = taxon_info[s]    
-        
+            row[s] = taxon_info[s]
+
         # We don't have counts, but we can differentiate between zero and 1
         if category_name == 'empty':
             row['number_of_objects'] = 0
         else:
             row['number_of_objects'] = 1
-            
+
         row['uncertainty'] = None
         row['timestamp'] = im['datetime']; assert isinstance(im['datetime'],str)
         row['highlighted'] = 0
@@ -272,10 +272,10 @@ def main():
         row['individual_id'] = None
         row['individual_animal_notes'] = None
         row['markings'] = None
-        
+
         assert len(row) == len(images_fields)
         rows.append(row)
-        
+
     df = pd.DataFrame(rows)
 
     df.to_csv(os.path.join(output_base,images_file_name),index=False)

megadetector/data_management/coco_to_labelme.py

@@ -4,7 +4,7 @@ coco_to_labelme.py
 
 Converts a COCO dataset to labelme format (one .json per image file).
 
-If you want to convert YOLO-formatted data to labelme format, use yolo_to_coco, then 
+If you want to convert YOLO-formatted data to labelme format, use yolo_to_coco, then
 coco_to_labelme.
 
 """
@@ -13,34 +13,37 @@ coco_to_labelme.
 
 import os
 import json
+import sys
+import argparse
 
 from tqdm import tqdm
 from collections import defaultdict
 
 from megadetector.visualization.visualization_utils import open_image
 from megadetector.detection.run_detector import FAILURE_IMAGE_OPEN
+from megadetector.utils import ct_utils
 
 
 #%% Functions
 
 def get_labelme_dict_for_image_from_coco_record(im,annotations,categories,info=None):
     """
-    For the given image struct in COCO format and associated list of annotations, reformats the 
-    detections into labelme format.  
-    
+    For the given image struct in COCO format and associated list of annotations, reformats the
+    detections into labelme format.
+
     Args:
         im (dict): image dict, as loaded from a COCO .json file; 'height' and 'width' are required
-        annotations (list): a list of annotations that refer to this image (this function errors if 
+        annotations (list): a list of annotations that refer to this image (this function errors if
             that's not the case)
         categories (list): a list of category in dicts in COCO format ({'id':x,'name':'s'})
         info (dict, optional): a dict to store in a non-standard "custom_info"  field in the output
-        
+
     Returns:
         dict: a dict in labelme format, suitable for writing to a labelme .json file
     """
-    
+
     image_base_name = os.path.basename(im['file_name'])
-    
+
     output_dict = {}
     if info is not None:
         output_dict['custom_info'] = info
@@ -51,43 +54,43 @@ def get_labelme_dict_for_image_from_coco_record(im,annotations,categories,info=N
     output_dict['imageHeight'] = im['height']
     output_dict['imageWidth'] = im['width']
     output_dict['imageData'] = None
-    
+
     # Store COCO categories in case we want to reconstruct the original IDs later
     output_dict['coco_categories'] = categories
-    
+
     category_id_to_name = {c['id']:c['name'] for c in categories}
-    
+
     if 'flags' in im:
         output_dict['flags'] = im['flags']
-        
+
     # ann = annotations[0]
     for ann in annotations:
-        
+
         assert ann['image_id'] == im['id'], 'Annotation {} does not refer to image {}'.format(
             ann['id'],im['id'])
-        
+
         if 'bbox' not in ann:
             continue
-        
+
         shape = {}
-        shape['label'] = category_id_to_name[ann['category_id']] 
+        shape['label'] = category_id_to_name[ann['category_id']]
         shape['shape_type'] = 'rectangle'
         shape['description'] = ''
         shape['group_id'] = None
-        
+
         # COCO boxes are [x_min, y_min, width_of_box, height_of_box] (absolute)
-        # 
+        #
         # labelme boxes are [[x0,y0],[x1,y1]] (absolute)
         x0 = ann['bbox'][0]
         y0 = ann['bbox'][1]
         x1 = ann['bbox'][0] + ann['bbox'][2]
         y1 = ann['bbox'][1] + ann['bbox'][3]
-        
+
         shape['points'] = [[x0,y0],[x1,y1]]
         output_dict['shapes'].append(shape)
-    
+
     # ...for each detection
-    
+
     return output_dict
 
 # ...def get_labelme_dict_for_image()
@@ -95,9 +98,9 @@ def get_labelme_dict_for_image_from_coco_record(im,annotations,categories,info=N
 
 def coco_to_labelme(coco_data,image_base,overwrite=False,bypass_image_size_check=False,verbose=False):
     """
-    For all the images in [coco_data] (a dict or a filename), write a .json file in 
+    For all the images in [coco_data] (a dict or a filename), write a .json file in
     labelme format alongside the corresponding relative path within image_base.
-    
+
     Args:
         coco_data (dict or str): path to a COCO-formatted .json file, or an already-loaded
             COCO-formatted dict
@@ -105,40 +108,40 @@ def coco_to_labelme(coco_data,image_base,overwrite=False,bypass_image_size_check
             [image_base]); this is also where labelme files will be written
         overwrite (bool, optional): overwrite existing .json files
         bypass_image_size_check (bool, optional): if you're sure that the COCO data already has
-            correct 'width' and 'height' fields, this bypasses the somewhat-slow loading of 
+            correct 'width' and 'height' fields, this bypasses the somewhat-slow loading of
             each image to fetch image sizes
         verbose (bool, optional): enable additional debug output
     """
-    
+
     # Load COCO data if necessary
     if isinstance(coco_data,str):
         with open(coco_data,'r') as f:
             coco_data = json.load(f)
     assert isinstance(coco_data,dict)
-     
-    
+
+
     ## Read image sizes if necessary
-    
+
     if bypass_image_size_check:
-        
+
         print('Bypassing size check')
-        
+
     else:
-    
+
         # TODO: parallelize this loop
-        
+
         print('Reading/validating image sizes...')
-        
+
         # im = coco_data['images'][0]
         for im in tqdm(coco_data['images']):
-            
+
             # Make sure this file exists
             im_full_path = os.path.join(image_base,im['file_name'])
             assert os.path.isfile(im_full_path), 'Image file {} does not exist'.format(im_full_path)
-            
+
             # Load w/h information if necessary
             if 'height' not in im or 'width' not in im:
-                
+
                 try:
                     pil_im = open_image(im_full_path)
                     im['width'] = pil_im.width
@@ -147,32 +150,32 @@ def coco_to_labelme(coco_data,image_base,overwrite=False,bypass_image_size_check
                     print('Warning: cannot open image {}'.format(im_full_path))
                     if 'failure' not in im:
                         im['failure'] = FAILURE_IMAGE_OPEN
-    
+
             # ...if we need to read w/h information
-            
+
         # ...for each image
-    
+
     # ...if we need to load image sizes
-    
-    
+
+
     ## Generate labelme files
-    
+
     print('Generating .json files...')
-    
+
     image_id_to_annotations = defaultdict(list)
     for ann in coco_data['annotations']:
         image_id_to_annotations[ann['image_id']].append(ann)
-        
+
     n_json_files_written = 0
     n_json_files_error = 0
     n_json_files_exist = 0
-    
+
     # Write output
     for im in tqdm(coco_data['images']):
-        
+
         # Skip this image if it failed to load in whatever system generated this COCO file
         skip_image = False
-        
+
         # Errors are represented differently depending on the source
         for error_string in ('failure','error'):
             if (error_string in im) and (im[error_string] is not None):
@@ -184,89 +187,86 @@ def coco_to_labelme(coco_data,image_base,overwrite=False,bypass_image_size_check
                 break
         if skip_image:
             continue
-                    
+
         im_full_path = os.path.join(image_base,im['file_name'])
         json_path = os.path.splitext(im_full_path)[0] + '.json'
-        
+
         if (not overwrite) and (os.path.isfile(json_path)):
             if verbose:
                 print('Skipping existing file {}'.format(json_path))
             n_json_files_exist += 1
             continue
-    
+
         annotations_this_image = image_id_to_annotations[im['id']]
         output_dict = get_labelme_dict_for_image_from_coco_record(im,
                                                                   annotations_this_image,
                                                                   coco_data['categories'],
                                                                   info=None)
-        
+
         n_json_files_written += 1
-        with open(json_path,'w') as f:
-            json.dump(output_dict,f,indent=1)
-            
+        ct_utils.write_json(json_path, output_dict)
+
     # ...for each image
-    
+
     print('\nWrote {} .json files (skipped {} for errors, {} because they exist)'.format(
         n_json_files_written,n_json_files_error,n_json_files_exist))
-    
+
 # ...def coco_to_labelme()
 
 
 #%% Interactive driver
 
 if False:
-    
+
     pass
 
     #%% Configure options
-    
+
     coco_file = \
         r'C:\\temp\\snapshot-exploration\\images\\training-images-good\\training-images-good_from_yolo.json'
-    image_folder = os.path.dirname(coco_file)    
-    overwrite = True    
-    
-    
+    image_folder = os.path.dirname(coco_file)
+    overwrite = True
+
+
     #%% Programmatic execution
-    
+
     coco_to_labelme(coco_data=coco_file,image_base=image_folder,overwrite=overwrite)
 
-    
+
     #%% Command-line execution
-    
+
     s = 'python coco_to_labelme.py "{}" "{}"'.format(coco_file,image_folder)
     if overwrite:
         s += ' --overwrite'
-        
+
     print(s)
     import clipboard; clipboard.copy(s)
 
 
     #%% Opening labelme
-    
+
     s = 'python labelme {}'.format(image_folder)
     print(s)
     import clipboard; clipboard.copy(s)
-    
 
-#%% Command-line driver
 
-import sys,argparse
+#%% Command-line driver
 
-def main():
+def main(): # noqa
 
     parser = argparse.ArgumentParser(
         description='Convert a COCO database to labelme annotation format')
-    
+
     parser.add_argument(
         'coco_file',
         type=str,
         help='Path to COCO data file (.json)')
-    
+
     parser.add_argument(
         'image_base',
         type=str,
         help='Path to images (also the output folder)')
-    
+
     parser.add_argument(
         '--overwrite',
         action='store_true',
@@ -278,7 +278,7 @@ def main():
 
     args = parser.parse_args()
 
-    coco_to_labelme(coco_data=args.coco_file,image_base=args.image_base,overwrite=args.overwrite)    
+    coco_to_labelme(coco_data=args.coco_file,image_base=args.image_base,overwrite=args.overwrite)
 
 if __name__ == '__main__':
     main()