megadetector 5.0.8__py3-none-any.whl → 5.0.10__py3-none-any.whl

data_management/labelme_to_coco.py

@@ -1,10 +1,10 @@
-########
-#
-# labelme_to_coco.py
-#
-# Converts a folder of labelme-formatted .json files to COCO format.
-#
-########
+"""
+
+labelme_to_coco.py
+
+Converts a folder of labelme-formatted .json files to COCO.
+
+"""
 
 #%% Constants and imports
 
@@ -23,10 +23,10 @@ from tqdm import tqdm
 
 #%% Support functions
 
-def add_category(category_name,category_name_to_id,candidate_category_id=0):
+def _add_category(category_name,category_name_to_id,candidate_category_id=0):
     """
-    Add the category [category_name] to the dict [category_name_to_id], by default
-    using the next available integer index.
+    Adds the category [category_name] to the dict [category_name_to_id], by default
+    using the next available integer index.    
     """
     
     if category_name in category_name_to_id:
@@ -121,7 +121,7 @@ def _process_labelme_file(image_fn_relative,input_folder,use_folders_as_labels,
     if len(shapes) == 0:
         
         if allow_new_categories:
-            category_id = add_category('empty',category_name_to_id)
+            category_id = _add_category('empty',category_name_to_id)
         else:
             assert 'empty' in category_name_to_id
             category_id = category_name_to_id['empty']
@@ -148,7 +148,7 @@ def _process_labelme_file(image_fn_relative,input_folder,use_folders_as_labels,
                 category_name = shape['label']                
             
             if allow_new_categories:
-                category_id = add_category(category_name,category_name_to_id)
+                category_id = _add_category(category_name,category_name_to_id)
             else:
                 assert category_name in category_name_to_id
                 category_id = category_name_to_id[category_name]
@@ -202,7 +202,7 @@ def labelme_to_coco(input_folder,
                     max_workers=1, 
                     use_threads=True):
     """
-    Find all images in [input_folder] that have corresponding .json files, and convert
+    Finds all images in [input_folder] that have corresponding .json files, and converts
     to a COCO .json file.
     
     Currently only supports bounding box annotations and image-level flags (i.e., does not
@@ -224,11 +224,38 @@ def labelme_to_coco(input_folder,
     file.  Empty images in the "lion" folder will still be given the label "empty" (or 
     [empty_category_name]).
     
-    no_json_handling can be:
+    Args:
+        input_folder (str): input folder to search for images and Labelme .json files
+        output_file (str, optional): output file to which we should write COCO-formatted data; if None
+            this function just returns the COCO-formatted dict
+        category_id_to_category_name (dict, optional): dict mapping category IDs to category names;
+            really used to map Labelme category names to COCO category IDs.  IDs will be auto-generated
+            if this is None.
+        empty_category_id (int, optional): category ID to use for the not-very-COCO-like "empty" category;
+            also see the no_json_handling parameter.
+        info_struct (dict, optional): dict to stash in the "info" field of the resulting COCO dict
+        relative_paths_to_include (list, optional): allowlist of relative paths to include in the COCO
+            dict; there's no reason to specify this along with relative_paths_to_exclude.
+        relative_paths_to_exclude (list, optional): blocklist of relative paths to exclude from the COCO
+            dict; there's no reason to specify this along with relative_paths_to_include.
+        use_folders_as_labels (bool, optional): if this is True, class names will be pulled from folder names,
+            useful if you have images like a/b/cat/image001.jpg, a/b/dog/image002.jpg, etc.
+        recursive (bool, optional): whether to recurse into [input_folder]
+        no_json_handling (str, optional): how to deal with image files that have no corresponding .json files, 
+            can be:
+                
+                - 'skip': ignore image files with no corresponding .json files
+                - 'empty': treat image files with no corresponding .json files as empty
+                - 'error': throw an error when an image file has no corresponding .json file
+        validate_image_sizes (bool, optional): whether to load images to verify that the sizes specified
+            in the labelme files are correct
+        max_workers (int, optional): number of workers to use for parallelization, set to <=1 to disable
+            parallelization
+        use_threads (bool, optional): whether to use threads (True) or processes (False) for parallelization,
+            not relevant if max_workers <= 1
         
-    * 'skip': ignore image files with no corresponding .json files
-    * 'empty': treat image files with no corresponding .json files as empty
-    * 'error': throw an error when an image file has no corresponding .json file    
+    Returns:
+        dict: a COCO-formatted dictionary, identical to what's written to [output_file] if [output_file] is not None.
     """
     
     if max_workers > 1:
@@ -288,7 +315,7 @@ def labelme_to_coco(input_folder,
             raise ValueError('Category IDs must be ints or string-formatted ints')
         
     if empty_category_id is None:
-        empty_category_id = add_category(empty_category_name,category_name_to_id)
+        empty_category_id = _add_category(empty_category_name,category_name_to_id)
             
     if max_workers <= 1:
         
@@ -366,12 +393,26 @@ def find_empty_labelme_files(input_folder,recursive=True):
     Returns a list of all image files in in [input_folder] associated with .json files that have 
     no boxes in them.  Also returns a list of images with no associated .json files.  Specifically,
     returns a dict:
-        
-    {
-       'images_with_empty_json_files':[list],
-       'images_with_no_json_files':[list],
-       'images_with_non_empty_json_files':[list]
-    }    
+    
+    .. code-block: none
+    
+        {
+            'images_with_empty_json_files':[list],
+            'images_with_no_json_files':[list],
+            'images_with_non_empty_json_files':[list]
+        }
+    
+    Args:
+        input_folder (str): the folder to search for empty (i.e., box-less) Labelme .json files
+        recursive (bool, optional): whether to recurse into [input_folder]
+    
+    Returns:
+        dict: a dict with fields:
+            - images_with_empty_json_files: a list of all image files in [input_folder] associated with 
+              .json files that have no boxes in them
+            - images_with_no_json_files: a list of images in [input_folder] with no associated .json files
+            - images_with_non_empty_json_files: a list of images in [input_folder] associated with .json
+              files that have at least one box        
     """
     image_filenames_relative = path_utils.find_images(input_folder,recursive=True,
                                                       return_relative_paths=True)

data_management/labelme_to_yolo.py

@@ -1,10 +1,10 @@
-########
-#
-# labelme_to_yolo.py
-#
-# Create YOLO .txt files in a folder containing labelme .json files.
-#
-########
+"""
+
+labelme_to_yolo.py
+
+Create YOLO .txt files in a folder containing labelme .json files.
+
+"""
 
 #%% Imports
 
@@ -77,7 +77,7 @@ def labelme_file_to_yolo_file(labelme_file,
         p0 = shape['points'][0]
         p1 = shape['points'][1]
         
-        # LabelMe: [[x0,y0],[x1,y1]] (arbitrarily sorted) (absolute coordinates)
+        # Labelme: [[x0,y0],[x1,y1]] (arbitrarily sorted) (absolute coordinates)
         #
         # YOLO: [class, x_center, y_center, width, height] (normalized coordinates)
         minx_abs = min(p0[0],p1[0])

data_management/lila/add_locations_to_island_camera_traps.py

@@ -1,12 +1,12 @@
-########
-#
-# add_locations_to_island_camera_traps.py
-#
-# The Island Conservation Camera Traps dataset had unique camera identifiers embedded
-# in filenames, but not in the proper metadata fields.  This script copies that information
-# to metadata.
-#
-########
+"""
+
+add_locations_to_island_camera_traps.py
+
+The Island Conservation Camera Traps dataset had unique camera identifiers embedded
+in filenames, but not in the proper metadata fields.  This script copies that information
+to metadata.
+
+"""
 
 #%% Imports and constants
 

data_management/lila/add_locations_to_nacti.py

@@ -1,147 +1,147 @@
-########
-#
-# add_locations_to_nacti.py
-#
-# As of 10.2023, NACTI metadata only has very coarse location information (e.g. "Florida"),
-# but camera IDs are embedded in filenames.  This script pulls that information from filenames
-# and adds it to metadata.
-#
-########
-
-#%% Imports and constants
-
-import os
-import json
-import shutil
-
-from tqdm import tqdm
-from collections import defaultdict
-
-input_file = r'd:\lila\nacti\nacti_metadata.json.1.13\nacti_metadata.json'
-output_file = r'g:\temp\nacti_metadata.1.14.json'
-
-
-#%% Read metadata
-
-with open(input_file,'r') as f:
-    d = json.load(f)
-    
-assert d['info']['version'] == 1.13
-
-
-#%% Map images to locations (according to the metadata)
-
-file_name_to_original_location = {}
-
-# im = dataset_labels['images'][0]
-for im in tqdm(d['images']):
-    file_name_to_original_location[im['file_name']] = im['location']
-
-original_locations = set(file_name_to_original_location.values())
-
-print('Found {} locations in the original metadata:'.format(len(original_locations)))
-for loc in original_locations:
-    print('[{}]'.format(loc))
-
-
-#%% Map images to new locations
-
-def path_to_location(relative_path):
-    
-    relative_path = relative_path.replace('\\','/')
-    if relative_path in file_name_to_original_location:
-        location_name = file_name_to_original_location[relative_path]
-        if location_name == 'San Juan Mntns, Colorado':
-            # "part0/sub000/2010_Unit150_Ivan097_img0003.jpg"
-            tokens = relative_path.split('/')[-1].split('_')
-            assert tokens[1].startswith('Unit')
-            location_name = 'sanjuan_{}_{}_{}'.format(tokens[0],tokens[1],tokens[2])
-        elif location_name == 'Lebec, California':
-            # "part0/sub035/CA-03_08_13_2015_CA-03_0009738.jpg"
-            tokens = relative_path.split('/')[-1].split('_')
-            assert tokens[0].startswith('CA-') or tokens[0].startswith('TAG-')
-            location_name = 'lebec_{}'.format(tokens[0])   
-        elif location_name == 'Archbold, FL':
-            # "part1/sub110/FL-01_01_25_2016_FL-01_0040421.jpg"
-            tokens = relative_path.split('/')[-1].split('_')
-            assert tokens[0].startswith('FL-')
-            location_name = 'archbold_{}'.format(tokens[0])   
-        else:
-            assert location_name == ''
-            tokens = relative_path.split('/')[-1].split('_')
-            if tokens[0].startswith('CA-') or tokens[0].startswith('TAG-') or tokens[0].startswith('FL-'):
-                location_name = '{}'.format(tokens[0])
-            
-    else:
-        
-        location_name = 'unknown'    
-    
-    # print('Returning location {} for file {}'.format(location_name,relative_path))          
-
-    return location_name
-
-file_name_to_updated_location = {}
-updated_location_to_count = defaultdict(int)
-for im in tqdm(d['images']):
-    
-    updated_location = path_to_location(im['file_name'])
-    file_name_to_updated_location[im['file_name']] = updated_location
-    updated_location_to_count[updated_location] += 1
-
-updated_location_to_count = {k: v for k, v in sorted(updated_location_to_count.items(), 
-                         key=lambda item: item[1],
-                         reverse=True)}
-
-updated_locations = set(file_name_to_updated_location.values())
-
-print('Found {} updated locations in the original metadata:'.format(len(updated_locations)))
-for loc in updated_location_to_count:
-    print('{}: {}'.format(loc,updated_location_to_count[loc]))
-
-
-#%% Re-write metadata
-
-for im in d['images']:
-    im['location'] = file_name_to_updated_location[im['file_name']]
-d['info']['version'] = 1.14
-
-with open(output_file,'w') as f:
-    json.dump(d,f,indent=1)
-    
-
-#%% For each location, sample some random images to make sure they look consistent
-
-input_base = r'd:\lila\nacti-unzipped'
-assert os.path.isdir(input_base)
-
-location_to_images = defaultdict(list)
-
-for im in d['images']:
-    location_to_images[im['location']].append(im)
-    
-n_to_sample = 10
-import random
-random.seed(0)
-sampling_folder_base = r'g:\temp\nacti_samples'
-
-for location in tqdm(location_to_images):
-    
-    images_this_location = location_to_images[location]
-    if len(images_this_location) > n_to_sample:
-        images_this_location = random.sample(images_this_location,n_to_sample)
-        
-    for i_image,im in enumerate(images_this_location):
-        
-        fn_relative = im['file_name']
-        source_fn_abs = os.path.join(input_base,fn_relative)
-        assert os.path.isfile(source_fn_abs)
-        ext = os.path.splitext(fn_relative)[1]            
-        target_fn_abs = os.path.join(sampling_folder_base,'{}/{}'.format(
-            location,'image_{}{}'.format(str(i_image).zfill(2),ext)))
-        os.makedirs(os.path.dirname(target_fn_abs),exist_ok=True)
-        shutil.copyfile(source_fn_abs,target_fn_abs)
-        
-    # ...for each image
-    
-# ...for each location
-            
+"""
+
+add_locations_to_nacti.py
+
+As of 10.2023, NACTI metadata only has very coarse location information (e.g. "Florida"),
+but camera IDs are embedded in filenames.  This script pulls that information from filenames
+and adds it to metadata.
+
+"""
+
+#%% Imports and constants
+
+import os
+import json
+import shutil
+
+from tqdm import tqdm
+from collections import defaultdict
+
+input_file = r'd:\lila\nacti\nacti_metadata.json.1.13\nacti_metadata.json'
+output_file = r'g:\temp\nacti_metadata.1.14.json'
+
+
+#%% Read metadata
+
+with open(input_file,'r') as f:
+    d = json.load(f)
+    
+assert d['info']['version'] == 1.13
+
+
+#%% Map images to locations (according to the metadata)
+
+file_name_to_original_location = {}
+
+# im = dataset_labels['images'][0]
+for im in tqdm(d['images']):
+    file_name_to_original_location[im['file_name']] = im['location']
+
+original_locations = set(file_name_to_original_location.values())
+
+print('Found {} locations in the original metadata:'.format(len(original_locations)))
+for loc in original_locations:
+    print('[{}]'.format(loc))
+
+
+#%% Map images to new locations
+
+def path_to_location(relative_path):
+    
+    relative_path = relative_path.replace('\\','/')
+    if relative_path in file_name_to_original_location:
+        location_name = file_name_to_original_location[relative_path]
+        if location_name == 'San Juan Mntns, Colorado':
+            # "part0/sub000/2010_Unit150_Ivan097_img0003.jpg"
+            tokens = relative_path.split('/')[-1].split('_')
+            assert tokens[1].startswith('Unit')
+            location_name = 'sanjuan_{}_{}_{}'.format(tokens[0],tokens[1],tokens[2])
+        elif location_name == 'Lebec, California':
+            # "part0/sub035/CA-03_08_13_2015_CA-03_0009738.jpg"
+            tokens = relative_path.split('/')[-1].split('_')
+            assert tokens[0].startswith('CA-') or tokens[0].startswith('TAG-')
+            location_name = 'lebec_{}'.format(tokens[0])   
+        elif location_name == 'Archbold, FL':
+            # "part1/sub110/FL-01_01_25_2016_FL-01_0040421.jpg"
+            tokens = relative_path.split('/')[-1].split('_')
+            assert tokens[0].startswith('FL-')
+            location_name = 'archbold_{}'.format(tokens[0])   
+        else:
+            assert location_name == ''
+            tokens = relative_path.split('/')[-1].split('_')
+            if tokens[0].startswith('CA-') or tokens[0].startswith('TAG-') or tokens[0].startswith('FL-'):
+                location_name = '{}'.format(tokens[0])
+            
+    else:
+        
+        location_name = 'unknown'    
+    
+    # print('Returning location {} for file {}'.format(location_name,relative_path))          
+
+    return location_name
+
+file_name_to_updated_location = {}
+updated_location_to_count = defaultdict(int)
+for im in tqdm(d['images']):
+    
+    updated_location = path_to_location(im['file_name'])
+    file_name_to_updated_location[im['file_name']] = updated_location
+    updated_location_to_count[updated_location] += 1
+
+updated_location_to_count = {k: v for k, v in sorted(updated_location_to_count.items(), 
+                         key=lambda item: item[1],
+                         reverse=True)}
+
+updated_locations = set(file_name_to_updated_location.values())
+
+print('Found {} updated locations in the original metadata:'.format(len(updated_locations)))
+for loc in updated_location_to_count:
+    print('{}: {}'.format(loc,updated_location_to_count[loc]))
+
+
+#%% Re-write metadata
+
+for im in d['images']:
+    im['location'] = file_name_to_updated_location[im['file_name']]
+d['info']['version'] = 1.14
+
+with open(output_file,'w') as f:
+    json.dump(d,f,indent=1)
+    
+
+#%% For each location, sample some random images to make sure they look consistent
+
+input_base = r'd:\lila\nacti-unzipped'
+assert os.path.isdir(input_base)
+
+location_to_images = defaultdict(list)
+
+for im in d['images']:
+    location_to_images[im['location']].append(im)
+    
+n_to_sample = 10
+import random
+random.seed(0)
+sampling_folder_base = r'g:\temp\nacti_samples'
+
+for location in tqdm(location_to_images):
+    
+    images_this_location = location_to_images[location]
+    if len(images_this_location) > n_to_sample:
+        images_this_location = random.sample(images_this_location,n_to_sample)
+        
+    for i_image,im in enumerate(images_this_location):
+        
+        fn_relative = im['file_name']
+        source_fn_abs = os.path.join(input_base,fn_relative)
+        assert os.path.isfile(source_fn_abs)
+        ext = os.path.splitext(fn_relative)[1]            
+        target_fn_abs = os.path.join(sampling_folder_base,'{}/{}'.format(
+            location,'image_{}{}'.format(str(i_image).zfill(2),ext)))
+        os.makedirs(os.path.dirname(target_fn_abs),exist_ok=True)
+        shutil.copyfile(source_fn_abs,target_fn_abs)
+        
+    # ...for each image
+    
+# ...for each location
+            

data_management/lila/create_lila_blank_set.py

@@ -1,16 +1,16 @@
-########
-#
-# create_lila_blank_set.py
-#
-# Create a folder of blank images sampled from LILA.  We'll aim for diversity, so less-common
-# locations will be oversampled relative to more common locations.  We'll also run MegaDetector
-# (with manual review) to remove some incorrectly-labeled, not-actually-empty images from our 
-# blank set.
-#
-# We'll store location information for each image in a .json file, so we can split locations
-# into train/val in downstream tasks.
-#
-########
+"""
+
+create_lila_blank_set.py
+
+Create a folder of blank images sampled from LILA.  We'll aim for diversity, so less-common
+locations will be oversampled relative to more common locations.  We'll also run MegaDetector
+(with manual review) to remove some incorrectly-labeled, not-actually-empty images from our 
+blank set.
+
+We'll store location information for each image in a .json file, so we can split locations
+into train/val in downstream tasks.
+
+"""
 
 #%% Constants and imports
 

data_management/lila/create_lila_test_set.py

@@ -1,11 +1,11 @@
-########
-#
-# create_lila_test_set.py
-#
-# Create a test set of camera trap images, containing N empty and N non-empty 
-# images from each LILA data set.
-#
-########
+"""
+
+create_lila_test_set.py
+
+Create a test set of camera trap images, containing N empty and N non-empty 
+images from each LILA data set.
+
+"""
 
 #%% Constants and imports