megadetector 5.0.5__py3-none-any.whl → 5.0.7__py3-none-any.whl

data_management/labelme_to_yolo.py

@@ -0,0 +1,229 @@
+########
+#
+# labelme_to_yolo.py
+#
+# Create YOLO .txt files in a folder containing labelme .json files.
+#
+########
+
+#%% Imports
+
+import os
+import json
+
+from md_utils.path_utils import recursive_file_list
+from tqdm import tqdm
+
+
+#%% Main function
+
+def labelme_file_to_yolo_file(labelme_file,
+                              category_name_to_category_id,
+                              yolo_file=None,
+                              required_token=None,
+                              right_edge_quantization_threshold=None,
+                              overwrite_behavior='overwrite'):
+    """
+    Convert the single .json file labelme_file to yolo format, writing the results to the text
+    file yolo_file (defaults to s/json/txt).
+    
+    If required_token is not None and the labelme_file does not contain the key [required_token],
+    no-ops.
+    
+    right_edge_quantization_threshold is an off-by-default hack to handle cases where 
+    boxes that really should be running off the right side of the image only extend like 99%
+    of the way there, due to what appears to be a slight bias inherent to MD.  If a box extends
+    within [right_edge_quantization_threshold] (a small number, from 0 to 1, but probably around 
+    0.02) of the right edge of the image, it will be extended to the far right edge.    
+    """
+    
+    assert os.path.isfile(labelme_file), 'Could not find labelme .json file {}'.format(labelme_file)
+    assert labelme_file.endswith('.json'), 'Illegal labelme .json file {}'.format(labelme_file)
+    
+    if yolo_file is None:
+        yolo_file = os.path.splitext(labelme_file)[0] + '.txt'
+    
+    if os.path.isfile(yolo_file):
+        if overwrite_behavior == 'skip':
+            return
+        else:
+            assert overwrite_behavior == 'overwrite', \
+                'Unrecognized overwrite behavior {}'.format(overwrite_behavior)
+                
+    with open(labelme_file,'r') as f:
+        labelme_data = json.load(f)
+        
+    if required_token is not None and required_token not in labelme_data:
+        return
+    
+    im_height = labelme_data['imageHeight']
+    im_width = labelme_data['imageWidth']
+    
+    yolo_lines = []
+    
+    for shape in labelme_data['shapes']:
+        
+        assert shape['shape_type'] == 'rectangle', \
+            'I only know how to convert rectangles to YOLO format'
+        assert shape['label'] in category_name_to_category_id, \
+            'Category {} not in category mapping'.format(shape['label'])
+        assert len(shape['points']) == 2, 'Illegal rectangle'
+        category_id = category_name_to_category_id[shape['label']]
+        
+        p0 = shape['points'][0]
+        p1 = shape['points'][1]
+        
+        # LabelMe: [[x0,y0],[x1,y1]] (arbitrarily sorted) (absolute coordinates)
+        #
+        # YOLO: [class, x_center, y_center, width, height] (normalized coordinates)
+        minx_abs = min(p0[0],p1[0])
+        maxx_abs = max(p0[0],p1[0])
+        miny_abs = min(p0[1],p1[1])
+        maxy_abs = max(p0[1],p1[1])
+        
+        if (minx_abs >= (im_width-1)) or (maxx_abs <= 0) or \
+            (miny_abs >= (im_height-1)) or (maxy_abs <= 0):
+                print('Skipping invalid shape in {}'.format(labelme_file))
+                continue
+            
+        # Clip to [0,1]
+        maxx_abs = min(maxx_abs,im_width-1)
+        maxy_abs = min(maxy_abs,im_height-1)
+        minx_abs = max(minx_abs,0.0)
+        miny_abs = max(miny_abs,0.0)
+        
+        minx_rel = minx_abs / (im_width-1)
+        maxx_rel = maxx_abs / (im_width-1)
+        miny_rel = miny_abs / (im_height-1)
+        maxy_rel = maxy_abs / (im_height-1)
+        
+        if (right_edge_quantization_threshold is not None):
+            right_edge_distance = 1.0 - maxx_rel
+            if right_edge_distance < right_edge_quantization_threshold:
+                maxx_rel = 1.0
+        
+        assert maxx_rel >= minx_rel
+        assert maxy_rel >= miny_rel
+        
+        xcenter_rel = (maxx_rel + minx_rel) / 2.0
+        ycenter_rel = (maxy_rel + miny_rel) / 2.0
+        w_rel = maxx_rel - minx_rel
+        h_rel = maxy_rel - miny_rel
+        
+        yolo_line = '{} {:.3f} {:.3f} {:.3f} {:.3f}'.format(category_id,
+            xcenter_rel, ycenter_rel, w_rel, h_rel)
+        yolo_lines.append(yolo_line)
+        
+    # ...for each shape
+    
+    with open(yolo_file,'w') as f:
+        for s in yolo_lines:
+            f.write(s + '\n')
+        
+
+def labelme_folder_to_yolo(labelme_folder,
+                           category_name_to_category_id=None,
+                           required_token=None,
+                           right_edge_quantization_threshold=None,
+                           overwrite_behavior='overwrite'):
+    """
+    Given a folder with images and labelme .json files, convert the .json files
+    to YOLO .txt format.  If category_name_to_category_id is None, first reads
+    all the labels in the folder to build a zero-indexed name --> ID mapping.
+    
+    If required_token is not None and a labelme_file does not contain the key [required_token],
+    it won't be converted.
+    
+    right_edge_quantization_threshold is an off-by-default hack to handle cases where 
+    boxes that really should be running off the right side of the image only extend like 99%
+    of the way there, due to what appears to be a slight bias inherent to MD.  If a box extends
+    within [right_edge_quantization_threshold] (a small number, from 0 to 1, but probably around 
+    0.02) of the right edge of the image, it will be extended to the far right edge.    
+    
+    returns category_name_to_category_id, whether it was passed in or constructed.
+    """
+    
+    labelme_files_relative = recursive_file_list(labelme_folder,return_relative_paths=True)
+    labelme_files_relative = [fn for fn in labelme_files_relative if fn.endswith('.json')]
+    
+    if required_token is None:
+        valid_labelme_files_relative = labelme_files_relative
+    else:        
+        valid_labelme_files_relative = []
+        
+        # fn_relative = labelme_files_relative[-1]
+        for fn_relative in labelme_files_relative:
+                        
+            fn_abs = os.path.join(labelme_folder,fn_relative)            
+            
+            with open(fn_abs,'r') as f:
+                labelme_data = json.load(f)
+                if required_token not in labelme_data:
+                    continue
+                
+            valid_labelme_files_relative.append(fn_relative)
+    
+    print('{} of {} files are valid'.format(len(valid_labelme_files_relative),
+                                           len(labelme_files_relative)))
+    
+    del labelme_files_relative
+        
+    if category_name_to_category_id is None:
+        
+        category_name_to_category_id = {}
+        
+        for fn_relative in valid_labelme_files_relative:
+            
+            fn_abs = os.path.join(labelme_folder,fn_relative)
+            with open(fn_abs,'r') as f:
+                labelme_data = json.load(f)
+                for shape in labelme_data['shapes']:
+                    label = shape['label']
+                    if label not in category_name_to_category_id:
+                        category_name_to_category_id[label] = len(category_name_to_category_id)
+        # ...for each file
+        
+    # ...if we need to build a category mapping
+            
+    for fn_relative in tqdm(valid_labelme_files_relative):
+        
+        fn_abs = os.path.join(labelme_folder,fn_relative)
+        labelme_file_to_yolo_file(fn_abs,
+                                  category_name_to_category_id,
+                                  yolo_file=None,
+                                  required_token=required_token,
+                                  right_edge_quantization_threshold=\
+                                      right_edge_quantization_threshold,
+                                  overwrite_behavior=overwrite_behavior)
+    
+    # ...for each file
+    
+    print('Converted {} labelme .json files to YOLO'.format(
+        len(valid_labelme_files_relative)))
+    
+    return category_name_to_category_id
+    
+        
+#%% Interactive driver
+
+if False:
+
+    pass
+
+    #%%
+
+    import os
+    labelme_file = os.path.expanduser('~/tmp/labels/x.json')
+    yolo_file = None
+    required_token = 'saved_by_labelme'
+    right_edge_quantization_threshold = 0.015
+    category_name_to_category_id = {'animal':0}
+
+    #%%
+    
+    labelme_folder = os.path.expanduser('~/tmp/labels')
+
+
+#%% Command-line driver
+
+# TODO

data_management/lila/add_locations_to_island_camera_traps.py

@@ -80,7 +80,7 @@ viz_options.add_search_links = False
 viz_options.sort_by_filename = False
 viz_options.parallelize_rendering = True
 viz_options.classes_to_exclude = ['test']
-html_output_file, image_db = visualize_db.process_images(db_path=output_fn,
+html_output_file, image_db = visualize_db.visualize_db(db_path=output_fn,
                                                          output_dir=preview_folder,
                                                          image_base_dir=image_directory,
                                                          options=viz_options)

data_management/lila/add_locations_to_nacti.py

@@ -0,0 +1,147 @@
+########
+#
+# add_locations_to_nacti.py
+#
+# As of 10.2023, NACTI metadata only has very coarse location information (e.g. "Florida"),
+# but camera IDs are embedded in filenames.  This script pulls that information from filenames
+# and adds it to metadata.
+#
+########
+
+#%% Imports and constants
+
+import os
+import json
+import shutil
+
+from tqdm import tqdm
+from collections import defaultdict
+
+input_file = r'd:\lila\nacti\nacti_metadata.json.1.13\nacti_metadata.json'
+output_file = r'g:\temp\nacti_metadata.1.14.json'
+
+
+#%% Read metadata
+
+with open(input_file,'r') as f:
+    d = json.load(f)
+    
+assert d['info']['version'] == 1.13
+
+
+#%% Map images to locations (according to the metadata)
+
+file_name_to_original_location = {}
+
+# im = dataset_labels['images'][0]
+for im in tqdm(d['images']):
+    file_name_to_original_location[im['file_name']] = im['location']
+
+original_locations = set(file_name_to_original_location.values())
+
+print('Found {} locations in the original metadata:'.format(len(original_locations)))
+for loc in original_locations:
+    print('[{}]'.format(loc))
+
+
+#%% Map images to new locations
+
+def path_to_location(relative_path):
+    
+    relative_path = relative_path.replace('\\','/')
+    if relative_path in file_name_to_original_location:
+        location_name = file_name_to_original_location[relative_path]
+        if location_name == 'San Juan Mntns, Colorado':
+            # "part0/sub000/2010_Unit150_Ivan097_img0003.jpg"
+            tokens = relative_path.split('/')[-1].split('_')
+            assert tokens[1].startswith('Unit')
+            location_name = 'sanjuan_{}_{}_{}'.format(tokens[0],tokens[1],tokens[2])
+        elif location_name == 'Lebec, California':
+            # "part0/sub035/CA-03_08_13_2015_CA-03_0009738.jpg"
+            tokens = relative_path.split('/')[-1].split('_')
+            assert tokens[0].startswith('CA-') or tokens[0].startswith('TAG-')
+            location_name = 'lebec_{}'.format(tokens[0])   
+        elif location_name == 'Archbold, FL':
+            # "part1/sub110/FL-01_01_25_2016_FL-01_0040421.jpg"
+            tokens = relative_path.split('/')[-1].split('_')
+            assert tokens[0].startswith('FL-')
+            location_name = 'archbold_{}'.format(tokens[0])   
+        else:
+            assert location_name == ''
+            tokens = relative_path.split('/')[-1].split('_')
+            if tokens[0].startswith('CA-') or tokens[0].startswith('TAG-') or tokens[0].startswith('FL-'):
+                location_name = '{}'.format(tokens[0])
+            
+    else:
+        
+        location_name = 'unknown'    
+    
+    # print('Returning location {} for file {}'.format(location_name,relative_path))          
+
+    return location_name
+
+file_name_to_updated_location = {}
+updated_location_to_count = defaultdict(int)
+for im in tqdm(d['images']):
+    
+    updated_location = path_to_location(im['file_name'])
+    file_name_to_updated_location[im['file_name']] = updated_location
+    updated_location_to_count[updated_location] += 1
+
+updated_location_to_count = {k: v for k, v in sorted(updated_location_to_count.items(), 
+                         key=lambda item: item[1],
+                         reverse=True)}
+
+updated_locations = set(file_name_to_updated_location.values())
+
+print('Found {} updated locations in the original metadata:'.format(len(updated_locations)))
+for loc in updated_location_to_count:
+    print('{}: {}'.format(loc,updated_location_to_count[loc]))
+
+
+#%% Re-write metadata
+
+for im in d['images']:
+    im['location'] = file_name_to_updated_location[im['file_name']]
+d['info']['version'] = 1.14
+
+with open(output_file,'w') as f:
+    json.dump(d,f,indent=1)
+    
+
+#%% For each location, sample some random images to make sure they look consistent
+
+input_base = r'd:\lila\nacti-unzipped'
+assert os.path.isdir(input_base)
+
+location_to_images = defaultdict(list)
+
+for im in d['images']:
+    location_to_images[im['location']].append(im)
+    
+n_to_sample = 10
+import random
+random.seed(0)
+sampling_folder_base = r'g:\temp\nacti_samples'
+
+for location in tqdm(location_to_images):
+    
+    images_this_location = location_to_images[location]
+    if len(images_this_location) > n_to_sample:
+        images_this_location = random.sample(images_this_location,n_to_sample)
+        
+    for i_image,im in enumerate(images_this_location):
+        
+        fn_relative = im['file_name']
+        source_fn_abs = os.path.join(input_base,fn_relative)
+        assert os.path.isfile(source_fn_abs)
+        ext = os.path.splitext(fn_relative)[1]            
+        target_fn_abs = os.path.join(sampling_folder_base,'{}/{}'.format(
+            location,'image_{}{}'.format(str(i_image).zfill(2),ext)))
+        os.makedirs(os.path.dirname(target_fn_abs),exist_ok=True)
+        shutil.copyfile(source_fn_abs,target_fn_abs)
+        
+    # ...for each image
+    
+# ...for each location
+