megadetector 5.0.10__py3-none-any.whl → 5.0.11__py3-none-any.whl

data_management/labelme_to_yolo.py

@@ -1,272 +0,0 @@
-"""
-
-labelme_to_yolo.py
-
-Create YOLO .txt files in a folder containing labelme .json files.
-
-"""
-
-#%% Imports
-
-import os
-import json
-
-from multiprocessing.pool import Pool, ThreadPool
-from functools import partial
-
-from md_utils.path_utils import recursive_file_list
-from tqdm import tqdm
-
-
-#%% Main function
-
-def labelme_file_to_yolo_file(labelme_file,
-                              category_name_to_category_id,
-                              yolo_file=None,
-                              required_token=None,
-                              overwrite_behavior='overwrite'):
-    """
-    Convert the single .json file labelme_file to yolo format, writing the results to the text
-    file yolo_file (defaults to s/json/txt).
-    
-    If required_token is not None and the dict in labelme_file does not contain the key [required_token],
-    this function no-ops (i.e., does not generate a YOLO file).    
-    
-    overwrite_behavior should be 'skip' or 'overwrite' (default).
-    """
-    
-    result = {}
-    result['labelme_file'] = labelme_file
-    result['status'] = 'unknown'
-    
-    assert os.path.isfile(labelme_file), 'Could not find labelme .json file {}'.format(labelme_file)
-    assert labelme_file.endswith('.json'), 'Illegal labelme .json file {}'.format(labelme_file)
-    
-    if yolo_file is None:
-        yolo_file = os.path.splitext(labelme_file)[0] + '.txt'
-    
-    if os.path.isfile(yolo_file):
-        if overwrite_behavior == 'skip':
-            result['status'] = 'skip-exists'
-            return result
-        else:
-            assert overwrite_behavior == 'overwrite', \
-                'Unrecognized overwrite behavior {}'.format(overwrite_behavior)
-                
-    with open(labelme_file,'r') as f:
-        labelme_data = json.load(f)
-        
-    if required_token is not None and required_token not in labelme_data:
-        result['status'] = 'skip-no-required-token'
-        return result
-    
-    im_height = labelme_data['imageHeight']
-    im_width = labelme_data['imageWidth']
-    
-    yolo_lines = []
-    
-    for shape in labelme_data['shapes']:
-        
-        assert shape['shape_type'] == 'rectangle', \
-            'I only know how to convert rectangles to YOLO format'
-        assert shape['label'] in category_name_to_category_id, \
-            'Category {} not in category mapping'.format(shape['label'])
-        assert len(shape['points']) == 2, 'Illegal rectangle'
-        category_id = category_name_to_category_id[shape['label']]
-        
-        p0 = shape['points'][0]
-        p1 = shape['points'][1]
-        
-        # Labelme: [[x0,y0],[x1,y1]] (arbitrarily sorted) (absolute coordinates)
-        #
-        # YOLO: [class, x_center, y_center, width, height] (normalized coordinates)
-        minx_abs = min(p0[0],p1[0])
-        maxx_abs = max(p0[0],p1[0])
-        miny_abs = min(p0[1],p1[1])
-        maxy_abs = max(p0[1],p1[1])
-        
-        if (minx_abs >= (im_width-1)) or (maxx_abs <= 0) or \
-            (miny_abs >= (im_height-1)) or (maxy_abs <= 0):
-                print('Skipping invalid shape in {}'.format(labelme_file))                
-                continue
-            
-        # Clip to [0,1]... it's not obvious that the YOLO format doesn't allow bounding
-        # boxes to extend outside the image, but YOLOv5 and YOLOv8 get sad about boxes
-        # that extend outside the image.
-        maxx_abs = min(maxx_abs,im_width-1)
-        maxy_abs = min(maxy_abs,im_height-1)
-        minx_abs = max(minx_abs,0.0)
-        miny_abs = max(miny_abs,0.0)
-        
-        minx_rel = minx_abs / (im_width-1)
-        maxx_rel = maxx_abs / (im_width-1)
-        miny_rel = miny_abs / (im_height-1)
-        maxy_rel = maxy_abs / (im_height-1)
-        
-        assert maxx_rel >= minx_rel
-        assert maxy_rel >= miny_rel
-        
-        xcenter_rel = (maxx_rel + minx_rel) / 2.0
-        ycenter_rel = (maxy_rel + miny_rel) / 2.0
-        w_rel = maxx_rel - minx_rel
-        h_rel = maxy_rel - miny_rel
-        
-        yolo_line = '{} {:.3f} {:.3f} {:.3f} {:.3f}'.format(category_id,
-            xcenter_rel, ycenter_rel, w_rel, h_rel)
-        yolo_lines.append(yolo_line)
-        
-    # ...for each shape
-    
-    with open(yolo_file,'w') as f:
-        for s in yolo_lines:
-            f.write(s + '\n')
-    
-    result['status'] = 'converted'
-    return result
-
-
-def labelme_folder_to_yolo(labelme_folder,
-                           category_name_to_category_id=None,
-                           required_token=None,
-                           overwrite_behavior='overwrite',
-                           relative_filenames_to_convert=None,
-                           n_workers=1,
-                           use_threads=True):
-    """
-    Given a folder with images and labelme .json files, convert the .json files
-    to YOLO .txt format.  If category_name_to_category_id is None, first reads
-    all the labels in the folder to build a zero-indexed name --> ID mapping.
-    
-    If required_token is not None and a labelme_file does not contain the key [required_token],
-    it won't be converted.  Typically used to specify a field that indicates which files have
-    been reviewed.
-    
-    If relative_filenames_to_convert is not None, this should be a list of .json (not image)
-    files that should get converted, relative to the base folder.
-    
-    overwrite_behavior should be 'skip' or 'overwrite' (default).
-    
-    returns a dict with:
-        'category_name_to_category_id', whether it was passed in or constructed
-        'image_results': a list of results for each image (converted, skipped, error)
-        
-    """
-    
-    if relative_filenames_to_convert is not None:
-        labelme_files_relative = relative_filenames_to_convert
-        assert all([fn.endswith('.json') for fn in labelme_files_relative]), \
-            'relative_filenames_to_convert contains non-json files'
-    else:
-        labelme_files_relative = recursive_file_list(labelme_folder,return_relative_paths=True)
-        labelme_files_relative = [fn for fn in labelme_files_relative if fn.endswith('.json')]
-    
-    if required_token is None:
-        valid_labelme_files_relative = labelme_files_relative
-    else:        
-        valid_labelme_files_relative = []
-        
-        # fn_relative = labelme_files_relative[-1]
-        for fn_relative in labelme_files_relative:
-                        
-            fn_abs = os.path.join(labelme_folder,fn_relative)            
-            
-            with open(fn_abs,'r') as f:
-                labelme_data = json.load(f)
-                if required_token not in labelme_data:
-                    continue
-                
-            valid_labelme_files_relative.append(fn_relative)
-    
-        print('{} of {} files are valid'.format(len(valid_labelme_files_relative),
-                                                len(labelme_files_relative)))
-        
-    del labelme_files_relative
-        
-    if category_name_to_category_id is None:
-        
-        category_name_to_category_id = {}
-        
-        for fn_relative in valid_labelme_files_relative:
-            
-            fn_abs = os.path.join(labelme_folder,fn_relative)
-            with open(fn_abs,'r') as f:
-                labelme_data = json.load(f)
-                for shape in labelme_data['shapes']:
-                    label = shape['label']
-                    if label not in category_name_to_category_id:
-                        category_name_to_category_id[label] = len(category_name_to_category_id)
-        # ...for each file
-        
-    # ...if we need to build a category mapping
-    
-    image_results = []
-    
-    n_workers = min(n_workers,len(valid_labelme_files_relative))
-    
-    if n_workers <= 1:
-        for fn_relative in tqdm(valid_labelme_files_relative):
-            
-            fn_abs = os.path.join(labelme_folder,fn_relative)
-            image_result = labelme_file_to_yolo_file(fn_abs,
-                                      category_name_to_category_id,
-                                      yolo_file=None,
-                                      required_token=required_token,
-                                      overwrite_behavior=overwrite_behavior)
-            image_results.append(image_result)
-        # ...for each file
-    else:
-        if use_threads:
-            pool = ThreadPool(n_workers)
-        else:
-            pool = Pool(n_workers)
-
-        valid_labelme_files_abs = [os.path.join(labelme_folder,fn_relative) for \
-                                   fn_relative in valid_labelme_files_relative]
-            
-        image_results = list(tqdm(pool.imap(
-            partial(labelme_file_to_yolo_file,
-                    category_name_to_category_id=category_name_to_category_id,
-                    yolo_file=None,
-                    required_token=required_token,
-                    overwrite_behavior=overwrite_behavior),
-                    valid_labelme_files_abs), 
-                    total=len(valid_labelme_files_abs)))
-
-    assert len(valid_labelme_files_relative) == len(image_results)
-    
-    print('Converted {} labelme .json files to YOLO'.format(
-        len(valid_labelme_files_relative)))
-    
-    labelme_to_yolo_results = {}
-    labelme_to_yolo_results['category_name_to_category_id'] = category_name_to_category_id
-    labelme_to_yolo_results['image_results'] = image_results
-    
-    return labelme_to_yolo_results
-    
-# ...def labelme_folder_to_yolo(...)        
-
-
-#%% Interactive driver
-
-if False:
-
-    pass
-
-    #%%
-
-    labelme_file = os.path.expanduser('~/tmp/labels/x.json')
-    required_token = 'saved_by_labelme'
-    category_name_to_category_id = {'animal':0}
-    labelme_folder = os.path.expanduser('~/tmp/labels')
-
-    #%%
-    
-    category_name_to_category_id = \
-        labelme_folder_to_yolo(labelme_folder,
-                               category_name_to_category_id=category_name_to_category_id,
-                               required_token=required_token,
-                               overwrite_behavior='overwrite')
-    
-#%% Command-line driver
-
-# TODO

data_management/lila/add_locations_to_island_camera_traps.py

@@ -1,97 +0,0 @@
-"""
-
-add_locations_to_island_camera_traps.py
-
-The Island Conservation Camera Traps dataset had unique camera identifiers embedded
-in filenames, but not in the proper metadata fields.  This script copies that information
-to metadata.
-
-"""
-
-#%% Imports and constants
-
-import os
-import json
-from tqdm import tqdm
-
-input_fn = os.path.expanduser('~/lila/metadata/island_conservation.json')
-output_fn = os.path.expanduser('~/tmp/island_conservation.json')
-preview_folder = os.path.expanduser('~/tmp/island_conservation_preview')
-image_directory = os.path.expanduser('~/data/icct/public/')
-
-
-#%% Read input file
-
-with open(input_fn,'r') as f:
-    d = json.load(f)
-    
-d['info']
-d['info']['version'] = '1.01'
-
-
-#%% Find locations
-
-images = d['images']
-
-locations = set()
-
-for i_image,im in tqdm(enumerate(images),total=len(images)):
-    tokens_fn = im['file_name'].split('/')
-    tokens_id = im['id'].split('_')
-    assert tokens_fn[0] == tokens_id[0]
-    assert tokens_fn[1] == tokens_id[1]
-    location = tokens_fn[0] + '_' + tokens_fn[1]
-    im['location'] = location
-    locations.add(location)
-
-locations = sorted(list(locations))
-    
-for s in locations:
-    print(s)
-    
-    
-#%% Write output file
-
-with open(output_fn,'w') as f:
-    json.dump(d,f,indent=1)
-    
-
-#%% Validate .json files
-
-from data_management.databases import integrity_check_json_db
-
-options = integrity_check_json_db.IntegrityCheckOptions()
-options.baseDir = image_directory
-options.bCheckImageSizes = False
-options.bCheckImageExistence = True
-options.bFindUnusedImages = True
-
-sorted_categories, data, error_info = integrity_check_json_db.integrity_check_json_db(output_fn, options)
-
-
-#%% Preview labels
-
-from md_visualization import visualize_db
-
-viz_options = visualize_db.DbVizOptions()
-viz_options.num_to_visualize = 2000
-viz_options.trim_to_images_with_bboxes = False
-viz_options.add_search_links = False
-viz_options.sort_by_filename = False
-viz_options.parallelize_rendering = True
-viz_options.classes_to_exclude = ['test']
-html_output_file, image_db = visualize_db.visualize_db(db_path=output_fn,
-                                                         output_dir=preview_folder,
-                                                         image_base_dir=image_directory,
-                                                         options=viz_options)
-
-from md_utils import path_utils
-path_utils.open_file(html_output_file)
-
-
-#%% Zip output file
-
-from md_utils.path_utils import zip_file
-
-zip_file(output_fn, verbose=True)
-assert os.path.isfile(output_fn + '.zip')

data_management/lila/add_locations_to_nacti.py

@@ -1,147 +0,0 @@
-"""
-
-add_locations_to_nacti.py
-
-As of 10.2023, NACTI metadata only has very coarse location information (e.g. "Florida"),
-but camera IDs are embedded in filenames.  This script pulls that information from filenames
-and adds it to metadata.
-
-"""
-
-#%% Imports and constants
-
-import os
-import json
-import shutil
-
-from tqdm import tqdm
-from collections import defaultdict
-
-input_file = r'd:\lila\nacti\nacti_metadata.json.1.13\nacti_metadata.json'
-output_file = r'g:\temp\nacti_metadata.1.14.json'
-
-
-#%% Read metadata
-
-with open(input_file,'r') as f:
-    d = json.load(f)
-    
-assert d['info']['version'] == 1.13
-
-
-#%% Map images to locations (according to the metadata)
-
-file_name_to_original_location = {}
-
-# im = dataset_labels['images'][0]
-for im in tqdm(d['images']):
-    file_name_to_original_location[im['file_name']] = im['location']
-
-original_locations = set(file_name_to_original_location.values())
-
-print('Found {} locations in the original metadata:'.format(len(original_locations)))
-for loc in original_locations:
-    print('[{}]'.format(loc))
-
-
-#%% Map images to new locations
-
-def path_to_location(relative_path):
-    
-    relative_path = relative_path.replace('\\','/')
-    if relative_path in file_name_to_original_location:
-        location_name = file_name_to_original_location[relative_path]
-        if location_name == 'San Juan Mntns, Colorado':
-            # "part0/sub000/2010_Unit150_Ivan097_img0003.jpg"
-            tokens = relative_path.split('/')[-1].split('_')
-            assert tokens[1].startswith('Unit')
-            location_name = 'sanjuan_{}_{}_{}'.format(tokens[0],tokens[1],tokens[2])
-        elif location_name == 'Lebec, California':
-            # "part0/sub035/CA-03_08_13_2015_CA-03_0009738.jpg"
-            tokens = relative_path.split('/')[-1].split('_')
-            assert tokens[0].startswith('CA-') or tokens[0].startswith('TAG-')
-            location_name = 'lebec_{}'.format(tokens[0])   
-        elif location_name == 'Archbold, FL':
-            # "part1/sub110/FL-01_01_25_2016_FL-01_0040421.jpg"
-            tokens = relative_path.split('/')[-1].split('_')
-            assert tokens[0].startswith('FL-')
-            location_name = 'archbold_{}'.format(tokens[0])   
-        else:
-            assert location_name == ''
-            tokens = relative_path.split('/')[-1].split('_')
-            if tokens[0].startswith('CA-') or tokens[0].startswith('TAG-') or tokens[0].startswith('FL-'):
-                location_name = '{}'.format(tokens[0])
-            
-    else:
-        
-        location_name = 'unknown'    
-    
-    # print('Returning location {} for file {}'.format(location_name,relative_path))          
-
-    return location_name
-
-file_name_to_updated_location = {}
-updated_location_to_count = defaultdict(int)
-for im in tqdm(d['images']):
-    
-    updated_location = path_to_location(im['file_name'])
-    file_name_to_updated_location[im['file_name']] = updated_location
-    updated_location_to_count[updated_location] += 1
-
-updated_location_to_count = {k: v for k, v in sorted(updated_location_to_count.items(), 
-                         key=lambda item: item[1],
-                         reverse=True)}
-
-updated_locations = set(file_name_to_updated_location.values())
-
-print('Found {} updated locations in the original metadata:'.format(len(updated_locations)))
-for loc in updated_location_to_count:
-    print('{}: {}'.format(loc,updated_location_to_count[loc]))
-
-
-#%% Re-write metadata
-
-for im in d['images']:
-    im['location'] = file_name_to_updated_location[im['file_name']]
-d['info']['version'] = 1.14
-
-with open(output_file,'w') as f:
-    json.dump(d,f,indent=1)
-    
-
-#%% For each location, sample some random images to make sure they look consistent
-
-input_base = r'd:\lila\nacti-unzipped'
-assert os.path.isdir(input_base)
-
-location_to_images = defaultdict(list)
-
-for im in d['images']:
-    location_to_images[im['location']].append(im)
-    
-n_to_sample = 10
-import random
-random.seed(0)
-sampling_folder_base = r'g:\temp\nacti_samples'
-
-for location in tqdm(location_to_images):
-    
-    images_this_location = location_to_images[location]
-    if len(images_this_location) > n_to_sample:
-        images_this_location = random.sample(images_this_location,n_to_sample)
-        
-    for i_image,im in enumerate(images_this_location):
-        
-        fn_relative = im['file_name']
-        source_fn_abs = os.path.join(input_base,fn_relative)
-        assert os.path.isfile(source_fn_abs)
-        ext = os.path.splitext(fn_relative)[1]            
-        target_fn_abs = os.path.join(sampling_folder_base,'{}/{}'.format(
-            location,'image_{}{}'.format(str(i_image).zfill(2),ext)))
-        os.makedirs(os.path.dirname(target_fn_abs),exist_ok=True)
-        shutil.copyfile(source_fn_abs,target_fn_abs)
-        
-    # ...for each image
-    
-# ...for each location
-