megadetector 5.0.9__py3-none-any.whl → 5.0.11__py3-none-any.whl

data_management/databases/subset_json_db.py

@@ -1,115 +0,0 @@
-"""
-
-subset_json_db.py
-
-Select a subset of images (and associated annotations) from a .json file in COCO 
-Camera Traps format based on a string query.
-
-To subset .json files in the MegaDetector output format, see
-subset_json_detector_output.py.
-
-"""
-    
-#%% Constants and imports
-
-import sys
-import json
-import argparse
-
-from tqdm import tqdm
-
-
-#%% Functions
-
-def subset_json_db(input_json, query, output_json=None, ignore_case=False):
-    """
-    Given a json file (or dictionary already loaded from a json file), produce a new 
-    database containing only the images whose filenames contain the string 'query', 
-    optionally writing that DB output to a new json file.
-    
-    Args:
-        input_json (str): COCO Camera Traps .json file to load, or an already-loaded dict
-        query (str): string to query for, only include images in the output whose filenames 
-            contain this string.
-        output_json (str, optional): file to write the resulting .json file to
-        ignore_case (bool, optional): whether to perform a case-insensitive search for [query]
-        
-    Returns:
-        dict: possibly-modified CCT dictionary
-    """
-    
-    if ignore_case:
-        query = query.lower()
-        
-    # Load the input file if necessary
-    if isinstance(input_json,str):
-        print('Loading input .json...')
-        with open(input_json, 'r') as f:
-            data = json.load(f)
-    else:
-        data = input_json
-
-    # Find images matching the query
-    images = []
-    image_ids = set()
-    
-    for im in tqdm(data['images']):
-        fn = im['file_name']
-        if ignore_case:
-            fn = fn.lower()
-        if query in fn:
-            images.append(im)
-            image_ids.add(im['id'])        
-    
-    # Find annotations referring to those images
-    annotations = []
-    
-    for ann in tqdm(data['annotations']):
-        if ann['image_id'] in image_ids:
-            annotations.append(ann)
-    
-    output_data = data
-    output_data['images'] = images
-    output_data['annotations'] = annotations
-    
-    # Write the output file if requested
-    if output_json is not None:
-        print('Writing output .json...')
-        json.dump(output_data,open(output_json,'w'),indent=1)
-        
-    return output_data
-
-
-#%% Interactive driver
-
-if False:
-    
-    #%%
-    
-    input_json = r"e:\Statewide_wolf_container\idfg_20190409.json"
-    output_json = r"e:\Statewide_wolf_container\idfg_20190409_clearcreek.json"
-    query = 'clearcreek'
-    ignore_case = True
-    db = subset_json_db(input_json, query, output_json, ignore_case)
-    
-
-#%% Command-line driver
-
-def main():
-    
-    parser = argparse.ArgumentParser()
-    parser.add_argument('input_json', type=str, help='Input file (a COCO Camera Traps .json file)')
-    parser.add_argument('output_json', type=str, help='Output file')    
-    parser.add_argument('query', type=str, help='Filename query')    
-    parser.add_argument('--ignore_case', action='store_true')
-    
-    if len(sys.argv[1:]) == 0:
-        parser.print_help()
-        parser.exit()
-
-    args = parser.parse_args()    
-    
-    subset_json_db(args.input_json,args.query,args.output_json,args.ignore_case)
-
-if __name__ == '__main__':    
-    main()

data_management/generate_crops_from_cct.py

@@ -1,149 +0,0 @@
-"""
-
-generate_crops_from_cct.py
-
-Given a .json file in COCO Camera Traps format, creates a cropped image for
-each bounding box.
-
-"""
-
-#%% Imports and constants
-
-import os
-import json
-
-from tqdm import tqdm
-from PIL import Image
-
-
-#%% Functions
-
-def generate_crops_from_cct(cct_file,image_dir,output_dir,padding=0,flat_output=True):
-    """
-    Given a .json file in COCO Camera Traps format, creates a cropped image for
-    each bounding box.
-    
-    Args:
-        cct_file (str): the COCO .json file from which we should load data
-        image_dir (str): the folder where the images live; filenames in the .json
-            file should be relative to this folder
-        output_dir (str): the folder where we should write cropped images
-        padding (float, optional): number of pixels we should expand each box before
-            cropping
-        flat_output (bool, optional): if False, folder structure will be preserved
-            in the output, e.g. the image a/b/c/d.jpg will result in image files 
-            in the output folder called, e.g., a/b/c/d_crop_000_id_12345.jpg.  If
-            [flat_output] is True, the corresponding output image will be 
-            a_b_c_d_crop_000_id_12345.jpg.            
-    """
-    
-    ## Read and validate input
-    
-    assert os.path.isfile(cct_file)
-    assert os.path.isdir(image_dir)
-    os.makedirs(output_dir,exist_ok=True)
-
-    with open(cct_file,'r') as f:
-        d = json.load(f)
-   
-    
-    ## Find annotations for each image
-    
-    from collections import defaultdict
-    
-    # This actually maps image IDs to annotations, but only to annotations
-    # containing boxes
-    image_id_to_boxes = defaultdict(list)
-    
-    n_boxes = 0
-    
-    for ann in d['annotations']:
-        if 'bbox' in ann:
-            image_id_to_boxes[ann['image_id']].append(ann)
-            n_boxes += 1
-            
-    print('Found {} boxes in {} annotations for {} images'.format(
-        n_boxes,len(d['annotations']),len(d['images'])))
-    
-    
-    ## Generate crops
-        
-    # im = d['images'][0]
-    for im in tqdm(d['images']):
-        
-        input_image_fn = os.path.join(os.path.join(image_dir,im['file_name']))
-        assert os.path.isfile(input_image_fn), 'Could not find image {}'.format(input_image_fn)
-        
-        if im['id'] not in image_id_to_boxes:
-            continue
-        
-        annotations_this_image = image_id_to_boxes[im['id']]
-        
-        # Load the image
-        img = Image.open(input_image_fn)
-        
-        # Generate crops
-        # i_ann = 0; ann = annotations_this_image[i_ann]
-        for i_ann,ann in enumerate(annotations_this_image):
-            
-            # x/y/w/h, origin at the upper-left
-            bbox = ann['bbox']
-            
-            xmin = bbox[0]
-            ymin = bbox[1]
-            xmax = xmin + bbox[2]
-            ymax = ymin + bbox[3]
-            
-            xmin -= padding / 2
-            ymin -= padding / 2
-            xmax += padding / 2
-            ymax += padding / 2
-            
-            xmin = max(xmin,0)
-            ymin = max(ymin,0)
-            xmax = min(xmax,img.width-1)
-            ymax = min(ymax,img.height-1)
-                        
-            crop = img.crop(box=[xmin, ymin, xmax, ymax])
-            
-            output_fn = os.path.splitext(im['file_name'])[0].replace('\\','/')
-            if flat_output:
-                output_fn = output_fn.replace('/','_')
-            output_fn = output_fn + '_crop' + str(i_ann).zfill(3) + '_id_' + ann['id']
-            output_fn = output_fn + '.jpg'
-                
-            output_full_path = os.path.join(output_dir,output_fn)
-            
-            if not flat_output:
-                os.makedirs(os.path.dirname(output_full_path),exist_ok=True)
-                
-            crop.save(output_full_path)
-            
-        # ...for each box
-        
-    # ...for each image
-    
-# ...generate_crops_from_cct()
-
-
-#%% Interactive driver
-
-if False:
-    
-    pass
-
-    #%%
-    
-    cct_file = os.path.expanduser('~/data/noaa/noaa_estuary_fish.json')
-    image_dir = os.path.expanduser('~/data/noaa/JPEGImages')
-    padding = 50
-    flat_output = True
-    output_dir = '/home/user/tmp/noaa-fish-crops'
-    
-    generate_crops_from_cct(cct_file,image_dir,output_dir,padding,flat_output=True)
-    files = os.listdir(output_dir)
-    
-    
-#%% Command-line driver
-
-# TODO

data_management/get_image_sizes.py

@@ -1,188 +0,0 @@
-"""
-
-get_image_sizes.py
-
-Given a json-formatted list of image filenames, retrieves the width and height of 
-every image, optionally writing the results to a new .json file.
-
-"""
-
-#%% Constants and imports
-
-import argparse
-import json
-import os
-from PIL import Image
-import sys
-
-from multiprocessing.pool import ThreadPool
-from multiprocessing.pool import Pool
-from functools import partial
-from tqdm import tqdm
-
-from md_utils.path_utils import find_images
-
-image_base = ''
-default_n_threads = 1
-use_threads = False
-
-
-#%% Processing functions
-
-def _get_image_size(image_path,image_prefix=None):
-    """
-    Support function to get the size of a single image.  Returns a (path,w,h) tuple.
-    w and h will be -1 if the image fails to load.
-    """
-    
-    if image_prefix is not None:
-        full_path = os.path.join(image_prefix,image_path)
-    else:
-        full_path = image_path
-    
-    # Is this image on disk?
-    if not os.path.isfile(full_path):
-        print('Could not find image {}'.format(full_path))
-        return (image_path,-1,-1)
-
-    try:        
-        pil_im = Image.open(full_path)
-        w = pil_im.width            
-        h = pil_im.height
-        return (image_path,w,h)
-    except Exception as e:    
-        print('Error reading image {}: {}'.format(full_path,str(e)))
-        return (image_path,-1,-1)
-    
-    
-def get_image_sizes(filenames,image_prefix=None,output_file=None,
-                    n_workers=default_n_threads,use_threads=True,
-                    recursive=True):
-    """
-    Gets the width and height of all images in [filenames], which can be:
-        
-    * A .json-formatted file containing list of strings
-    * A folder
-    * A list of files
-
-    ...returning a list of (path,w,h) tuples, and optionally writing the results to [output_file].
-    
-    Args:
-        filenames (str or list): the image filenames for which we should retrieve sizes, 
-            can be the name of a .json-formatted file containing list of strings, a folder 
-            in which we should enumerate images, or a list of files.
-        image_prefix (str, optional): optional prefix to add to images to get to full paths;
-            useful when [filenames] contains relative files, in which case [image_prefix] is the 
-            base folder for the source images.
-        output_file (str, optional): a .json file to write the imgae sizes
-        n_workers (int, optional): number of parallel workers to use, set to <=1 to
-            disable parallelization
-        use_threads (bool, optional): whether to use threads (True) or processes (False)
-            for parallelization; not relevant if [n_workers] <= 1
-        recursive (bool, optional): only relevant if [filenames] is actually a folder,
-            determines whether image enumeration within that folder will be recursive
-        
-    Returns:
-        list: list of (path,w,h) tuples
-    """        
-    
-    if output_file is not None:
-        assert os.path.isdir(os.path.dirname(output_file)), \
-            'Illegal output file {}, parent folder does not exist'.format(output_file)
-        
-    if isinstance(filenames,str) and os.path.isfile(filenames):
-        with open(filenames,'r') as f:        
-            filenames = json.load(f)
-        filenames = [s.strip() for s in filenames]
-    elif isinstance(filenames,str) and os.path.isdir(filenames):
-        filenames = find_images(filenames,recursive=recursive,
-                                return_relative_paths=False,convert_slashes=True)
-    else:
-        assert isinstance(filenames,list)        
-    
-    if n_workers <= 1:
-        
-        all_results = []
-        for i_file,fn in tqdm(enumerate(filenames),total=len(filenames)):
-            all_results.append(_get_image_size(fn,image_prefix=image_prefix))
-    
-    else:
-        
-        print('Creating a pool with {} workers'.format(n_workers))
-        if use_threads:
-            pool = ThreadPool(n_workers)        
-        else:
-            pool = Pool(n_workers)
-        # all_results = list(tqdm(pool.imap(process_image, filenames), total=len(filenames)))
-        all_results = list(tqdm(pool.imap(
-            partial(_get_image_size,image_prefix=image_prefix), filenames), total=len(filenames)))
-    
-    if output_file is not None:
-        with open(output_file,'w') as f:
-            json.dump(all_results,f,indent=1)
-            
-    return all_results
-
-    
-#%% Interactive driver
-
-if False:
-
-    pass    
-
-    #%%
-    
-    # List images in a test folder
-    base_dir = r'c:\temp\test_images'
-    image_list_file = os.path.join(base_dir,'images.json')
-    relative_image_list_file = os.path.join(base_dir,'images_relative.json')
-    image_size_file = os.path.join(base_dir,'image_sizes.json')
-    from md_utils import path_utils
-    image_names = path_utils.find_images(base_dir,recursive=True)
-    
-    with open(image_list_file,'w') as f:
-        json.dump(image_names,f,indent=1)
-        
-    relative_image_names = []
-    for s in image_names:
-        relative_image_names.append(os.path.relpath(s,base_dir))
-    
-    with open(relative_image_list_file,'w') as f:
-        json.dump(relative_image_names,f,indent=1)
-    
-    
-    #%%
-    
-    get_image_sizes(relative_image_list_file,image_size_file,image_prefix=base_dir,n_threads=4)
-    
-    
-#%% Command-line driver
-    
-def main():
-    
-    parser = argparse.ArgumentParser()
-    parser.add_argument('filenames',type=str,
-                        help='Folder from which we should fetch image sizes, or .json file with a list of filenames')
-    parser.add_argument('output_file',type=str,
-                        help='Output file (.json) to which we should write image size information')
-    parser.add_argument('--image_prefix', type=str, default=None,
-                        help='Prefix to append to image filenames, only relevant if [filenames] points to a list of ' + \
-                             'relative paths')        
-    parser.add_argument('--n_threads', type=int, default=default_n_threads,
-                        help='Number of concurrent workers, set to <=1 to disable parallelization (default {})'.format(
-                            default_n_threads))
-                        
-    if len(sys.argv[1:])==0:
-        parser.print_help()
-        parser.exit()
-        
-    args = parser.parse_args()
-    
-    _ = get_image_sizes(filenames=args.filenames,
-                        output_file=args.output_file,
-                        image_prefix=args.image_prefix,
-                        n_workers=args.n_threads)
-        
-if __name__ == '__main__':
-    
-    main()

data_management/importers/add_nacti_sizes.py

@@ -1,52 +0,0 @@
-"""
-
- add_nacti_sizes.py
-
- NACTI bounding box metadata was posted before we inclduded width and height as semi-standard
- fields; pull size information from the main metadata file and add to the bbox file.
-
-"""
-
-#%% Constants and environment
-
-import json
-from tqdm import tqdm
-
-input_file = 'G:/temp/nacti_metadata.json'
-input_bbox_file = 'G:/temp/nacti_20200401_bboxes.json'
-output_bbox_file = 'G:/temp/nacti_20230920_bboxes.json'
-
-
-#%% Read .json files
-
-with open(input_file,'r') as f:
-    input_data = json.load(f)
-
-with open(input_bbox_file,'r') as f:
-    input_bbox_data = json.load(f)
-    
-print('Finished reading .json data')
-
-
-#%% Map image names to width and height
-
-filename_to_size = {}
-for im in tqdm(input_data['images']):
-    filename_to_size[im['file_name']] = (im['width'],im['height'])
-    
-
-#%% Add to output data
-
-for im in tqdm(input_bbox_data['images']):
-    size = filename_to_size[im['file_name']]
-    im['width'] = size[0]
-    im['height'] = size[1]
-    
-
-#%% Write output
-
-output_bbox_data = input_bbox_data
-output_bbox_data['version'] = '2023-09-20'
-
-with open(output_bbox_file,'w') as f:
-    json.dump(output_bbox_data,f,indent=1)

data_management/importers/add_timestamps_to_icct.py

@@ -1,79 +0,0 @@
-"""
-
- add_timestamps_to_icct.py
-
- The Island Conservation Camera Traps dataset was originally posted without timestamps
- in either .json metadata or EXIF metadata.  We pulled timestamps out using ocr_tools.py,
- this script adds those timestamps into the .json metadata.
-
-"""
-
-#%% Imports and constants
-
-import json
-
-ocr_results_file = r'g:\temp\ocr_results.2023.10.31.07.37.54.json'
-input_metadata_file = r'd:\lila\islandconservationcameratraps\island_conservation.json'
-output_metadata_file = r'g:\temp\island_conservation_camera_traps_1.02.json'
-ocr_results_file_base = 'g:/temp/island_conservation_camera_traps/'
-assert ocr_results_file_base.endswith('/')
-
-
-#%% Read input metadata
-
-with open(input_metadata_file,'r') as f:
-    input_metadata = json.load(f)
-    
-assert input_metadata['info']['version'] == '1.01'
-
-# im = input_metadata['images'][0]
-for im in input_metadata['images']:
-    assert 'datetime' not in im
-    
-
-#%% Read OCR results
-
-with open(ocr_results_file,'r') as f:
-    abs_filename_to_ocr_results = json.load(f)
-    
-relative_filename_to_ocr_results = {}
-
-for fn_abs in abs_filename_to_ocr_results:
-    assert ocr_results_file_base in fn_abs
-    fn_relative = fn_abs.replace(ocr_results_file_base,'')
-    relative_filename_to_ocr_results[fn_relative] = abs_filename_to_ocr_results[fn_abs]
-    
-
-#%% Add datetimes to metadata
-
-images_not_in_datetime_results = []
-images_with_failed_datetimes = []
-
-for i_image,im in enumerate(input_metadata['images']):
-    if im['file_name'] not in relative_filename_to_ocr_results:
-        images_not_in_datetime_results.append(im)
-        im['datetime'] = None
-        continue
-    ocr_results = relative_filename_to_ocr_results[im['file_name']]
-    if ocr_results['datetime'] is None:
-        images_with_failed_datetimes.append(im)
-        im['datetime'] = None
-        continue
-    im['datetime'] = ocr_results['datetime']
-
-print('{} of {} images were not in datetime results'.format(
-    len(images_not_in_datetime_results),len(input_metadata['images'])))
-
-print('{} of {} images were had failed datetime results'.format(
-    len(images_with_failed_datetimes),len(input_metadata['images'])))
-
-for im in input_metadata['images']:
-    assert 'datetime' in im
-
-
-#%% Write output
-
-input_metadata['info']['version'] = '1.02'
-
-with open(output_metadata_file,'w') as f:
-    json.dump(input_metadata,f,indent=1)