megadetector 5.0.11__py3-none-any.whl → 5.0.12__py3-none-any.whl

megadetector/data_management/databases/subset_json_db.py

@@ -0,0 +1,115 @@
+"""
+
+subset_json_db.py
+
+Select a subset of images (and associated annotations) from a .json file in COCO 
+Camera Traps format based on a string query.
+
+To subset .json files in the MegaDetector output format, see
+subset_json_detector_output.py.
+
+"""
+    
+#%% Constants and imports
+
+import sys
+import json
+import argparse
+
+from tqdm import tqdm
+
+
+#%% Functions
+
+def subset_json_db(input_json, query, output_json=None, ignore_case=False):
+    """
+    Given a json file (or dictionary already loaded from a json file), produce a new 
+    database containing only the images whose filenames contain the string 'query', 
+    optionally writing that DB output to a new json file.
+    
+    Args:
+        input_json (str): COCO Camera Traps .json file to load, or an already-loaded dict
+        query (str): string to query for, only include images in the output whose filenames 
+            contain this string.
+        output_json (str, optional): file to write the resulting .json file to
+        ignore_case (bool, optional): whether to perform a case-insensitive search for [query]
+        
+    Returns:
+        dict: possibly-modified CCT dictionary
+    """
+    
+    if ignore_case:
+        query = query.lower()
+        
+    # Load the input file if necessary
+    if isinstance(input_json,str):
+        print('Loading input .json...')
+        with open(input_json, 'r') as f:
+            data = json.load(f)
+    else:
+        data = input_json
+
+    # Find images matching the query
+    images = []
+    image_ids = set()
+    
+    for im in tqdm(data['images']):
+        fn = im['file_name']
+        if ignore_case:
+            fn = fn.lower()
+        if query in fn:
+            images.append(im)
+            image_ids.add(im['id'])        
+    
+    # Find annotations referring to those images
+    annotations = []
+    
+    for ann in tqdm(data['annotations']):
+        if ann['image_id'] in image_ids:
+            annotations.append(ann)
+    
+    output_data = data
+    output_data['images'] = images
+    output_data['annotations'] = annotations
+    
+    # Write the output file if requested
+    if output_json is not None:
+        print('Writing output .json...')
+        json.dump(output_data,open(output_json,'w'),indent=1)
+        
+    return output_data
+
+
+#%% Interactive driver
+
+if False:
+    
+    #%%
+    
+    input_json = r"e:\Statewide_wolf_container\idfg_20190409.json"
+    output_json = r"e:\Statewide_wolf_container\idfg_20190409_clearcreek.json"
+    query = 'clearcreek'
+    ignore_case = True
+    db = subset_json_db(input_json, query, output_json, ignore_case)
+    
+
+#%% Command-line driver
+
+def main():
+    
+    parser = argparse.ArgumentParser()
+    parser.add_argument('input_json', type=str, help='Input file (a COCO Camera Traps .json file)')
+    parser.add_argument('output_json', type=str, help='Output file')    
+    parser.add_argument('query', type=str, help='Filename query')    
+    parser.add_argument('--ignore_case', action='store_true')
+    
+    if len(sys.argv[1:]) == 0:
+        parser.print_help()
+        parser.exit()
+
+    args = parser.parse_args()    
+    
+    subset_json_db(args.input_json,args.query,args.output_json,args.ignore_case)
+
+if __name__ == '__main__':    
+    main()

megadetector/data_management/generate_crops_from_cct.py

@@ -0,0 +1,149 @@
+"""
+
+generate_crops_from_cct.py
+
+Given a .json file in COCO Camera Traps format, creates a cropped image for
+each bounding box.
+
+"""
+
+#%% Imports and constants
+
+import os
+import json
+
+from tqdm import tqdm
+from PIL import Image
+
+
+#%% Functions
+
+def generate_crops_from_cct(cct_file,image_dir,output_dir,padding=0,flat_output=True):
+    """
+    Given a .json file in COCO Camera Traps format, creates a cropped image for
+    each bounding box.
+    
+    Args:
+        cct_file (str): the COCO .json file from which we should load data
+        image_dir (str): the folder where the images live; filenames in the .json
+            file should be relative to this folder
+        output_dir (str): the folder where we should write cropped images
+        padding (float, optional): number of pixels we should expand each box before
+            cropping
+        flat_output (bool, optional): if False, folder structure will be preserved
+            in the output, e.g. the image a/b/c/d.jpg will result in image files 
+            in the output folder called, e.g., a/b/c/d_crop_000_id_12345.jpg.  If
+            [flat_output] is True, the corresponding output image will be 
+            a_b_c_d_crop_000_id_12345.jpg.            
+    """
+    
+    ## Read and validate input
+    
+    assert os.path.isfile(cct_file)
+    assert os.path.isdir(image_dir)
+    os.makedirs(output_dir,exist_ok=True)
+
+    with open(cct_file,'r') as f:
+        d = json.load(f)
+   
+    
+    ## Find annotations for each image
+    
+    from collections import defaultdict
+    
+    # This actually maps image IDs to annotations, but only to annotations
+    # containing boxes
+    image_id_to_boxes = defaultdict(list)
+    
+    n_boxes = 0
+    
+    for ann in d['annotations']:
+        if 'bbox' in ann:
+            image_id_to_boxes[ann['image_id']].append(ann)
+            n_boxes += 1
+            
+    print('Found {} boxes in {} annotations for {} images'.format(
+        n_boxes,len(d['annotations']),len(d['images'])))
+    
+    
+    ## Generate crops
+        
+    # im = d['images'][0]
+    for im in tqdm(d['images']):
+        
+        input_image_fn = os.path.join(os.path.join(image_dir,im['file_name']))
+        assert os.path.isfile(input_image_fn), 'Could not find image {}'.format(input_image_fn)
+        
+        if im['id'] not in image_id_to_boxes:
+            continue
+        
+        annotations_this_image = image_id_to_boxes[im['id']]
+        
+        # Load the image
+        img = Image.open(input_image_fn)
+        
+        # Generate crops
+        # i_ann = 0; ann = annotations_this_image[i_ann]
+        for i_ann,ann in enumerate(annotations_this_image):
+            
+            # x/y/w/h, origin at the upper-left
+            bbox = ann['bbox']
+            
+            xmin = bbox[0]
+            ymin = bbox[1]
+            xmax = xmin + bbox[2]
+            ymax = ymin + bbox[3]
+            
+            xmin -= padding / 2
+            ymin -= padding / 2
+            xmax += padding / 2
+            ymax += padding / 2
+            
+            xmin = max(xmin,0)
+            ymin = max(ymin,0)
+            xmax = min(xmax,img.width-1)
+            ymax = min(ymax,img.height-1)
+                        
+            crop = img.crop(box=[xmin, ymin, xmax, ymax])
+            
+            output_fn = os.path.splitext(im['file_name'])[0].replace('\\','/')
+            if flat_output:
+                output_fn = output_fn.replace('/','_')
+            output_fn = output_fn + '_crop' + str(i_ann).zfill(3) + '_id_' + ann['id']
+            output_fn = output_fn + '.jpg'
+                
+            output_full_path = os.path.join(output_dir,output_fn)
+            
+            if not flat_output:
+                os.makedirs(os.path.dirname(output_full_path),exist_ok=True)
+                
+            crop.save(output_full_path)
+            
+        # ...for each box
+        
+    # ...for each image
+    
+# ...generate_crops_from_cct()
+
+
+#%% Interactive driver
+
+if False:
+    
+    pass
+
+    #%%
+    
+    cct_file = os.path.expanduser('~/data/noaa/noaa_estuary_fish.json')
+    image_dir = os.path.expanduser('~/data/noaa/JPEGImages')
+    padding = 50
+    flat_output = True
+    output_dir = '/home/user/tmp/noaa-fish-crops'
+    
+    generate_crops_from_cct(cct_file,image_dir,output_dir,padding,flat_output=True)
+    files = os.listdir(output_dir)
+    
+    
+#%% Command-line driver
+
+# TODO

megadetector/data_management/get_image_sizes.py

@@ -0,0 +1,189 @@
+"""
+
+get_image_sizes.py
+
+Given a json-formatted list of image filenames, retrieves the width and height of 
+every image, optionally writing the results to a new .json file.
+
+"""
+
+#%% Constants and imports
+
+import argparse
+import json
+import os
+import sys
+
+from PIL import Image
+
+from multiprocessing.pool import ThreadPool
+from multiprocessing.pool import Pool
+from functools import partial
+from tqdm import tqdm
+
+from megadetector.utils.path_utils import find_images
+
+image_base = ''
+default_n_threads = 1
+use_threads = False
+
+
+#%% Processing functions
+
+def _get_image_size(image_path,image_prefix=None):
+    """
+    Support function to get the size of a single image.  Returns a (path,w,h) tuple.
+    w and h will be -1 if the image fails to load.
+    """
+    
+    if image_prefix is not None:
+        full_path = os.path.join(image_prefix,image_path)
+    else:
+        full_path = image_path
+    
+    # Is this image on disk?
+    if not os.path.isfile(full_path):
+        print('Could not find image {}'.format(full_path))
+        return (image_path,-1,-1)
+
+    try:        
+        pil_im = Image.open(full_path)
+        w = pil_im.width            
+        h = pil_im.height
+        return (image_path,w,h)
+    except Exception as e:    
+        print('Error reading image {}: {}'.format(full_path,str(e)))
+        return (image_path,-1,-1)
+    
+    
+def get_image_sizes(filenames,image_prefix=None,output_file=None,
+                    n_workers=default_n_threads,use_threads=True,
+                    recursive=True):
+    """
+    Gets the width and height of all images in [filenames], which can be:
+        
+    * A .json-formatted file containing list of strings
+    * A folder
+    * A list of files
+
+    ...returning a list of (path,w,h) tuples, and optionally writing the results to [output_file].
+    
+    Args:
+        filenames (str or list): the image filenames for which we should retrieve sizes, 
+            can be the name of a .json-formatted file containing list of strings, a folder 
+            in which we should enumerate images, or a list of files.
+        image_prefix (str, optional): optional prefix to add to images to get to full paths;
+            useful when [filenames] contains relative files, in which case [image_prefix] is the 
+            base folder for the source images.
+        output_file (str, optional): a .json file to write the imgae sizes
+        n_workers (int, optional): number of parallel workers to use, set to <=1 to
+            disable parallelization
+        use_threads (bool, optional): whether to use threads (True) or processes (False)
+            for parallelization; not relevant if [n_workers] <= 1
+        recursive (bool, optional): only relevant if [filenames] is actually a folder,
+            determines whether image enumeration within that folder will be recursive
+        
+    Returns:
+        list: list of (path,w,h) tuples
+    """        
+    
+    if output_file is not None:
+        assert os.path.isdir(os.path.dirname(output_file)), \
+            'Illegal output file {}, parent folder does not exist'.format(output_file)
+        
+    if isinstance(filenames,str) and os.path.isfile(filenames):
+        with open(filenames,'r') as f:        
+            filenames = json.load(f)
+        filenames = [s.strip() for s in filenames]
+    elif isinstance(filenames,str) and os.path.isdir(filenames):
+        filenames = find_images(filenames,recursive=recursive,
+                                return_relative_paths=False,convert_slashes=True)
+    else:
+        assert isinstance(filenames,list)        
+    
+    if n_workers <= 1:
+        
+        all_results = []
+        for i_file,fn in tqdm(enumerate(filenames),total=len(filenames)):
+            all_results.append(_get_image_size(fn,image_prefix=image_prefix))
+    
+    else:
+        
+        print('Creating a pool with {} workers'.format(n_workers))
+        if use_threads:
+            pool = ThreadPool(n_workers)        
+        else:
+            pool = Pool(n_workers)
+        # all_results = list(tqdm(pool.imap(process_image, filenames), total=len(filenames)))
+        all_results = list(tqdm(pool.imap(
+            partial(_get_image_size,image_prefix=image_prefix), filenames), total=len(filenames)))
+    
+    if output_file is not None:
+        with open(output_file,'w') as f:
+            json.dump(all_results,f,indent=1)
+            
+    return all_results
+
+    
+#%% Interactive driver
+
+if False:
+
+    pass    
+
+    #%%
+    
+    # List images in a test folder
+    base_dir = r'c:\temp\test_images'
+    image_list_file = os.path.join(base_dir,'images.json')
+    relative_image_list_file = os.path.join(base_dir,'images_relative.json')
+    image_size_file = os.path.join(base_dir,'image_sizes.json')
+    from megadetector.utils import path_utils
+    image_names = path_utils.find_images(base_dir,recursive=True)
+    
+    with open(image_list_file,'w') as f:
+        json.dump(image_names,f,indent=1)
+        
+    relative_image_names = []
+    for s in image_names:
+        relative_image_names.append(os.path.relpath(s,base_dir))
+    
+    with open(relative_image_list_file,'w') as f:
+        json.dump(relative_image_names,f,indent=1)
+    
+    
+    #%%
+    
+    get_image_sizes(relative_image_list_file,image_size_file,image_prefix=base_dir,n_threads=4)
+    
+    
+#%% Command-line driver
+    
+def main():
+    
+    parser = argparse.ArgumentParser()
+    parser.add_argument('filenames',type=str,
+                        help='Folder from which we should fetch image sizes, or .json file with a list of filenames')
+    parser.add_argument('output_file',type=str,
+                        help='Output file (.json) to which we should write image size information')
+    parser.add_argument('--image_prefix', type=str, default=None,
+                        help='Prefix to append to image filenames, only relevant if [filenames] points to a list of ' + \
+                             'relative paths')        
+    parser.add_argument('--n_threads', type=int, default=default_n_threads,
+                        help='Number of concurrent workers, set to <=1 to disable parallelization (default {})'.format(
+                            default_n_threads))
+                        
+    if len(sys.argv[1:])==0:
+        parser.print_help()
+        parser.exit()
+        
+    args = parser.parse_args()
+    
+    _ = get_image_sizes(filenames=args.filenames,
+                        output_file=args.output_file,
+                        image_prefix=args.image_prefix,
+                        n_workers=args.n_threads)
+        
+if __name__ == '__main__':
+    
+    main()

megadetector/data_management/importers/add_nacti_sizes.py

@@ -0,0 +1,52 @@
+"""
+
+ add_nacti_sizes.py
+
+ NACTI bounding box metadata was posted before we inclduded width and height as semi-standard
+ fields; pull size information from the main metadata file and add to the bbox file.
+
+"""
+
+#%% Constants and environment
+
+import json
+from tqdm import tqdm
+
+input_file = 'G:/temp/nacti_metadata.json'
+input_bbox_file = 'G:/temp/nacti_20200401_bboxes.json'
+output_bbox_file = 'G:/temp/nacti_20230920_bboxes.json'
+
+
+#%% Read .json files
+
+with open(input_file,'r') as f:
+    input_data = json.load(f)
+
+with open(input_bbox_file,'r') as f:
+    input_bbox_data = json.load(f)
+    
+print('Finished reading .json data')
+
+
+#%% Map image names to width and height
+
+filename_to_size = {}
+for im in tqdm(input_data['images']):
+    filename_to_size[im['file_name']] = (im['width'],im['height'])
+    
+
+#%% Add to output data
+
+for im in tqdm(input_bbox_data['images']):
+    size = filename_to_size[im['file_name']]
+    im['width'] = size[0]
+    im['height'] = size[1]
+    
+
+#%% Write output
+
+output_bbox_data = input_bbox_data
+output_bbox_data['version'] = '2023-09-20'
+
+with open(output_bbox_file,'w') as f:
+    json.dump(output_bbox_data,f,indent=1)

megadetector/data_management/importers/add_timestamps_to_icct.py

@@ -0,0 +1,79 @@
+"""
+
+ add_timestamps_to_icct.py
+
+ The Island Conservation Camera Traps dataset was originally posted without timestamps
+ in either .json metadata or EXIF metadata.  We pulled timestamps out using ocr_tools.py,
+ this script adds those timestamps into the .json metadata.
+
+"""
+
+#%% Imports and constants
+
+import json
+
+ocr_results_file = r'g:\temp\ocr_results.2023.10.31.07.37.54.json'
+input_metadata_file = r'd:\lila\islandconservationcameratraps\island_conservation.json'
+output_metadata_file = r'g:\temp\island_conservation_camera_traps_1.02.json'
+ocr_results_file_base = 'g:/temp/island_conservation_camera_traps/'
+assert ocr_results_file_base.endswith('/')
+
+
+#%% Read input metadata
+
+with open(input_metadata_file,'r') as f:
+    input_metadata = json.load(f)
+    
+assert input_metadata['info']['version'] == '1.01'
+
+# im = input_metadata['images'][0]
+for im in input_metadata['images']:
+    assert 'datetime' not in im
+    
+
+#%% Read OCR results
+
+with open(ocr_results_file,'r') as f:
+    abs_filename_to_ocr_results = json.load(f)
+    
+relative_filename_to_ocr_results = {}
+
+for fn_abs in abs_filename_to_ocr_results:
+    assert ocr_results_file_base in fn_abs
+    fn_relative = fn_abs.replace(ocr_results_file_base,'')
+    relative_filename_to_ocr_results[fn_relative] = abs_filename_to_ocr_results[fn_abs]
+    
+
+#%% Add datetimes to metadata
+
+images_not_in_datetime_results = []
+images_with_failed_datetimes = []
+
+for i_image,im in enumerate(input_metadata['images']):
+    if im['file_name'] not in relative_filename_to_ocr_results:
+        images_not_in_datetime_results.append(im)
+        im['datetime'] = None
+        continue
+    ocr_results = relative_filename_to_ocr_results[im['file_name']]
+    if ocr_results['datetime'] is None:
+        images_with_failed_datetimes.append(im)
+        im['datetime'] = None
+        continue
+    im['datetime'] = ocr_results['datetime']
+
+print('{} of {} images were not in datetime results'.format(
+    len(images_not_in_datetime_results),len(input_metadata['images'])))
+
+print('{} of {} images were had failed datetime results'.format(
+    len(images_with_failed_datetimes),len(input_metadata['images'])))
+
+for im in input_metadata['images']:
+    assert 'datetime' in im
+
+
+#%% Write output
+
+input_metadata['info']['version'] = '1.02'
+
+with open(output_metadata_file,'w') as f:
+    json.dump(input_metadata,f,indent=1)