megadetector 5.0.11__py3-none-any.whl → 5.0.12__py3-none-any.whl

megadetector/data_management/remap_coco_categories.py

@@ -0,0 +1,84 @@
+"""
+
+remap_coco_categories.py
+
+Given a COCO-formatted dataset, remap the categories to a new mapping.
+
+"""
+
+#%% Imports and constants
+
+import os
+import json
+
+from copy import deepcopy
+
+
+#%% Main function
+
+def remap_coco_categories(input_data,
+                          output_category_name_to_id,
+                          input_category_name_to_output_category_name,
+                          output_file=None):
+    """
+    Given a COCO-formatted dataset, remap the categories to a new categories mapping, optionally
+    writing the results to a new file.
+    
+    output_category_name_to_id is a dict mapping strings to ints.
+    
+    input_category_name_to_output_category_name is a dict mapping strings to strings.
+    
+    [input_data] can be a COCO-formatted dict or a filename.  If it's a dict, it will be copied,
+    not modified in place.
+    """
+    
+    if isinstance(input_data,str):
+        assert os.path.isfile(input_data), "Can't find file {}".format(input_data)
+        with open(input_data,'r') as f:
+            input_data = json.load(f)
+        assert isinstance(input_data,dict), 'Illegal COCO input data'
+    else:
+        assert isinstance(input_data,dict), 'Illegal COCO input data'
+        input_data = deepcopy(input_data)
+    
+    # It's safe to modify in-place now
+    output_data = input_data
+    
+    # Read input name --> ID mapping
+    input_category_name_to_input_category_id = {}
+    for c in input_data['categories']:
+        input_category_name_to_input_category_id[c['name']] = c['id']
+    
+    # Map input IDs --> output IDs
+    input_category_id_to_output_category_id = {}
+    for input_name in input_category_name_to_output_category_name.keys():
+        output_name = input_category_name_to_output_category_name[input_name]
+        assert output_name in output_category_name_to_id, \
+            'No output ID for {} --> {}'.format(input_name,output_name)
+        input_id = input_category_name_to_input_category_id[input_name]
+        output_id = output_category_name_to_id[output_name]
+        input_category_id_to_output_category_id[input_id] = output_id
+        
+    # Map annotations
+    for ann in output_data['annotations']:
+        assert ann['category_id'] in input_category_id_to_output_category_id, \
+            'Unrecognized category ID {}'.format(ann['category_id'])
+        ann['category_id'] = input_category_id_to_output_category_id[ann['category_id']]
+        
+    # Update the category list
+    output_categories = []
+    for output_name in output_category_name_to_id:
+        category = {'name':output_name,'id':output_category_name_to_id[output_name]}
+        output_categories.append(category)
+    output_data['categories'] = output_categories
+        
+    if output_file is not None:
+        with open(output_file,'w') as f:
+            json.dump(output_data,f,indent=1)
+            
+    return input_data
+
+
+#%% Command-line driver
+
+# TODO

megadetector/data_management/remove_exif.py

@@ -0,0 +1,66 @@
+"""
+
+remove_exif.py
+
+Removes all EXIF/IPTC/XMP metadata from a folder of images, without making 
+backup copies, using pyexiv2.
+
+TODO: This is a one-off script waiting to be cleaned up for more general use.
+
+"""
+
+input_base = r'f:\images'
+
+
+#%% Imports and constants
+
+import os
+import glob
+
+def main():
+
+    assert os.path.isdir(input_base)
+
+    ##%% List files
+
+    all_files = [f for f in glob.glob(input_base + "*/**", recursive=True)]
+    image_files = [s for s in all_files if (s.lower().endswith('.jpg'))]
+        
+
+    ##%% Remove EXIF data (support)
+
+    import pyexiv2
+
+    # PYEXIV2 IS NOT THREAD SAFE; DO NOT CALL THIS IN PARALLEL FROM A SINGLE PROCESS
+    def remove_exif(fn):
+        
+        try:
+            img = pyexiv2.Image(fn)
+            # data = img.read_exif(); print(data)
+            img.clear_exif()
+            img.clear_iptc()
+            img.clear_xmp()
+            img.close()        
+        except Exception as e:
+            print('EXIF error on {}: {}'.format(fn,str(e)))
+        
+
+    ##%% Remove EXIF data (execution)
+
+    from joblib import Parallel, delayed
+
+    n_exif_threads = 50
+        
+    if n_exif_threads == 1:
+        
+        # fn = image_files[0]
+        for fn in image_files:
+            remove_exif(fn)
+            
+    else:
+        # joblib.Parallel defaults to a process-based backend, but let's be sure
+        # results = Parallel(n_jobs=n_exif_threads,verbose=2,prefer='processes')(delayed(remove_exif)(fn) for fn in image_files[0:10])
+        _ = Parallel(n_jobs=n_exif_threads,verbose=2,prefer='processes')(delayed(remove_exif)(fn) for fn in image_files)
+            
+if __name__ == '__main__':
+    main()

megadetector/data_management/resize_coco_dataset.py

@@ -0,0 +1,189 @@
+"""
+
+resize_coco_dataset.py
+
+Given a COCO-formatted dataset, resizes all the images to a target size,
+scaling bounding boxes accordingly.
+
+"""
+
+#%% Imports and constants
+
+import os
+import json
+import shutil
+
+from tqdm import tqdm
+from collections import defaultdict
+
+from megadetector.utils.path_utils import insert_before_extension
+from megadetector.visualization.visualization_utils import \
+    open_image, resize_image, exif_preserving_save
+
+
+#%% Functions
+
+def resize_coco_dataset(input_folder,input_filename,
+                        output_folder,output_filename,
+                        target_size=(-1,-1),
+                        correct_size_image_handling='copy'):
+    """
+    Given a COCO-formatted dataset (images in input_folder, data in input_filename), resizes
+    all the images to a target size (in output_folder) and scales bounding boxes accordingly.
+    
+    Args:
+        input_folder (str): the folder where images live; filenames in [input_filename] should 
+            be relative to [input_folder]
+        input_filename (str): the (input) COCO-formatted .json file containing annotations
+        output_folder (str): the folder to which we should write resized images; can be the
+            same as [input_folder], in which case images are over-written
+        output_filename (str): the COCO-formatted .json file we should generate that refers to
+            the resized images
+        target_size (list or tuple of ints): this should be tuple/list of ints, with length 2 (w,h).
+            If either dimension is -1, aspect ratio will be preserved.  If both dimensions are -1, this means 
+            "keep the original size".  If  both dimensions are -1 and correct_size_image_handling is copy, this 
+            function is basically a no-op.    
+        correct_size_image_handling (str): can be 'copy' (in which case the original image is just copied 
+            to the output folder) or 'rewrite' (in which case the image is opened via PIL and re-written,
+            attempting to preserve the same quality).  The only reason to do use 'rewrite' 'is the case where 
+            you're superstitious about biases coming from images in a training set being written by different 
+            image encoders.
+   
+    Returns:
+        dict: the COCO database with resized images, identical to the content of [output_filename]
+    """
+    
+    # Read input data
+    with open(input_filename,'r') as f:
+        d = json.load(f)
+    
+    # Map image IDs to annotations
+    image_id_to_annotations = defaultdict(list)
+    for ann in d['annotations']:
+        image_id_to_annotations[ann['image_id']].append(ann)
+                                          
+    # For each image
+    
+    # TODO: this is trivially parallelizable
+    #
+    # im = d['images'][0]
+    for im in tqdm(d['images']):
+    
+        input_fn_relative = im['file_name']
+        input_fn_abs = os.path.join(input_folder,input_fn_relative)
+        assert os.path.isfile(input_fn_abs), "Can't find image file {}".format(input_fn_abs)
+        
+        output_fn_abs = os.path.join(output_folder,input_fn_relative)
+        os.makedirs(os.path.dirname(output_fn_abs),exist_ok=True)
+        
+        pil_im = open_image(input_fn_abs)
+        input_w = pil_im.width
+        input_h = pil_im.height
+        
+        image_is_already_target_size = \
+            (input_w == target_size[0]) and (input_h == target_size[1])
+        preserve_original_size = \
+            (target_size[0] == -1) and (target_size[1] == -1)
+            
+        # If the image is already the right size...
+        if (image_is_already_target_size or preserve_original_size):
+            output_w = input_w
+            output_h = input_h
+            if correct_size_image_handling == 'copy':
+                shutil.copyfile(input_fn_abs,output_fn_abs)
+            elif correct_size_image_handling == 'rewrite':
+                exif_preserving_save(pil_im,output_fn_abs)
+            else:
+                raise ValueError('Unrecognized value {} for correct_size_image_handling'.format(
+                    correct_size_image_handling))
+        else:
+            pil_im = resize_image(pil_im, target_size[0], target_size[1])
+            output_w = pil_im.width
+            output_h = pil_im.height
+            exif_preserving_save(pil_im,output_fn_abs)
+
+        im['width'] = output_w
+        im['height'] = output_h
+        
+        # For each box
+        annotations_this_image = image_id_to_annotations[im['id']]
+        
+        # ann = annotations_this_image[0]
+        for ann in annotations_this_image:
+            
+            if 'bbox' in ann:
+        
+                # boxes are [x,y,w,h]
+                bbox = ann['bbox']
+                
+                # Do we need to scale this box?
+                if (output_w != input_w) or (output_h != input_h):
+                    width_scale = output_w/input_w
+                    height_scale = output_h/input_h
+                    bbox = \
+                           [bbox[0] * width_scale,
+                            bbox[1] * height_scale,
+                            bbox[2] * width_scale,
+                            bbox[3] * height_scale]
+                
+                ann['bbox'] = bbox
+            
+            # ...if this annotation has a box
+    
+        # ...for each annotation
+    
+    # ...for each image
+    
+    # Write output file
+    with open(output_filename,'w') as f:
+        json.dump(d,f,indent=1)
+    
+    return d
+
+# ...def resize_coco_dataset(...)
+    
+
+#%% Interactive driver
+
+if False:
+    
+    pass
+
+    #%% Test resizing
+        
+    input_folder = os.path.expanduser('~/data/usgs-tegus/usgs-kissel-training')
+    input_filename = os.path.expanduser('~/data/usgs-tegus/usgs-kissel-training.json')
+    target_size = (1600,-1)
+    
+    output_filename = insert_before_extension(input_filename,'resized-test')
+    output_folder = input_folder + '-resized-test'
+    
+    correct_size_image_handling = 'rewrite'
+    
+    resize_coco_dataset(input_folder,input_filename,
+                        output_folder,output_filename,
+                        target_size=target_size,
+                        correct_size_image_handling=correct_size_image_handling)
+    
+    
+    #%% Preview
+    
+    from megadetector.visualization import visualize_db
+    options = visualize_db.DbVizOptions()
+    options.parallelize_rendering = True
+    options.viz_size = (900, -1)
+    options.num_to_visualize = 5000
+
+    html_file,_ = visualize_db.visualize_db(output_filename,
+                                              os.path.expanduser('~/tmp/resize_coco_preview'),
+                                              output_folder,options)
+    
+
+    from megadetector.utils import path_utils # noqa
+    path_utils.open_file(html_file)
+    
+    
+#%% Command-line driver
+
+# TODO
+

megadetector/data_management/wi_download_csv_to_coco.py

@@ -0,0 +1,246 @@
+"""
+
+wi_download_csv_to_coco.py
+
+Converts a .csv file from a Wildlife Insights project export to a COCO camera traps .json file.
+
+Currently assumes that common names are unique identifiers, which is convenient but unreliable.
+
+"""
+
+#%% Imports and constants
+
+import os
+import json
+import pandas as pd
+
+from tqdm import tqdm
+from collections import defaultdict
+
+from megadetector.visualization import visualization_utils as vis_utils
+from megadetector.utils.ct_utils import isnan
+
+wi_extra_annotation_columns = \
+    ('is_blank','identified_by','wi_taxon_id','class','order','family','genus','species','uncertainty',
+             'number_of_objects','age','sex','animal_recognizable','individual_id','individual_animal_notes',
+             'behavior','highlighted','markings')
+
+wi_extra_image_columns = ('project_id','deployment_id')
+
+def _make_location_id(project_id,deployment_id):    
+    return 'project_' + str(project_id) + '_deployment_' + deployment_id
+
+default_category_remappings = {
+    'Homo Species':'Human',
+    'Human-Camera Trapper':'Human',
+    'No CV Result':'Unknown'
+}
+
+
+#%% Main function
+
+def wi_download_csv_to_coco(csv_file_in,
+                            coco_file_out=None,
+                            image_folder=None,
+                            validate_images=False,
+                            gs_prefix=None,
+                            verbose=True,
+                            category_remappings=default_category_remappings):
+    """
+    Converts a .csv file from a Wildlife Insights project export to a COCO 
+    Camera Traps .json file.
+    
+    Args:
+        csv_file_in (str): the downloaded .csv file we should convert to COCO
+        coco_file_out (str, optional): the .json file we should write; if [coco_file_out] is None, 
+            uses [csv_file_in].json
+        image_folder (str, optional): the folder where images live, only relevant if 
+            [validate_images] is True
+        validate_images (bool, optional): whether to check images for corruption and load
+            image sizes; if this is True, [image_folder] must be a valid folder
+        gs_prefix (str, optional): a string to remove from GS URLs to convert to path names... 
+            for example, if your gs:// URLs look like:
+    
+            `gs://11234134_xyz/deployment/55554/dfadfasdfs.jpg`
+    
+            ...and you specify gs_prefix='11234134_xyz/deployment/', the filenames in
+            the .json file will look like:
+        
+            `55554/dfadfasdfs.jpg`
+        verbose (bool, optional): enable additional debug console output
+        category_remappings (dict, optional): str --> str dict that maps any number of
+            WI category names to output category names; for example defaults to mapping
+            "Homo Species" to "Human", but leaves 99.99% of categories unchanged.        
+            
+    Returns: 
+        dict: COCO-formatted data, identical to what's written to [coco_file_out]
+    """
+    
+    ##%% Create COCO dictionaries
+    
+    category_name_to_id = {}
+    category_name_to_id['empty'] = 0
+    
+    df = pd.read_csv(csv_file_in)
+    
+    print('Read {} rows from {}'.format(len(df),csv_file_in))
+    
+    image_id_to_image = {}
+    image_id_to_annotations = defaultdict(list)
+    
+    # i_row = 0; row = df.iloc[i_row]
+    for i_row,row in df.iterrows():
+        
+        image_id = row['image_id']
+        
+        if image_id not in image_id_to_image:
+            
+            im = {}
+            image_id_to_image[image_id] = im
+            
+            im['id'] = image_id
+            
+            gs_url = row['location']
+            assert gs_url.startswith('gs://')
+            
+            file_name = gs_url.replace('gs://','')
+            if gs_prefix is not None:
+                file_name = file_name.replace(gs_prefix,'')
+                
+            location_id = _make_location_id(row['project_id'],row['deployment_id'])
+            im['file_name'] = file_name
+            im['location'] = location_id
+            im['datetime'] = row['timestamp']
+            
+            im['wi_image_info'] = {}
+            for s in wi_extra_image_columns:
+                im['wi_image_info'][s] = str(row[s])
+            
+        else:
+            
+            im = image_id_to_image[image_id]
+            assert im['datetime'] == row['timestamp']
+            location_id = _make_location_id(row['project_id'],row['deployment_id'])
+            assert im['location'] == location_id
+            
+        category_name = row['common_name']
+        if category_remappings is not None and category_name in category_remappings:
+            category_name = category_remappings[category_name]
+            
+        if category_name == 'Blank':
+            category_name = 'empty'
+            assert row['is_blank'] == 1
+        else:
+            assert row['is_blank'] == 0
+        assert isinstance(category_name,str)
+        if category_name in category_name_to_id:
+            category_id = category_name_to_id[category_name]
+        else:
+            category_id = len(category_name_to_id)
+            category_name_to_id[category_name] = category_id
+        
+        ann = {}
+        ann['image_id'] = image_id
+        annotations_this_image = image_id_to_annotations[image_id]
+        annotation_number = len(annotations_this_image)
+        ann['id'] = image_id + '_' + str(annotation_number).zfill(2)        
+        ann['category_id'] = category_id
+        annotations_this_image.append(ann)
+        
+        extra_info = {}
+        for s in wi_extra_annotation_columns:            
+            v = row[s]
+            if not isnan(v):
+                extra_info[s] = v
+        ann['wi_extra_info'] = extra_info
+        
+    # ...for each row
+    
+    images = list(image_id_to_image.values())
+    categories = []
+    for category_name in category_name_to_id:
+        category_id = category_name_to_id[category_name]
+        categories.append({'id':category_id,'name':category_name})
+    annotations = []
+    for image_id in image_id_to_annotations:
+        annotations_this_image = image_id_to_annotations[image_id]
+        for ann in annotations_this_image:
+            annotations.append(ann)
+    info = {'version':'1.00','description':'converted from WI export'}
+    info['source_file'] = csv_file_in
+    coco_data = {}
+    coco_data['info'] = info
+    coco_data['images'] = images
+    coco_data['annotations'] = annotations
+    coco_data['categories'] = categories
+    
+    
+    ##%% Validate images, add sizes        
+    
+    if validate_images:
+        
+        print('Validating images')
+        # TODO: trivially parallelizable
+        
+        assert os.path.isdir(image_folder), \
+            'Must specify a valid image folder if you specify validate_images=True'
+            
+        # im = images[0]
+        for im in tqdm(images):
+            file_name_relative = im['file_name']
+            file_name_abs = os.path.join(image_folder,file_name_relative)
+            assert os.path.isfile(file_name_abs)
+                
+            im['corrupt'] = False
+            try:
+                pil_im = vis_utils.load_image(file_name_abs)
+            except Exception:
+                im['corrupt'] = True
+            if not im['corrupt']:
+                im['width'] = pil_im.width
+                im['height'] = pil_im.height
+    
+    
+    ##%% Write output json
+        
+    if coco_file_out is None:        
+        coco_file_out = csv_file_in + '.json'
+        
+    with open(coco_file_out,'w') as f:
+        json.dump(coco_data,f,indent=1)
+
+
+    ##%% Validate output
+    
+    from megadetector.data_management.databases.integrity_check_json_db import \
+        IntegrityCheckOptions,integrity_check_json_db
+    options = IntegrityCheckOptions()
+    options.baseDir = image_folder
+    options.bCheckImageExistence = True
+    options.verbose = verbose
+    _ = integrity_check_json_db(coco_file_out,options)
+    
+    return coco_data
+
+# ...def wi_download_csv_to_coco(...)    
+
+
+#%% Interactive driver
+
+if False:
+    
+    #%%
+    
+    base_folder = r'a/b/c'
+    csv_file_in = os.path.join(base_folder,'images.csv')
+    coco_file_out = None
+    gs_prefix = 'a_b_c_main/'
+    image_folder = os.path.join(base_folder,'images')
+    validate_images = False
+    verbose = True
+    category_remappings = default_category_remappings
+
+
+#%% Command-line driver
+
+# TODO