megadetector 5.0.9__py3-none-any.whl → 5.0.11__py3-none-any.whl

data_management/remap_coco_categories.py

@@ -1,84 +0,0 @@
-"""
-
-remap_coco_categories.py
-
-Given a COCO-formatted dataset, remap the categories to a new mapping.
-
-"""
-
-#%% Imports and constants
-
-import os
-import json
-
-from copy import deepcopy
-
-
-#%% Main function
-
-def remap_coco_categories(input_data,
-                          output_category_name_to_id,
-                          input_category_name_to_output_category_name,
-                          output_file=None):
-    """
-    Given a COCO-formatted dataset, remap the categories to a new categories mapping, optionally
-    writing the results to a new file.
-    
-    output_category_name_to_id is a dict mapping strings to ints.
-    
-    input_category_name_to_output_category_name is a dict mapping strings to strings.
-    
-    [input_data] can be a COCO-formatted dict or a filename.  If it's a dict, it will be copied,
-    not modified in place.
-    """
-    
-    if isinstance(input_data,str):
-        assert os.path.isfile(input_data), "Can't find file {}".format(input_data)
-        with open(input_data,'r') as f:
-            input_data = json.load(f)
-        assert isinstance(input_data,dict), 'Illegal COCO input data'
-    else:
-        assert isinstance(input_data,dict), 'Illegal COCO input data'
-        input_data = deepcopy(input_data)
-    
-    # It's safe to modify in-place now
-    output_data = input_data
-    
-    # Read input name --> ID mapping
-    input_category_name_to_input_category_id = {}
-    for c in input_data['categories']:
-        input_category_name_to_input_category_id[c['name']] = c['id']
-    
-    # Map input IDs --> output IDs
-    input_category_id_to_output_category_id = {}
-    for input_name in input_category_name_to_output_category_name.keys():
-        output_name = input_category_name_to_output_category_name[input_name]
-        assert output_name in output_category_name_to_id, \
-            'No output ID for {} --> {}'.format(input_name,output_name)
-        input_id = input_category_name_to_input_category_id[input_name]
-        output_id = output_category_name_to_id[output_name]
-        input_category_id_to_output_category_id[input_id] = output_id
-        
-    # Map annotations
-    for ann in output_data['annotations']:
-        assert ann['category_id'] in input_category_id_to_output_category_id, \
-            'Unrecognized category ID {}'.format(ann['category_id'])
-        ann['category_id'] = input_category_id_to_output_category_id[ann['category_id']]
-        
-    # Update the category list
-    output_categories = []
-    for output_name in output_category_name_to_id:
-        category = {'name':output_name,'id':output_category_name_to_id[output_name]}
-        output_categories.append(category)
-    output_data['categories'] = output_categories
-        
-    if output_file is not None:
-        with open(output_file,'w') as f:
-            json.dump(output_data,f,indent=1)
-            
-    return input_data
-
-
-#%% Command-line driver
-
-# TODO

data_management/remove_exif.py

@@ -1,66 +0,0 @@
-"""
-
-remove_exif.py
-
-Removes all EXIF/IPTC/XMP metadata from a folder of images, without making 
-backup copies, using pyexiv2.
-
-TODO: This is a one-off script waiting to be cleaned up for more general use.
-
-"""
-
-input_base = r'f:\images'
-
-
-#%% Imports and constants
-
-import os
-import glob
-
-def main():
-
-    assert os.path.isdir(input_base)
-
-    ##%% List files
-
-    all_files = [f for f in glob.glob(input_base + "*/**", recursive=True)]
-    image_files = [s for s in all_files if (s.lower().endswith('.jpg'))]
-        
-
-    ##%% Remove EXIF data (support)
-
-    import pyexiv2
-
-    # PYEXIV2 IS NOT THREAD SAFE; DO NOT CALL THIS IN PARALLEL FROM A SINGLE PROCESS
-    def remove_exif(fn):
-        
-        try:
-            img = pyexiv2.Image(fn)
-            # data = img.read_exif(); print(data)
-            img.clear_exif()
-            img.clear_iptc()
-            img.clear_xmp()
-            img.close()        
-        except Exception as e:
-            print('EXIF error on {}: {}'.format(fn,str(e)))
-        
-
-    ##%% Remove EXIF data (execution)
-
-    from joblib import Parallel, delayed
-
-    n_exif_threads = 50
-        
-    if n_exif_threads == 1:
-        
-        # fn = image_files[0]
-        for fn in image_files:
-            remove_exif(fn)
-            
-    else:
-        # joblib.Parallel defaults to a process-based backend, but let's be sure
-        # results = Parallel(n_jobs=n_exif_threads,verbose=2,prefer='processes')(delayed(remove_exif)(fn) for fn in image_files[0:10])
-        _ = Parallel(n_jobs=n_exif_threads,verbose=2,prefer='processes')(delayed(remove_exif)(fn) for fn in image_files)
-            
-if __name__ == '__main__':
-    main()

data_management/resize_coco_dataset.py

@@ -1,189 +0,0 @@
-"""
-
-resize_coco_dataset.py
-
-Given a COCO-formatted dataset, resizes all the images to a target size,
-scaling bounding boxes accordingly.
-
-"""
-
-#%% Imports and constants
-
-import os
-import json
-import shutil
-
-from tqdm import tqdm
-from collections import defaultdict
-
-from md_utils.path_utils import insert_before_extension
-from md_visualization.visualization_utils import \
-    open_image, resize_image, exif_preserving_save
-
-
-#%% Functions
-
-def resize_coco_dataset(input_folder,input_filename,
-                        output_folder,output_filename,
-                        target_size=(-1,-1),
-                        correct_size_image_handling='copy'):
-    """
-    Given a COCO-formatted dataset (images in input_folder, data in input_filename), resizes
-    all the images to a target size (in output_folder) and scales bounding boxes accordingly.
-    
-    Args:
-        input_folder (str): the folder where images live; filenames in [input_filename] should 
-            be relative to [input_folder]
-        input_filename (str): the (input) COCO-formatted .json file containing annotations
-        output_folder (str): the folder to which we should write resized images; can be the
-            same as [input_folder], in which case images are over-written
-        output_filename (str): the COCO-formatted .json file we should generate that refers to
-            the resized images
-        target_size (list or tuple of ints): this should be tuple/list of ints, with length 2 (w,h).
-            If either dimension is -1, aspect ratio will be preserved.  If both dimensions are -1, this means 
-            "keep the original size".  If  both dimensions are -1 and correct_size_image_handling is copy, this 
-            function is basically a no-op.    
-        correct_size_image_handling (str): can be 'copy' (in which case the original image is just copied 
-            to the output folder) or 'rewrite' (in which case the image is opened via PIL and re-written,
-            attempting to preserve the same quality).  The only reason to do use 'rewrite' 'is the case where 
-            you're superstitious about biases coming from images in a training set being written by different 
-            image encoders.
-   
-    Returns:
-        dict: the COCO database with resized images, identical to the content of [output_filename]
-    """
-    
-    # Read input data
-    with open(input_filename,'r') as f:
-        d = json.load(f)
-    
-    # Map image IDs to annotations
-    image_id_to_annotations = defaultdict(list)
-    for ann in d['annotations']:
-        image_id_to_annotations[ann['image_id']].append(ann)
-                                          
-    # For each image
-    
-    # TODO: this is trivially parallelizable
-    #
-    # im = d['images'][0]
-    for im in tqdm(d['images']):
-    
-        input_fn_relative = im['file_name']
-        input_fn_abs = os.path.join(input_folder,input_fn_relative)
-        assert os.path.isfile(input_fn_abs), "Can't find image file {}".format(input_fn_abs)
-        
-        output_fn_abs = os.path.join(output_folder,input_fn_relative)
-        os.makedirs(os.path.dirname(output_fn_abs),exist_ok=True)
-        
-        pil_im = open_image(input_fn_abs)
-        input_w = pil_im.width
-        input_h = pil_im.height
-        
-        image_is_already_target_size = \
-            (input_w == target_size[0]) and (input_h == target_size[1])
-        preserve_original_size = \
-            (target_size[0] == -1) and (target_size[1] == -1)
-            
-        # If the image is already the right size...
-        if (image_is_already_target_size or preserve_original_size):
-            output_w = input_w
-            output_h = input_h
-            if correct_size_image_handling == 'copy':
-                shutil.copyfile(input_fn_abs,output_fn_abs)
-            elif correct_size_image_handling == 'rewrite':
-                exif_preserving_save(pil_im,output_fn_abs)
-            else:
-                raise ValueError('Unrecognized value {} for correct_size_image_handling'.format(
-                    correct_size_image_handling))
-        else:
-            pil_im = resize_image(pil_im, target_size[0], target_size[1])
-            output_w = pil_im.width
-            output_h = pil_im.height
-            exif_preserving_save(pil_im,output_fn_abs)
-
-        im['width'] = output_w
-        im['height'] = output_h
-        
-        # For each box
-        annotations_this_image = image_id_to_annotations[im['id']]
-        
-        # ann = annotations_this_image[0]
-        for ann in annotations_this_image:
-            
-            if 'bbox' in ann:
-        
-                # boxes are [x,y,w,h]
-                bbox = ann['bbox']
-                
-                # Do we need to scale this box?
-                if (output_w != input_w) or (output_h != input_h):
-                    width_scale = output_w/input_w
-                    height_scale = output_h/input_h
-                    bbox = \
-                           [bbox[0] * width_scale,
-                            bbox[1] * height_scale,
-                            bbox[2] * width_scale,
-                            bbox[3] * height_scale]
-                
-                ann['bbox'] = bbox
-            
-            # ...if this annotation has a box
-    
-        # ...for each annotation
-    
-    # ...for each image
-    
-    # Write output file
-    with open(output_filename,'w') as f:
-        json.dump(d,f,indent=1)
-    
-    return d
-
-# ...def resize_coco_dataset(...)
-    
-
-#%% Interactive driver
-
-if False:
-    
-    pass
-
-    #%% Test resizing
-        
-    input_folder = os.path.expanduser('~/data/usgs-tegus/usgs-kissel-training')
-    input_filename = os.path.expanduser('~/data/usgs-tegus/usgs-kissel-training.json')
-    target_size = (1600,-1)
-    
-    output_filename = insert_before_extension(input_filename,'resized-test')
-    output_folder = input_folder + '-resized-test'
-    
-    correct_size_image_handling = 'rewrite'
-    
-    resize_coco_dataset(input_folder,input_filename,
-                        output_folder,output_filename,
-                        target_size=target_size,
-                        correct_size_image_handling=correct_size_image_handling)
-    
-    
-    #%% Preview
-    
-    from md_visualization import visualize_db
-    options = visualize_db.DbVizOptions()
-    options.parallelize_rendering = True
-    options.viz_size = (900, -1)
-    options.num_to_visualize = 5000
-
-    html_file,_ = visualize_db.visualize_db(output_filename,
-                                              os.path.expanduser('~/tmp/resize_coco_preview'),
-                                              output_folder,options)
-    
-
-    from md_utils import path_utils # noqa
-    path_utils.open_file(html_file)
-    
-    
-#%% Command-line driver
-
-# TODO
-

data_management/wi_download_csv_to_coco.py

@@ -1,246 +0,0 @@
-"""
-
-wi_download_csv_to_coco.py
-
-Converts a .csv file from a Wildlife Insights project export to a COCO camera traps .json file.
-
-Currently assumes that common names are unique identifiers, which is convenient but unreliable.
-
-"""
-
-#%% Imports and constants
-
-import os
-import json
-import pandas as pd
-
-from tqdm import tqdm
-from collections import defaultdict
-
-from md_visualization import visualization_utils as vis_utils
-from md_utils.ct_utils import isnan
-
-wi_extra_annotation_columns = \
-    ('is_blank','identified_by','wi_taxon_id','class','order','family','genus','species','uncertainty',
-             'number_of_objects','age','sex','animal_recognizable','individual_id','individual_animal_notes',
-             'behavior','highlighted','markings')
-
-wi_extra_image_columns = ('project_id','deployment_id')
-
-def _make_location_id(project_id,deployment_id):    
-    return 'project_' + str(project_id) + '_deployment_' + deployment_id
-
-default_category_remappings = {
-    'Homo Species':'Human',
-    'Human-Camera Trapper':'Human',
-    'No CV Result':'Unknown'
-}
-
-
-#%% Main function
-
-def wi_download_csv_to_coco(csv_file_in,
-                            coco_file_out=None,
-                            image_folder=None,
-                            validate_images=False,
-                            gs_prefix=None,
-                            verbose=True,
-                            category_remappings=default_category_remappings):
-    """
-    Converts a .csv file from a Wildlife Insights project export to a COCO 
-    Camera Traps .json file.
-    
-    Args:
-        csv_file_in (str): the downloaded .csv file we should convert to COCO
-        coco_file_out (str, optional): the .json file we should write; if [coco_file_out] is None, 
-            uses [csv_file_in].json
-        image_folder (str, optional): the folder where images live, only relevant if 
-            [validate_images] is True
-        validate_images (bool, optional): whether to check images for corruption and load
-            image sizes; if this is True, [image_folder] must be a valid folder
-        gs_prefix (str, optional): a string to remove from GS URLs to convert to path names... 
-            for example, if your gs:// URLs look like:
-    
-            `gs://11234134_xyz/deployment/55554/dfadfasdfs.jpg`
-    
-            ...and you specify gs_prefix='11234134_xyz/deployment/', the filenames in
-            the .json file will look like:
-        
-            `55554/dfadfasdfs.jpg`
-        verbose (bool, optional): enable additional debug console output
-        category_remappings (dict, optional): str --> str dict that maps any number of
-            WI category names to output category names; for example defaults to mapping
-            "Homo Species" to "Human", but leaves 99.99% of categories unchanged.        
-            
-    Returns: 
-        dict: COCO-formatted data, identical to what's written to [coco_file_out]
-    """
-    
-    ##%% Create COCO dictionaries
-    
-    category_name_to_id = {}
-    category_name_to_id['empty'] = 0
-    
-    df = pd.read_csv(csv_file_in)
-    
-    print('Read {} rows from {}'.format(len(df),csv_file_in))
-    
-    image_id_to_image = {}
-    image_id_to_annotations = defaultdict(list)
-    
-    # i_row = 0; row = df.iloc[i_row]
-    for i_row,row in df.iterrows():
-        
-        image_id = row['image_id']
-        
-        if image_id not in image_id_to_image:
-            
-            im = {}
-            image_id_to_image[image_id] = im
-            
-            im['id'] = image_id
-            
-            gs_url = row['location']
-            assert gs_url.startswith('gs://')
-            
-            file_name = gs_url.replace('gs://','')
-            if gs_prefix is not None:
-                file_name = file_name.replace(gs_prefix,'')
-                
-            location_id = _make_location_id(row['project_id'],row['deployment_id'])
-            im['file_name'] = file_name
-            im['location'] = location_id
-            im['datetime'] = row['timestamp']
-            
-            im['wi_image_info'] = {}
-            for s in wi_extra_image_columns:
-                im['wi_image_info'][s] = str(row[s])
-            
-        else:
-            
-            im = image_id_to_image[image_id]
-            assert im['datetime'] == row['timestamp']
-            location_id = _make_location_id(row['project_id'],row['deployment_id'])
-            assert im['location'] == location_id
-            
-        category_name = row['common_name']
-        if category_remappings is not None and category_name in category_remappings:
-            category_name = category_remappings[category_name]
-            
-        if category_name == 'Blank':
-            category_name = 'empty'
-            assert row['is_blank'] == 1
-        else:
-            assert row['is_blank'] == 0
-        assert isinstance(category_name,str)
-        if category_name in category_name_to_id:
-            category_id = category_name_to_id[category_name]
-        else:
-            category_id = len(category_name_to_id)
-            category_name_to_id[category_name] = category_id
-        
-        ann = {}
-        ann['image_id'] = image_id
-        annotations_this_image = image_id_to_annotations[image_id]
-        annotation_number = len(annotations_this_image)
-        ann['id'] = image_id + '_' + str(annotation_number).zfill(2)        
-        ann['category_id'] = category_id
-        annotations_this_image.append(ann)
-        
-        extra_info = {}
-        for s in wi_extra_annotation_columns:            
-            v = row[s]
-            if not isnan(v):
-                extra_info[s] = v
-        ann['wi_extra_info'] = extra_info
-        
-    # ...for each row
-    
-    images = list(image_id_to_image.values())
-    categories = []
-    for category_name in category_name_to_id:
-        category_id = category_name_to_id[category_name]
-        categories.append({'id':category_id,'name':category_name})
-    annotations = []
-    for image_id in image_id_to_annotations:
-        annotations_this_image = image_id_to_annotations[image_id]
-        for ann in annotations_this_image:
-            annotations.append(ann)
-    info = {'version':'1.00','description':'converted from WI export'}
-    info['source_file'] = csv_file_in
-    coco_data = {}
-    coco_data['info'] = info
-    coco_data['images'] = images
-    coco_data['annotations'] = annotations
-    coco_data['categories'] = categories
-    
-    
-    ##%% Validate images, add sizes        
-    
-    if validate_images:
-        
-        print('Validating images')
-        # TODO: trivially parallelizable
-        
-        assert os.path.isdir(image_folder), \
-            'Must specify a valid image folder if you specify validate_images=True'
-            
-        # im = images[0]
-        for im in tqdm(images):
-            file_name_relative = im['file_name']
-            file_name_abs = os.path.join(image_folder,file_name_relative)
-            assert os.path.isfile(file_name_abs)
-                
-            im['corrupt'] = False
-            try:
-                pil_im = vis_utils.load_image(file_name_abs)
-            except Exception:
-                im['corrupt'] = True
-            if not im['corrupt']:
-                im['width'] = pil_im.width
-                im['height'] = pil_im.height
-    
-    
-    ##%% Write output json
-        
-    if coco_file_out is None:        
-        coco_file_out = csv_file_in + '.json'
-        
-    with open(coco_file_out,'w') as f:
-        json.dump(coco_data,f,indent=1)
-
-
-    ##%% Validate output
-    
-    from data_management.databases.integrity_check_json_db import \
-        IntegrityCheckOptions,integrity_check_json_db
-    options = IntegrityCheckOptions()
-    options.baseDir = image_folder
-    options.bCheckImageExistence = True
-    options.verbose = verbose
-    _ = integrity_check_json_db(coco_file_out,options)
-    
-    return coco_data
-
-# ...def wi_download_csv_to_coco(...)    
-
-
-#%% Interactive driver
-
-if False:
-    
-    #%%
-    
-    base_folder = r'a/b/c'
-    csv_file_in = os.path.join(base_folder,'images.csv')
-    coco_file_out = None
-    gs_prefix = 'a_b_c_main/'
-    image_folder = os.path.join(base_folder,'images')
-    validate_images = False
-    verbose = True
-    category_remappings = default_category_remappings
-
-
-#%% Command-line driver
-
-# TODO