megadetector 5.0.28__py3-none-any.whl → 5.0.29__py3-none-any.whl

megadetector/data_management/importers/raic_csv_to_md_results.py

@@ -1,416 +0,0 @@
-"""
-
-raic_csv_to_md_results.py
-
-Converts classification+detection results in the .csv format provided to the Snapshot
-Serengeti program by the RAIC team to the MD results format.
-
-The input format is two .csv files:
-    
-* One with results, with columns [unnamed], filename, category, x_center, y_center,
-  width, height, confidence, datetime
-
-* One with class IDs and names, with columns CLASS, SPECIES
-
-Filenames are relative paths to .txt files, but with slashes replaced by underscores, e.g. this 
-file:
-    
-    B04_R1/I__00122.JPG
-    
-...appears in the .csv file as:
-    
-    B04_R1_I__00122.txt
-    
-Image coordinates are in absolute floating-point units, with an upper-left origin.
-  
-Unknowns at the time I'm writing this:
-    
-* I don't know what the unnamed column is, but it looks like an ID I can safely ignore.
-
-* I believe that MegaDetector was run, then a classifier was run, but there is a 
-  single "confidence" column in the output.  I am writing out the results as if they were a 
-  single multi-class detector.  This is suspicious given the lack of a human class, which suggests
-  that this is intended to be run in conjunection with MD.
-
-* There is no concept of "empty" in this file format, so by default I assume that images with
-  no annotations in the .csv file were processed and determine to have no detections above some
-  (unknown) threshold.
-  
-* I'm not currently handling EXIF rotations, as part of the effort to simplify this file
-  for conversion to R (see below).
-  
-Note to self: this file should not take dependencies on other components of the MD
-repo, at the risk of creating some redundancy.  I am going to convert this to R,
-which will be easier if it's not using any non-standard libraries.  Anything in the 
-"interactive driver" cells gets a pass.
-
-"""
-
-#%% Imports and constants
-
-import os
-import glob
-import json
-import sys
-import argparse
-
-import pandas as pd
-from PIL import Image
-
-
-#%% Functions from the MD python package
-
-# ...that I'm choosing to copy and paste to facilitate a conversion of this
-# script to R.
-
-# Should all be lower-case
-IMG_EXTENSIONS = ('.jpg', '.jpeg', '.gif', '.png', '.tif', '.tiff', '.bmp')
-
-def _is_image_file(s, img_extensions=IMG_EXTENSIONS):
-    """
-    Checks a file's extension against a hard-coded set of image file
-    extensions.  Uses case-insensitive comparison.
-    
-    Does not check whether the file exists, only determines whether the filename
-    implies it's an image file.
-    
-    Args:
-        s (str): filename to evaluate for image-ness
-        img_extensions (list, optional): list of known image file extensions
-        
-    Returns:
-        bool: True if [s] appears to be an image file, else False
-    """
-    
-    ext = os.path.splitext(s)[1]
-    return ext.lower() in img_extensions
-
-
-def _find_image_strings(strings):
-    """
-    Given a list of strings that are potentially image file names, looks for
-    strings that actually look like image file names (based on extension).
-    
-    Args:
-        strings (list): list of filenames to check for image-ness
-        
-    Returns:
-        list: the subset of [strings] that appear to be image filenames
-    """
-    
-    return [s for s in strings if _is_image_file(s)]
-
-
-def _find_images(dirname, 
-                recursive=False, 
-                return_relative_paths=False, 
-                convert_slashes=True):
-    """
-    Finds all files in a directory that look like image file names. Returns
-    absolute paths unless return_relative_paths is set.  Uses the OS-native
-    path separator unless convert_slashes is set, in which case will always
-    use '/'.
-    
-    Args:
-        dirname (str): the folder to search for images
-        recursive (bool, optional): whether to search recursively
-        return_relative_paths (str, optional): return paths that are relative
-            to [dirname], rather than absolute paths
-        convert_slashes (bool, optional): force forward slashes in return values
-
-    Returns:
-        list: list of image filenames found in [dirname]
-    """
-    
-    assert os.path.isdir(dirname), '{} is not a folder'.format(dirname)
-    
-    if recursive:
-        strings = glob.glob(os.path.join(dirname, '**', '*.*'), recursive=True)
-    else:
-        strings = glob.glob(os.path.join(dirname, '*.*'))
-    
-    image_files = _find_image_strings(strings)
-    
-    if return_relative_paths:
-        image_files = [os.path.relpath(fn,dirname) for fn in image_files]
-    
-    image_files = sorted(image_files)
-    
-    if convert_slashes:
-        image_files = [fn.replace('\\', '/') for fn in image_files]
-        
-    return image_files
-
-
-#%% Main conversion function
-
-def raic_csv_to_md_results(result_csv_file,
-                           class_mapping_csv_file,
-                           image_folder,
-                           output_file=None,
-                           unannotated_image_handling='empty'):
-    """
-    Converts a pair of .csv files (see file header for details) to MD results format.
-    
-    Currently errors if image filenames are ambiguous, or if any images referred to in 
-    the results are not available.
-    
-    Args:
-        result_csv_file (str): the results file to read (.csv)
-        class_mapping_csv_file (str): the class mapping file (.csv)
-        image_folder (str): the folder containing all the images referred to in
-            [result_csv_file]
-        output_file (str, optional): the .json file to which we should write results.  Defaults
-            to [result_csv_file].json
-        unannotated_image_handling (str, optional): can be "empty" (default) to assume
-            that images without annotations are empty, "warning", "error", or "skip"
-            
-    Returns:
-        str: the output file written, identical to [output_file] if [output_file] was not None
-    """
-    
-    # Validate arguments    
-    assert os.path.isfile(result_csv_file), \
-        'Result file {} not found'.format(result_csv_file)
-    assert os.path.isfile(class_mapping_csv_file), \
-        'Class mapping file {} not found'.format(class_mapping_csv_file)
-    assert os.path.isdir(image_folder), \
-        'Image folder {} not found'.format(image_folder)
-    
-    if output_file is None:
-        output_file = result_csv_file + '.json'
-        
-    image_files_relative = _find_images(image_folder,
-                                        recursive=True,
-                                        return_relative_paths=True,
-                                        convert_slashes=True)
-    image_file_base_flattened_to_image_file_relative = {}
-    for fn in image_files_relative:
-        # Convert, e.g. B04_R1/I__00108.JPG to B04_R1_I__00108
-        fn_flattened = fn.replace('/','_')
-        fn_flattened_base = os.path.splitext(fn_flattened)[0]
-        image_file_base_flattened_to_image_file_relative[fn_flattened_base] = \
-            fn    
-        
-    # Read the .csv files
-    df_results = pd.read_csv(result_csv_file)
-    df_class_mapping = pd.read_csv(class_mapping_csv_file)
-    
-    assert 'CLASS' in df_class_mapping.columns and 'SPECIES' in df_class_mapping.columns, \
-        'Unexpected column names in class mapping file {}'.format(class_mapping_csv_file)
-    
-    category_id_to_name = {}
-    for i_row,row in df_class_mapping.iterrows():
-        class_id = int(row['CLASS'])
-        assert class_id not in category_id_to_name, \
-            'Class ID {} occurs more than once in class mapping file {}'.format(
-                class_id,class_mapping_csv_file)
-        category_id_to_name[class_id] = row['SPECIES']
-    
-    if len(category_id_to_name) != len(set(category_id_to_name.values())):
-        print('Warning: one or more categories are used more than once in class mapping file {}'.format(
-            class_mapping_csv_file))
-        
-    # Convert results
-    
-    fn_relative_to_im = {}
-    
-    # i_row = 0; row = df_results.iloc[i_row]
-    for i_row,row in df_results.iterrows():
-        
-        # Map the .txt filename base to a relative path
-        bn = row['filename']
-        assert bn.lower().endswith('.txt')
-        bn_no_ext = os.path.splitext(bn)[0]
-        assert bn_no_ext in image_file_base_flattened_to_image_file_relative, \
-            'No image found for result row {}'.format(row['filename'])
-        
-        image_fn_relative = image_file_base_flattened_to_image_file_relative[bn_no_ext]
-        
-        # Have we seen another detection for this image?
-        if image_fn_relative in fn_relative_to_im:
-            
-            im = fn_relative_to_im[image_fn_relative]
-        
-        # If not, load this image so we can read its size
-        else:
-            
-            image_fn_abs = os.path.join(image_folder,image_fn_relative)
-            image = Image.open(image_fn_abs)
-            w = image.size[0]
-            h = image.size[1]
-            
-            im = {}
-            im['file'] = image_fn_relative
-            im['width'] = w
-            im['height'] = h
-            im['detections'] = []
-            im['datetime'] = str(row['datetime'])
-            fn_relative_to_im[image_fn_relative] = im
-            
-        # Convert annotation
-        x_center_abs = row['x_center']
-        y_center_abs = row['y_center']
-        box_width_abs = row['width']
-        box_height_abs = row['height']
-        
-        # Convert to relative coordinates
-        box_left_abs = x_center_abs - (box_width_abs/2.0)
-        box_top_abs = y_center_abs - (box_height_abs/2.0)
-        bbox_normalized = [box_left_abs/im['width'],
-                           box_top_abs/im['height'],
-                           box_width_abs/im['width'],
-                           box_height_abs/im['height']]
-        
-        category_id = str(int(row['category']))
-        confidence = row['confidence']        
-        assert isinstance(confidence,float) and confidence <= 1.0 and confidence >= 0.0
-        
-        det = {}
-        im['detections'].append(det)
-        det['category'] = category_id
-        det['conf'] = confidence
-        det['bbox'] = bbox_normalized
-                
-    # ...for each row
-        
-    n_empty_images = 0
-    
-    # Handle images without annotations
-    for fn_relative in image_files_relative:
-        
-        if fn_relative not in fn_relative_to_im:
-            if unannotated_image_handling == 'empty':
-                im = {}
-                im['file'] = fn_relative
-                im['detections'] = []                
-                fn_relative_to_im[fn_relative] = im
-                n_empty_images += 1
-                # Don't bother to read width and height here
-            elif unannotated_image_handling == 'warning':
-                print('Warning: image {} is not represented in the .csv results file'.format(fn_relative))
-            elif unannotated_image_handling == 'error':
-                raise ValueError('Image {} is not represented in the .csv results file'.format(fn_relative))
-            elif unannotated_image_handling == 'skip':
-                continue
-            
-    # ...for each image file
-    
-    if n_empty_images > 0:
-        print('Warning: assuming {} of {} images without annotations are empty'.format(
-            n_empty_images,len(image_files_relative)))
-        
-    images = list(fn_relative_to_im.values())
-            
-    # The MD output format uses string-ints for category IDs, right now we have ints
-    detection_categories = {}
-    for category_id_int in category_id_to_name:
-        detection_categories[str(category_id_int)] = category_id_to_name[category_id_int]
-    
-    info = {}
-    info['format_version'] = '1.4'
-    info['detector'] = 'RAIC .csv converter'
-    
-    d = {}
-    d['images'] = images
-    d['detection_categories'] = detection_categories
-    d['info'] = info
-        
-    with open(output_file,'w') as f:
-        json.dump(d,f,indent=1)
-        
-    return output_file
-
-# ...def raic_csv_to_md_results(...)
-
-    
-#%% Interactive driver
-
-if False:
-    
-    pass
-
-    #%% Test conversion
-    
-    base_folder = r'G:\temp\S24_B04_R1_output_annotations_for_Dan'
-    result_csv_file = os.path.join(base_folder,'S24_B04_R1_output_annotations_for_Dan.csv')
-    class_mapping_csv_file = os.path.join(base_folder,'categories_key.csv')
-    
-    # This is wrong, B04_R1 has to be part of the image paths
-    # image_folder = os.path.join(base_folder,'B04_R1')
-    
-    image_folder = base_folder 
-    
-    output_file = None
-    unannotated_image_handling='empty'
-    
-    output_file = raic_csv_to_md_results(result_csv_file=result_csv_file, 
-                                         class_mapping_csv_file=class_mapping_csv_file,
-                                         image_folder=image_folder,
-                                         output_file=output_file,
-                                         unannotated_image_handling=unannotated_image_handling)
-
-    #%% Validate results file
-    
-    from megadetector.postprocessing.validate_batch_results import \
-        ValidateBatchResultsOptions, validate_batch_results
-        
-    validation_options = ValidateBatchResultsOptions()
-    validation_options.check_image_existence = True
-    validation_options.relative_path_base = image_folder
-    validation_options.return_data = True
-    
-    results = validate_batch_results(output_file,validation_options)
-    assert len(results['validation_results']['errors']) == 0
-    assert len(results['validation_results']['warnings']) == 0
-    
-    
-    #%% Preview results
-    
-    from megadetector.postprocessing.postprocess_batch_results import \
-        PostProcessingOptions, process_batch_results
-        
-    postprocessing_options = PostProcessingOptions()
-    
-    postprocessing_options.md_results_file = output_file    
-    postprocessing_options.output_dir = r'g:\temp\serengeti-conversion-preview'
-    postprocessing_options.image_base_dir = image_folder
-    postprocessing_options.confidence_threshold = 0.2
-    postprocessing_options.num_images_to_sample = None
-    postprocessing_options.viz_target_width = 1280
-    postprocessing_options.line_thickness = 4    
-    postprocessing_options.parallelize_rendering_n_cores = 10
-    postprocessing_options.parallelize_rendering_with_threads = True
-        
-    postprocessing_results = process_batch_results(postprocessing_options)
-    
-    from megadetector.utils.path_utils import open_file
-    open_file(postprocessing_results.output_html_file)
-    
-    
-#%% Command-line driver
-
-def main():
-        
-    parser = argparse.ArgumentParser()
-    parser.add_argument('result_csv_file', type=str, 
-                        help='csv file containing AI results')
-    parser.add_argument('class_mapping_csv_file', type=str, 
-                        help='csv file containing class mappings (with columns CLASS, SPECIES)')
-    parser.add_argument('image_folder', type=str, 
-                        help='folder containing the images referred to in [result_csv_file]')
-    parser.add_argument('--output_file', type=str, default=None,
-                        help='.json file to which we should write results (defaults to [result_csv_file].json)')
-    
-    if len(sys.argv[1:])==0:
-        parser.print_help()
-        parser.exit()
-        
-    args = parser.parse_args()
-    raic_csv_to_md_results(result_csv_file=args.result_csv_file,
-                           class_mapping_csv_file=args.class_mapping_csv_file,
-                           image_folder=args.image_folder,
-                           output_file=args.output_file)
-
-if __name__ == '__main__':    
-    main()