megadetector 5.0.11__py3-none-any.whl → 5.0.12__py3-none-any.whl

megadetector/postprocessing/convert_output_format.py

@@ -0,0 +1,396 @@
+"""
+
+convert_output_format.py
+
+Converts between file formats output by our batch processing API.  Currently
+supports json <--> csv conversion, but this should be the landing place for any
+conversion - including between hypothetical alternative .json versions - that we support 
+in the future.
+
+The .csv format is largely obsolete, don't use it unless you're super-duper sure you need it.
+
+"""
+
+#%% Constants and imports
+
+import argparse
+import json
+import csv
+import sys
+import os
+
+from tqdm import tqdm
+
+from megadetector.postprocessing.load_api_results import load_api_results_csv
+from megadetector.data_management.annotations import annotation_constants
+from megadetector.utils import ct_utils
+
+CONF_DIGITS = 3
+
+
+#%% Conversion functions
+
+def convert_json_to_csv(input_path,output_path=None,min_confidence=None,
+                        omit_bounding_boxes=False,output_encoding=None,
+                        overwrite=True):
+    """
+    Converts a MD results .json file to a totally non-standard .csv format.
+    
+    If [output_path] is None, will convert x.json to x.csv.
+    
+    TODO: this function should obviously be using Pandas or some other sensible structured
+    representation of tabular data.  Even a list of dicts.  This implementation is quite
+    brittle and depends on adding fields to every row in exactly the right order.
+    
+    Args:
+        input_path (str): the input .json file to convert
+        output_path (str, optional): the output .csv file to generate; if this is None, uses 
+            [input_path].csv
+        min_confidence (float, optional): the minimum-confidence detection we should include 
+            in the "detections" column; has no impact on the other columns
+        omit_bounding_boxes (bool): whether to leave out the json-formatted bounding boxes
+            that make up the "detections" column, which are not generally useful for someone who
+            wants to consume this data as a .csv file
+        output_encoding (str, optional): encoding to use for the .csv file
+        overwrite (bool): whether to overwrite an existing .csv file; if this is False and the
+            output file exists, no-ops and returns
+           
+    """
+    
+    if output_path is None:
+        output_path = os.path.splitext(input_path)[0]+'.csv'
+        
+    if os.path.isfile(output_path) and (not overwrite):
+        print('File {} exists, skipping json --> csv conversion'.format(output_path))
+        return
+    
+    print('Loading json results from {}...'.format(input_path))
+    json_output = json.load(open(input_path))
+
+    rows = []
+    
+    fixed_columns = ['image_path', 'max_confidence', 'detections']
+    
+    # We add an output column for each class other than 'empty', 
+    # containing the maximum probability of  that class for each image
+    # n_non_empty_detection_categories = len(annotation_constants.annotation_bbox_categories) - 1
+    n_non_empty_detection_categories = annotation_constants.NUM_DETECTOR_CATEGORIES
+    detection_category_column_names = []
+    assert annotation_constants.detector_bbox_categories[0] == 'empty'
+    for cat_id in range(1,n_non_empty_detection_categories+1):
+        cat_name = annotation_constants.detector_bbox_categories[cat_id]
+        detection_category_column_names.append('max_conf_' + cat_name)
+    
+    n_classification_categories = 0
+    
+    if 'classification_categories' in json_output.keys():
+        classification_category_id_to_name = json_output['classification_categories']
+        classification_category_ids = list(classification_category_id_to_name.keys())
+        classification_category_id_to_column_number = {}
+        classification_category_column_names = []
+        for i_category,category_id in enumerate(classification_category_ids):
+            category_name = classification_category_id_to_name[category_id].\
+                replace(' ','_').replace(',','')
+            classification_category_column_names.append('max_classification_conf_' + category_name)
+            classification_category_id_to_column_number[category_id] = i_category
+
+        n_classification_categories = len(classification_category_ids)
+    
+    # There are several .json fields for which we add .csv columns; other random bespoke fields
+    # will be ignored.
+    optional_fields = ['width','height','datetime','exif_metadata']
+    optional_fields_present = set()
+    
+    # Iterate once over the data to check for optional fields
+    print('Looking for optional fields...')
+    
+    for im in tqdm(json_output['images']):
+        # Which optional fields are present for this image?
+        for k in im.keys():
+            if k in optional_fields:
+                optional_fields_present.add(k)
+                
+    optional_fields_present = sorted(list(optional_fields_present))
+    if len(optional_fields_present) > 0:
+        print('Found {} optional fields'.format(len(optional_fields_present)))
+            
+    expected_row_length = len(fixed_columns) + len(detection_category_column_names) + \
+        n_classification_categories + len(optional_fields_present)
+        
+    print('Formatting results...')
+    
+    # i_image = 0; im = json_output['images'][i_image]
+    for im in tqdm(json_output['images']):
+        
+        image_id = im['file']
+        
+        if 'failure' in im and im['failure'] is not None:
+            row = [image_id, 'failure', im['failure']]
+            rows.append(row)
+            # print('Skipping failed image {} ({})'.format(im['file'],im['failure']))
+            continue
+
+        max_conf = ct_utils.get_max_conf(im)
+        detections = []
+        max_detection_category_probabilities = [None] * n_non_empty_detection_categories
+        max_classification_category_probabilities = [0] * n_classification_categories
+              
+        # d = im['detections'][0]
+        for d in im['detections']:
+            
+            # Skip sub-threshold detections
+            if (min_confidence is not None) and (d['conf'] < min_confidence):
+                continue
+            
+            input_bbox = d['bbox']
+            
+            # Our .json format is xmin/ymin/w/h
+            #
+            # Our .csv format was ymin/xmin/ymax/xmax
+            xmin = input_bbox[0]
+            ymin = input_bbox[1]
+            xmax = input_bbox[0] + input_bbox[2]
+            ymax = input_bbox[1] + input_bbox[3]
+            output_detection = [ymin, xmin, ymax, xmax]
+                
+            output_detection.append(d['conf'])
+            
+            # Category 0 is empty, for which we don't have a column, so the max
+            # confidence for category N goes in column N-1
+            detection_category_id = int(d['category'])
+            assert detection_category_id > 0 and detection_category_id <= \
+                n_non_empty_detection_categories
+            detection_category_column = detection_category_id - 1
+            detection_category_max = max_detection_category_probabilities[detection_category_column]
+            if detection_category_max is None or d['conf'] > detection_category_max:
+                max_detection_category_probabilities[detection_category_column] = d['conf']
+            
+            output_detection.append(detection_category_id)
+            detections.append(output_detection)
+    
+            if 'classifications' in d:
+                assert n_classification_categories > 0,\
+                    'Oops, I have classification results, but no classification metadata'
+                for c in d['classifications']:
+                    category_id = c[0]
+                    p = c[1]
+                    category_index = classification_category_id_to_column_number[category_id]
+                    if (max_classification_category_probabilities[category_index] < p):
+                        max_classification_category_probabilities[category_index] = p
+                
+                # ...for each classification
+                
+            # ...if we have classification results for this detection
+            
+        # ...for each detection
+        
+        detection_string = ''
+        if not omit_bounding_boxes:
+            detection_string = json.dumps(detections)
+            
+        row = [image_id, max_conf, detection_string]
+        row.extend(max_detection_category_probabilities)
+        row.extend(max_classification_category_probabilities)
+        
+        for field_name in optional_fields_present:
+            if field_name not in im:
+                row.append('')
+            else:
+                row.append(str(im[field_name]))
+                
+        assert len(row) == expected_row_length
+        rows.append(row)
+        
+    # ...for each image
+
+    print('Writing to csv...')
+    
+    with open(output_path, 'w', newline='', encoding=output_encoding) as f:
+        writer = csv.writer(f, delimiter=',')
+        header = fixed_columns
+        header.extend(detection_category_column_names)
+        if n_classification_categories > 0:
+            header.extend(classification_category_column_names)
+        for field_name in optional_fields_present:
+            header.append(field_name)
+        writer.writerow(header)
+        writer.writerows(rows)
+
+# ...def convert_json_to_csv(...)
+
+    
+def convert_csv_to_json(input_path,output_path=None,overwrite=True):
+    """
+    Convert .csv to .json.  If output_path is None, will convert x.csv to x.json.
+    
+    Args:
+        input_path (str): .csv filename to convert to .json
+        output_path (str, optional): the output .json file to generate; if this is None, uses 
+            [input_path].json
+        overwrite (bool): whether to overwrite an existing .json file; if this is False and the
+            output file exists, no-ops and returns    
+        
+    """
+    
+    if output_path is None:
+        output_path = os.path.splitext(input_path)[0]+'.json'
+        
+    if os.path.isfile(output_path) and (not overwrite):
+        print('File {} exists, skipping csv --> json conversion'.format(output_path))
+        return
+            
+    # Format spec:
+    #
+    # https://github.com/agentmorris/MegaDetector/tree/main/megadetector/api/batch_processing
+    
+    print('Loading csv results...')
+    df = load_api_results_csv(input_path)
+
+    info = {
+        "format_version":"1.2",
+        "detector": "unknown",
+        "detection_completion_time" : "unknown",
+        "classifier": "unknown",
+        "classification_completion_time": "unknown"
+    }
+    
+    classification_categories = {}
+    detection_categories = annotation_constants.detector_bbox_categories
+
+    images = []
+    
+    # iFile = 0; row = df.iloc[iFile]
+    for iFile,row in df.iterrows():
+        
+        image = {}
+        image['file'] = row['image_path']
+        image['max_detection_conf'] = round(row['max_confidence'], CONF_DIGITS)
+        src_detections = row['detections']        
+        out_detections = []
+        
+        for iDetection,detection in enumerate(src_detections):
+            
+            # Our .csv format was ymin/xmin/ymax/xmax
+            #
+            # Our .json format is xmin/ymin/w/h
+            ymin = detection[0]
+            xmin = detection[1]
+            ymax = detection[2]
+            xmax = detection[3]
+            bbox = [xmin, ymin, xmax-xmin, ymax-ymin]
+            conf = detection[4]
+            iClass = detection[5]
+            out_detection = {}
+            out_detection['category'] = str(iClass)
+            out_detection['conf'] = conf
+            out_detection['bbox'] = bbox            
+            out_detections.append(out_detection)
+            
+        # ...for each detection
+        
+        image['detections'] = out_detections
+        images.append(image)
+        
+    # ...for each image        
+    json_out = {}
+    json_out['info'] = info
+    json_out['detection_categories'] = detection_categories
+    json_out['classification_categories'] = classification_categories
+    json_out['images'] = images
+    
+    json.dump(json_out,open(output_path,'w'),indent=1)
+    
+# ...def convert_csv_to_json(...)
+
+
+#%% Interactive driver
+
+if False:    
+
+    #%%
+    
+    input_path = r'c:\temp\test.json'
+    min_confidence = None
+    output_path = input_path + '.csv'
+    convert_json_to_csv(input_path,output_path,min_confidence=min_confidence,
+                        omit_bounding_boxes=False)
+            
+    #%%
+    
+    base_path = r'c:\temp\json'
+    input_paths = os.listdir(base_path)
+    input_paths = [os.path.join(base_path,s) for s in input_paths]
+    
+    min_confidence = None    
+    for input_path in input_paths:
+        output_path = input_path + '.csv'
+        convert_json_to_csv(input_path,output_path,min_confidence=min_confidence,
+                            omit_bounding_boxes=True)    
+    
+    #%% Concatenate .csv files from a folder
+
+    import glob
+    csv_files = glob.glob(os.path.join(base_path,'*.json.csv' ))
+    master_csv = os.path.join(base_path,'all.csv')
+    
+    print('Concatenating {} files to {}'.format(len(csv_files),master_csv))
+    
+    header = None
+    with open(master_csv, 'w') as fout:
+        
+        for filename in tqdm(csv_files):
+            
+            with open(filename) as fin:
+                
+                lines = fin.readlines()
+                
+                if header is not None:
+                    assert lines[0] == header
+                else:
+                    header = lines[0]
+                    fout.write(header)
+                    
+                for line in lines[1:]:
+                    if len(line.strip()) == 0:
+                        continue                    
+                    fout.write(line)
+                    
+        # ...for each .csv file
+        
+    # with open(master_csv)
+    
+    
+#%% Command-line driver
+        
+def main():
+    
+    parser = argparse.ArgumentParser()
+    parser.add_argument('input_path',type=str,
+                        help='Input filename ending in .json or .csv')
+    parser.add_argument('--output_path',type=str,default=None,
+                        help='Output filename ending in .json or .csv (defaults to ' + \
+                             'input file, with .json/.csv replaced by .csv/.json)')
+    
+    if len(sys.argv[1:]) == 0:
+        parser.print_help()
+        parser.exit()
+
+    args = parser.parse_args()
+    
+    if args.output_path is None:
+        if args.input_path.endswith('.csv'):
+            args.output_path = args.input_path[:-4] + '.json'
+        elif args.input_path.endswith('.json'):
+            args.output_path = args.input_path[:-5] + '.csv'
+        else:
+            raise ValueError('Illegal input file extension')    
+    
+    if args.input_path.endswith('.csv') and args.output_path.endswith('.json'):
+        convert_csv_to_json(args.input_path,args.output_path)
+    elif args.input_path.endswith('.json') and args.output_path.endswith('.csv'):
+        convert_json_to_csv(args.input_path,args.output_path)
+    else:
+        raise ValueError('Illegal format combination')            
+
+if __name__ == '__main__':
+    main()

megadetector/postprocessing/load_api_results.py

@@ -0,0 +1,195 @@
+"""
+
+load_api_results.py
+
+DEPRECATED
+
+As of 2023.12, this module is used in postprocessing and RDE.  Not recommended
+for new code.
+
+Loads the output of the batch processing API (json) into a Pandas dataframe.
+
+Includes functions to read/write the (very very old) .csv results format.
+
+"""
+
+#%% Imports
+
+import json
+import os
+
+from typing import Dict, Mapping, Optional, Tuple
+
+import pandas as pd
+
+from megadetector.utils import ct_utils
+
+
+#%% Functions for loading .json results into a Pandas DataFrame, and writing back to .json
+
+def load_api_results(api_output_path: str, normalize_paths: bool = True,
+                     filename_replacements: Optional[Mapping[str, str]] = None,
+                     force_forward_slashes: bool = True
+                     ) -> Tuple[pd.DataFrame, Dict]:
+    r"""
+    Loads json-formatted MegaDetector results to a Pandas DataFrame.
+
+    Args:
+        api_output_path: path to the output json file
+        normalize_paths: whether to apply os.path.normpath to the 'file' field
+            in each image entry in the output file
+        filename_replacements: replace some path tokens to match local paths to
+            the original blob structure
+        force_forward_slashes: whether to convert backslashes to forward slashes
+            in filenames
+
+    Returns:
+        detection_results: pd.DataFrame, contains at least the columns ['file', 'detections','failure']
+        other_fields: a dict containing fields in the results other than 'images'
+    """
+    
+    print('Loading results from {}'.format(api_output_path))
+
+    with open(api_output_path) as f:
+        detection_results = json.load(f)
+
+    # Validate that this is really a detector output file
+    for s in ['info', 'detection_categories', 'images']:
+        assert s in detection_results, 'Missing field {} in detection results'.format(s)
+
+    # Fields in the output json other than 'images'
+    other_fields = {}
+    for k, v in detection_results.items():
+        if k != 'images':
+            other_fields[k] = v
+
+    if normalize_paths:
+        for image in detection_results['images']:
+            image['file'] = os.path.normpath(image['file'])            
+
+    if force_forward_slashes:
+        for image in detection_results['images']:
+            image['file'] = image['file'].replace('\\','/')
+            
+    # Replace some path tokens to match local paths to original blob structure
+    if filename_replacements is not None:
+        for string_to_replace in filename_replacements.keys():
+            replacement_string = filename_replacements[string_to_replace]
+            for im in detection_results['images']:
+                im['file'] = im['file'].replace(string_to_replace,replacement_string)
+
+    print('Converting results to dataframe')
+    
+    # If this is a newer file that doesn't include maximum detection confidence values,
+    # add them, because our unofficial internal dataframe format includes this.
+    for im in detection_results['images']:
+        if 'max_detection_conf' not in im:
+            im['max_detection_conf'] = ct_utils.get_max_conf(im)
+    
+    # Pack the json output into a Pandas DataFrame
+    detection_results = pd.DataFrame(detection_results['images'])
+    
+    print('Finished loading MegaDetector results for {} images from {}'.format(
+            len(detection_results),api_output_path))
+
+    return detection_results, other_fields
+
+
+def write_api_results(detection_results_table, other_fields, out_path):
+    """
+    Writes a Pandas DataFrame to the MegaDetector .json format.
+    """
+
+    print('Writing detection results to {}'.format(out_path))
+
+    fields = other_fields
+
+    images = detection_results_table.to_json(orient='records',
+                                             double_precision=3)
+    images = json.loads(images)
+    fields['images'] = images
+    
+    # Convert the 'version' field back to a string as per format convention
+    try:
+        version = other_fields['info']['format_version']
+        if not isinstance(version,str):
+            other_fields['info']['format_version'] = str(version)
+    except Exception:
+        print('Warning: error determining format version')
+        pass
+    
+    # Remove 'max_detection_conf' as per newer file convention (format >= v1.3)
+    try:
+        version = other_fields['info']['format_version']
+        version = float(version)
+        if version >= 1.3:
+            for im in images:
+                if 'max_detection_conf' in im:
+                    del im['max_detection_conf']
+    except Exception:
+        print('Warning: error removing max_detection_conf from output')
+        pass
+
+    with open(out_path, 'w') as f:
+        json.dump(fields, f, indent=1)
+
+    print('Finished writing detection results to {}'.format(out_path))
+
+
+def load_api_results_csv(filename, normalize_paths=True, filename_replacements={}, nrows=None):
+    """
+    [DEPRECATED]
+    
+    Loads .csv-formatted MegaDetector results to a pandas table
+    """
+
+    print('Loading MegaDetector results from {}'.format(filename))
+
+    detection_results = pd.read_csv(filename,nrows=nrows)
+
+    print('De-serializing MegaDetector results from {}'.format(filename))
+
+    # Confirm that this is really a detector output file
+    for s in ['image_path','max_confidence','detections']:
+        assert s in detection_results.columns
+
+    # Normalize paths to simplify comparisons later
+    if normalize_paths:
+        detection_results['image_path'] = detection_results['image_path'].apply(os.path.normpath)
+
+    # De-serialize detections
+    detection_results['detections'] = detection_results['detections'].apply(json.loads)
+
+    # Optionally replace some path tokens to match local paths to the original blob structure
+    # string_to_replace = list(options.detector_output_filename_replacements.keys())[0]
+    for string_to_replace in filename_replacements:
+
+        replacement_string = filename_replacements[string_to_replace]
+
+        # iRow = 0
+        for iRow in range(0,len(detection_results)):
+            row = detection_results.iloc[iRow]
+            fn = row['image_path']
+            fn = fn.replace(string_to_replace,replacement_string)
+            detection_results.at[iRow,'image_path'] = fn
+
+    print('Finished loading and de-serializing MD results for {} images from {}'.format(
+        len(detection_results),filename))
+
+    return detection_results
+
+
+def write_api_results_csv(detection_results, filename):
+    """    
+    [DEPRECATED]
+    
+    Writes a Pandas table to csv in a way that's compatible with the .csv output
+    format.  Currently just a wrapper around to_csv that forces output writing
+    to go through a common code path.
+    """
+
+    print('Writing detection results to {}'.format(filename))
+
+    detection_results.to_csv(filename, index=False)
+
+    print('Finished writing detection results to {}'.format(filename))