megadetector 5.0.26__py3-none-any.whl → 5.0.28__py3-none-any.whl

megadetector/postprocessing/generate_csv_report.py

@@ -0,0 +1,499 @@
+"""
+
+generate_csv_report.py
+
+Generates a .csv report from a MD-formatted .json file with the following columns:
+
+* filename
+* datetime (if images or EXIF information is supplied)
+* detection_category
+* max_detection_confidence
+* classification_category
+* max_classification_confidence
+* count
+
+One row is generated per category pair per image.  For example, these would be unique rows:
+
+image0001.jpg,animal,deer,4
+image0001.jpg,animal,lion,4
+image0001.jpg,animal,[none],4
+image0001.jpg,person,[none],2
+
+Images with no above-threshold detections will have a single row:
+
+image0001.jpg,empty,[none],-1
+
+Images with processing errors will have a single row:
+
+image0001.jpg,error,error_string,-1
+
+"""
+
+#%% Constants and imports
+
+import os
+import json
+import tempfile
+import uuid
+import pandas as pd
+
+from copy import deepcopy
+
+from megadetector.utils.wi_utils import load_md_or_speciesnet_file
+from megadetector.utils.ct_utils import get_max_conf
+from megadetector.utils.ct_utils import is_list_sorted
+
+from megadetector.detection.run_detector import \
+    get_typical_confidence_threshold_from_results
+
+from megadetector.data_management.read_exif import \
+    read_exif_from_folder, ReadExifOptions, minimal_exif_tags
+
+default_classification_threshold = 0.3
+unknown_datetime_tag = ''
+
+
+#%% Functions
+
+def generate_csv_report(md_results_file,
+                        output_file=None,
+                        datetime_source=None,
+                        folder_level_columns=None,
+                        detection_confidence_threshold=None,
+                        classification_confidence_threshold=None,
+                        verbose=True):
+    """
+    Generates a .csv report from a MD-formatted .json file
+    
+    Args:
+        md_results_file (str): MD results .json file for which we should generate a report
+        output_file (str, optional): .csv file to write; if this is None, we'll use md_results_file.csv
+        datetime_source (str, optional): if datetime information is required, this should point to 
+            a folder of images, a MD results .json file (can be the same as the input file), or
+            an exif_info.json file created with read_exif().
+        folder_level_columns (list of int, optional): list of folder levels (where zero is the top-level
+            folder in a path name) for which we should create separate columns.  Should be zero-indexed ints,
+            or a comma-delimited list of zero-indexed int-strings.
+        detection_confidence_threshold (float, optional): detections below this confidence threshold will not 
+            be included in the output data.  Defaults to the recommended value based on the .json file.
+        classification_confidence_threshold (float, optional): classifications below this confidence threshold will 
+            not be included in the output data (i.e., detections will be considered "animal").
+        verbose (bool, optional): enable debug output, including the progress bar,        
+    """
+    
+    ##%% Load results file
+
+    results = load_md_or_speciesnet_file(md_results_file)
+    
+    print('Loaded results for {} images'.format(len(results['images'])))
+
+    detection_category_id_to_name = results['detection_categories']
+    classification_category_id_to_name = None
+    if 'classification_categories' in results:
+        classification_category_id_to_name = results['classification_categories']
+    
+    if output_file is None:
+        output_file = md_results_file + '.csv'
+
+    ##%% Read datetime information if necessary
+
+    filename_to_datetime_string = None
+
+    if datetime_source is not None:
+
+        all_exif_results = None
+        
+        if os.path.isdir(datetime_source):
+            
+            # Read EXIF info from images
+            read_exif_options = ReadExifOptions()
+            read_exif_options.tags_to_include = minimal_exif_tags
+            read_exif_options.byte_handling = 'delete'
+            exif_cache_file = os.path.join(tempfile.gettempdir(),
+                                           'md-exif-data',
+                                           str(uuid.uuid1())+'.json')
+            print('Reading EXIF datetime info from {}, writing to {}'.format(
+                datetime_source,exif_cache_file))
+            os.makedirs(os.path.dirname(exif_cache_file),exist_ok=True)
+
+            all_exif_results = read_exif_from_folder(input_folder=datetime_source,
+                                                     output_file=exif_cache_file,
+                                                     options=read_exif_options,
+                                                     recursive=True)
+
+        else:
+            assert os.path.isfile(datetime_source), \
+                'datetime source {} is neither a folder nor a file'.format(datetime_source)
+            
+            # Is this the same file we've already read?
+
+            # Load this, decide whether it's a MD file or an exif_info file
+            with open(datetime_source,'r') as f:
+                d = json.load(f)
+
+            if isinstance(d,list):
+                all_exif_results = d
+            else:
+                assert isinstance(d,dict), 'Unrecognized file format supplied as datetime source'
+                assert 'images' in d,\
+                    'The datetime source you provided doesn\'t look like a valid source .json file'
+                all_exif_results = []
+                found_datetime = False
+                for im in d['images']:
+                    exif_result = {'file_name':im['file']}
+                    if 'datetime' in im:
+                        found_datetime = True
+                        exif_result['exif_tags'] = {'DateTimeOriginal':im['datetime']}
+                    all_exif_results.append(exif_result)
+                if not found_datetime:
+                    print('Warning: a MD results file was supplied as the datetime source, but it does not appear '
+                          'to contain datetime information.')            
+
+        assert all_exif_results is not None
+                
+        filename_to_datetime_string = {}
+
+        for exif_result in all_exif_results:
+            datetime_string = unknown_datetime_tag
+            if ('exif_tags' in exif_result) and \
+               (exif_result['exif_tags'] is not None) and \
+               ('DateTimeOriginal' in exif_result['exif_tags']):
+                datetime_string = exif_result['exif_tags']['DateTimeOriginal']
+                if datetime_string is None:
+                    datetime_string = ''
+                else:
+                    assert isinstance(datetime_string,str), 'Unrecognized datetime format'
+            filename_to_datetime_string[exif_result['file_name']] = datetime_string
+
+        image_files = [im['file'] for im in results['images']]
+        image_files_set = set(image_files)
+
+        files_in_exif_but_not_in_results = []
+        files_in_results_but_not_in_exif = []
+        files_with_no_datetime_info = []
+
+        for fn in filename_to_datetime_string:
+            dts = filename_to_datetime_string[fn]
+            if (dts is None) or (dts == unknown_datetime_tag) or (len(dts) == 0):
+                files_with_no_datetime_info.append(fn)
+            if fn not in image_files_set:
+                files_in_exif_but_not_in_results.append(fn)
+        
+        for fn in image_files_set:
+            if fn not in filename_to_datetime_string:
+                files_in_results_but_not_in_exif.append(fn)
+
+        print('{} files (of {}) in EXIF info not found in MD results'.format(
+            len(files_in_exif_but_not_in_results),len(filename_to_datetime_string)
+        ))
+
+        print('{} files (of {}) in MD results not found in MD EXIF info'.format(
+            len(files_in_results_but_not_in_exif),len(image_files_set)
+        ))
+
+        print('Failed to read datetime information for {} files (of {}) in EXIF info'.format(
+            len(files_with_no_datetime_info),len(filename_to_datetime_string)
+        ))        
+
+    # ...if we need to deal with datetimes
+    
+
+    ##%% Parse folder level column specifier
+
+    if folder_level_columns is not None:
+        if isinstance(folder_level_columns,str):
+            tokens = folder_level_columns.split(',')
+            folder_level_columns = [int(s) for s in tokens]
+        for folder_level in folder_level_columns:
+            if (not isinstance(folder_level,int)) or (folder_level < 0):
+                raise ValueError('Illegal folder level specifier {}'.format(
+                    str(folder_level_columns)))
+
+
+    ##%% Fill in default thresholds
+
+    if classification_confidence_threshold is None:
+        classification_confidence_threshold = default_classification_threshold
+    if detection_confidence_threshold is None:
+        detection_confidence_threshold = \
+            get_typical_confidence_threshold_from_results(results)
+
+    assert detection_confidence_threshold is not None
+
+    
+    ##%% Fill in output records
+
+    output_records = []
+
+    # For each image
+    #
+    # im = results['images'][0]
+    for im in results['images']:
+    
+        """
+        * filename
+        * datetime (if images or EXIF information is supplied)
+        * detection_category
+        * max_detection_confidence
+        * classification_category
+        * max_classification_confidence
+        * count
+        """
+        
+        base_record = {}
+
+        base_record['filename'] = im['file'].replace('\\','/')
+
+        # Datetime (if necessary)
+        if filename_to_datetime_string is not None:
+            if im['file'] in filename_to_datetime_string:
+                datetime_string = filename_to_datetime_string[im['file']]
+            else:
+                datetime_string = ''
+        base_record['datetime'] = datetime_string
+
+        for s in ['detection_category','max_detection_confidence',
+                  'classification_category','max_classification_confidence',
+                  'count']:
+            base_record[s] = ''
+
+        # Folder level columns
+        tokens = im['file'].split('/')
+
+        for folder_level in folder_level_columns:
+            folder_level_column_name = 'folder_level_' + str(folder_level).zfill(2)
+            if folder_level >= len(tokens):
+                folder_level_value = ''
+            else:
+                folder_level_value = tokens[folder_level]
+            base_record[folder_level_column_name] = folder_level_value
+        
+        records_this_image = []
+
+        # Create one output row if this image failed
+        if 'failure' in im and im['failure'] is not None and len(im['failure']) > 0:
+
+            record = deepcopy(base_record)
+            record['detection_category'] = 'error'
+            record['classification_category'] = im['failure']
+            records_this_image.append(record)
+            assert 'detections' not in im or im['detections'] is None
+
+        else:
+
+            assert 'detections' in im and im['detections'] is not None
+        
+            # Count above-threshold detections
+            detections_above_threshold = []
+            for det in im['detections']:
+                if det['conf'] >= detection_confidence_threshold:
+                    detections_above_threshold.append(det)
+            max_detection_conf = get_max_conf(im)            
+
+            # Create one output row if this image is empty (i.e., has no 
+            # above-threshold detections)
+            if len(detections_above_threshold) == 0:
+
+                record = deepcopy(base_record)
+                record['detection_category'] = 'empty'
+                record['max_detection_confidence'] = max_detection_conf
+                records_this_image.append(record)
+
+            # ...if this image is empty
+
+            else:
+
+                # Maps a string of the form:
+                #
+                # detection_category:classification_category
+                #
+                # ...to a dict with fields ['max_detection_conf','max_classification_conf','count']
+                category_info_string_to_record = {}
+
+                for det in detections_above_threshold:
+
+                    assert det['conf'] >= detection_confidence_threshold
+
+                    detection_category_name = detection_category_id_to_name[det['category']]
+                    detection_confidence = det['conf']
+                    classification_category_name = ''
+                    classificaition_confidence = 0.0
+
+                    if ('classifications' in det) and (len(det['classifications']) > 0):
+
+                        # Classifications should always be sorted by confidence.  Not
+                        # technically required, but always true in practice.
+                        assert is_list_sorted([c[1] for c in det['classifications']]), \
+                            'This script does not yet support unsorted classifications'
+                        assert classification_category_id_to_name is not None, \
+                            'If classifications are present, category mappings should be present'
+                        
+                        # Only use the first classification
+                        classification = det['classifications'][0]
+                        if classification[1] >= classification_confidence_threshold:                            
+                            classification_category_name = \
+                                classification_category_id_to_name[classification[0]]
+                            classification_confidence = classification[1]
+                            
+                    # ...if classifications are present
+
+                    # E.g. "animal:rodent", or "vehicle:"
+                    category_info_string = detection_category_name + ':' + classification_category_name
+
+                    if category_info_string not in category_info_string_to_record:
+                        category_info_string_to_record[category_info_string] = {
+                            'max_detection_confidence':0.0,
+                            'max_classification_confidence':0.0,
+                            'count':0,
+                            'detection_category':detection_category_name,
+                            'classification_category':classification_category_name
+                        }
+                    
+                    record = category_info_string_to_record[category_info_string]
+                    record['count'] += 1
+                    if detection_confidence > record['max_detection_confidence']:
+                        record['max_detection_confidence'] = detection_confidence
+                    if classification_confidence > record['max_classification_confidence']:
+                        record['max_classification_confidence'] = classification_confidence
+
+                # ...for each detection
+
+                for record_in in category_info_string_to_record.values():
+                    assert record_in['count'] > 0
+                    record_out = deepcopy(base_record)
+                    for k in record_in.keys():
+                        assert k in record_out.keys()
+                        record_out[k] = record_in[k]
+                    records_this_image.append(record_out)
+                    
+            # ...is this empty/non-empty?
+
+        # ...if this image failed/didn't fail
+        
+        # Add to [records]
+        output_records.extend(records_this_image)
+
+    # ...for each image
+
+    # Make sure every record has the same columns
+    column_names = output_records[0].keys()
+    for record in output_records:
+        assert record.keys() == column_names
+
+    # Write to .csv
+    df = pd.DataFrame(output_records)
+    df.to_csv(output_file,header=True,index=False)
+    
+    # from megadetector.utils.path_utils import open_file; open_file(output_file)
+    
+# ...generate_csv_report(...)
+
+
+#%% Interactive driver
+
+if False:
+    
+    pass
+
+    #%% Configure options
+    
+    """
+    python run_detector_batch.py MDV5A "g:\temp\md-test-images" "g:\temp\md-test-images\md_results_with_datetime.json" --recursive --output_relative_filenames --include_image_timestamp --include_exif_data
+    """
+
+    md_results_file = 'g:/temp/csv-report-test/md-results.json'
+    datetime_source = 'g:/temp/csv-report-test/exif_data.json'
+    
+    # datetime_source = 'g:/temp/md-test-images'
+    # datetime_source = 'g:/temp/md-test-images/md_results_with_datetime.json'
+    # md_results_file = 'g:/temp/md-test-images/md_results_with_datetime.json'
+    # md_results_file = 'g:/temp/md-test-images/speciesnet_results_md_format.json'
+
+    output_file = None
+    folder_level_columns = [0,1,2,3]
+    detection_confidence_threshold = None
+    classification_confidence_threshold = None
+    verbose = True
+    
+    
+    #%% Programmatic execution
+    
+    generate_csv_report(md_results_file=md_results_file,
+                        output_file=output_file,
+                        datetime_source=datetime_source,
+                        folder_level_columns=folder_level_columns,
+                        detection_confidence_threshold=detection_confidence_threshold,
+                        classification_confidence_threshold=classification_confidence_threshold,
+                        verbose=verbose)
+    
+    
+#%% Command-line driver
+
+import sys,argparse
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        description='Generates a .csv report from a MD-formatted .json file')
+    
+    parser.add_argument(
+        'md_results_file',
+        type=str,
+        help='Path to MD results file (.json)')
+    
+    parser.add_argument(
+        '--output_file',
+        type=str,
+        help='Output filename (.csv) (if omitted, will append .csv to the input file)')
+    
+    parser.add_argument(
+        '--datetime_source',
+        type=str,
+        default=None,
+        help='Image folder, exif_info.json file, or MD results file from which we should read datetime information'
+        )
+    
+    parser.add_argument(
+        '--folder_level_columns',
+        type=str,
+        default=None,
+        help='Comma-separated list of zero-indexed folder levels that should become columns in the output file'
+        )
+    
+    parser.add_argument(
+        '--detection_confidence_threshold',
+        type=float,
+        default=None,
+        help='Detection threshold (if omitted, chooses a reasonable default based on the .json file)'
+        )
+
+    parser.add_argument(
+        '--classification_confidence_threshold',
+        type=float,
+        default=None,
+        help='Classification threshold (default {})'.format(default_classification_threshold)
+        )
+    
+    parser.add_argument(
+        '--verbose',
+        action='store_true',
+        help='Enable additional debug output'
+        )
+    
+
+    if len(sys.argv[1:]) == 0:
+        parser.print_help()
+        parser.exit()
+
+    args = parser.parse_args()
+
+    generate_csv_report(md_results_file=args.md_results_file,
+                        output_file=args.output_file,
+                        datetime_source=args.datetime_source,
+                        folder_level_columns=args.folder_level_columns,
+                        detection_confidence_threshold=args.detection_confidence_threshold,
+                        classification_confidence_threshold=args.classification_confidence_threshold,
+                        verbose=args.verbose)
+    
+if __name__ == '__main__':
+    main()

megadetector/postprocessing/load_api_results.py

@@ -4,7 +4,7 @@ load_api_results.py
 
 DEPRECATED
 
-As of 2023.12, this module is used in postprocessing and RDE.  Not recommended
+As of 2023.12, this module is still used in postprocessing and RDE, but it's not recommended
 for new code.
 
 Loads the output of the batch processing API (json) into a Pandas dataframe.
@@ -23,6 +23,7 @@ from typing import Dict, Mapping, Optional, Tuple
 import pandas as pd
 
 from megadetector.utils import ct_utils
+from megadetector.utils.wi_utils import load_md_or_speciesnet_file
 
 
 #%% Functions for loading .json results into a Pandas DataFrame, and writing back to .json
@@ -50,9 +51,8 @@ def load_api_results(api_output_path: str, normalize_paths: bool = True,
     
     print('Loading results from {}'.format(api_output_path))
 
-    with open(api_output_path) as f:
-        detection_results = json.load(f)
-
+    detection_results = load_md_or_speciesnet_file(api_output_path)
+    
     # Validate that this is really a detector output file
     for s in ['info', 'detection_categories', 'images']:
         assert s in detection_results, 'Missing field {} in detection results'.format(s)

megadetector/postprocessing/postprocess_batch_results.py

@@ -48,6 +48,7 @@ from tqdm import tqdm
 from megadetector.visualization import visualization_utils as vis_utils
 from megadetector.visualization import plot_utils
 from megadetector.utils.write_html_image_list import write_html_image_list
+from megadetector.utils.wi_utils import load_md_or_speciesnet_file
 from megadetector.utils import path_utils
 from megadetector.utils.ct_utils import args_to_object
 from megadetector.utils.ct_utils import sets_overlap
@@ -89,7 +90,9 @@ class PostProcessingOptions:
         ### Options
     
         #: Folder where images live (filenames in [md_results_file] should be relative to this folder)
-        self.image_base_dir = '.'
+        #:
+        #: Can be '' if [md_results_file] uses absolute paths.
+        self.image_base_dir = ''
     
         ## These apply only when we're doing ground-truth comparisons
         
@@ -493,7 +496,7 @@ def _render_bounding_boxes(
                                                    original_size=original_size,label_map=label_map,
                                                    thickness=4,expansion=4)
         
-            # Preprare per-category confidence thresholds
+            # Prepare per-category confidence thresholds
             if isinstance(options.confidence_threshold,float):
                 rendering_confidence_threshold = options.confidence_threshold
             else:
@@ -1873,8 +1876,7 @@ def process_batch_results(options):
                 
                 print('Generating classification category report')
                 
-                with open(options.md_results_file,'r') as f:
-                    d = json.load(f)
+                d = load_md_or_speciesnet_file(options.md_results_file)
                 
                 classification_category_to_count = {}
 

megadetector/taxonomy_mapping/preview_lila_taxonomy.py

@@ -66,9 +66,6 @@ df = pd.read_csv(lila_taxonomy_file)
 from megadetector.taxonomy_mapping.species_lookup import \
     initialize_taxonomy_lookup, get_preferred_taxonomic_match
 
-# from taxonomy_mapping.species_lookup import (
-#    get_taxonomic_info, print_taxonomy_matche)
-
 initialize_taxonomy_lookup()
 
 

megadetector/taxonomy_mapping/taxonomy_graph.py

@@ -303,7 +303,7 @@ def dag_to_tree(graph: nx.DiGraph,
     component separately.
 
     Args:
-        graph: nx.DiGraph, DAG representation of taxonomy hieararchy
+        graph: nx.DiGraph, DAG representation of taxonomy hierarchy
         taxon_to_node: dict, maps (taxon_level, taxon_name) to a TaxonNode
 
     Returns: nx.DiGraph, a tree-structured graph

megadetector/utils/ct_utils.py

@@ -193,7 +193,7 @@ def write_json(path, content, indent=1):
         indent (int, optional): indentation depth passed to json.dump
     """
     
-    with open(path, 'w') as f:
+    with open(path, 'w', newline='\n') as f:
         json.dump(content, f, indent=indent)
 
 
@@ -299,7 +299,8 @@ def _get_max_conf_from_detections(detections):
 def get_max_conf(im):
     """
     Given an image dict in the MD output format, computes the maximum detection confidence for any 
-    class.  Returns 0.0 (rather than None) if there was a failure or 'detections' isn't present.
+    class.  Returns 0.0 if there were no detections, if there was a failure, or if 'detections' isn't 
+    present.
     
     Args:
         im (dict): image dictionary in the MD output format (with a 'detections' field)