megadetector 5.0.29__py3-none-any.whl → 10.0.1__py3-none-any.whl

megadetector/postprocessing/create_crop_folder.py

@@ -11,6 +11,8 @@ of images representing all above-threshold crops from the original folder.
 
 import os
 import json
+import argparse
+
 from tqdm import tqdm
 
 from multiprocessing.pool import Pool, ThreadPool
@@ -19,7 +21,7 @@ from functools import partial
 
 from megadetector.utils.path_utils import insert_before_extension
 from megadetector.utils.ct_utils import invert_dictionary
-from megadetector.utils.ct_utils import is_list_sorted                
+from megadetector.utils.ct_utils import is_list_sorted
 from megadetector.visualization.visualization_utils import crop_image
 from megadetector.visualization.visualization_utils import exif_preserving_save
 
@@ -63,6 +65,7 @@ def _get_crop_filename(image_fn,crop_id):
     """
     Generate crop filenames in a consistent way.
     """
+
     if isinstance(crop_id,int):
         crop_id = str(crop_id).zfill(3)
     assert isinstance(crop_id,str)
@@ -75,7 +78,15 @@ def _generate_crops_for_single_image(crops_this_image,
                                      options):
     """
     Generate all the crops required for a single image.
+
+    Args:
+        crops_this_image (list of dict): list of dicts with at least keys
+            'image_fn_relative', 'crop_id'
+        input_folder (str): input folder (whole images)
+        output_folder (crops): output folder (crops)
+        options (CreateCropFolderOptions): cropping options
     """
+
     if len(crops_this_image) == 0:
         return
 
@@ -147,9 +158,9 @@ def crop_results_to_image_results(image_results_file_with_crop_ids,
         crop_results_prefix (str, optional): if not None, removes this prefix from crop
             results filenames.  Intended to support the case where the crop results
             use absolute paths.
-        detections_without_classification_handling (str, optional): what to do when we 
+        detections_without_classification_handling (str, optional): what to do when we
             encounter a crop that doesn't appear in classification results: 'error',
-            or 'include' ("include" means "leave the detection alone, without classifications"            
+            or 'include' ("include" means "leave the detection alone, without classifications"
     """
 
     ##%% Validate inputs
@@ -190,7 +201,7 @@ def crop_results_to_image_results(image_results_file_with_crop_ids,
     crop_filename_to_results = {}
 
     # im = crop_results['images'][0]
-    for im in crop_results['images']:        
+    for im in crop_results['images']:
         fn = im['file']
         # Possibly remove a prefix from each filename
         if (crop_results_prefix is not None) and (crop_results_prefix in fn):
@@ -260,14 +271,14 @@ def crop_results_to_image_results(image_results_file_with_crop_ids,
                     assert crop_results_this_detection['detections'][0]['bbox'] == [0,0,1,1], \
                         'Invalid crop bounding box'
 
-                    # This check was helpful for the case where crop-level results had already 
+                    # This check was helpful for the case where crop-level results had already
                     # taken detection confidence values from detector output by construct, but this isn't
                     # really meaningful for most cases.
                     # assert abs(crop_results_this_detection['detections'][0]['conf'] - det['conf']) < 0.01
-                    
+
                     if require_identical_detection_categories:
                         assert crop_results_this_detection['detections'][0]['category'] == det['category']
-                    
+
                     # Copy the crop-level classifications
                     det['classifications'] = crop_results_this_detection['detections'][0]['classifications']
                     confidence_values = [x[1] for x in det['classifications']]
@@ -381,8 +392,8 @@ def create_crop_folder(input_file,
             det['crop_id'] = i_detection
 
             crop_info = {'image_fn_relative':image_fn_relative,
-                            'crop_id':i_detection,
-                            'detection':det}
+                         'crop_id':i_detection,
+                         'detection':det}
 
             crop_filename_relative = _get_crop_filename(image_fn_relative,
                                                         crop_info['crop_id'])
@@ -490,4 +501,120 @@ def create_crop_folder(input_file,
 
 #%% Command-line driver
 
-# TODO
+def main():
+    """
+    Command-line interface for creating a crop folder from MegaDetector results.
+    """
+
+    parser = argparse.ArgumentParser(
+        description='Create a folder of crops from MegaDetector results'
+    )
+    parser.add_argument(
+        'input_file',
+        type=str,
+        help='Path to the MegaDetector .json results file'
+    )
+    parser.add_argument(
+        'input_folder',
+        type=str,
+        help='Path to the folder containing the original images'
+    )
+    parser.add_argument(
+        'output_folder',
+        type=str,
+        help='Path to the folder where cropped images will be saved'
+    )
+    parser.add_argument(
+        '--output_file',
+        type=str,
+        default=None,
+        help='Path to save the modified MegaDetector .json file (with crop IDs and filenames)'
+    )
+    parser.add_argument(
+        '--crops_output_file',
+        type=str,
+        default=None,
+        help='Path to save a new .json file for the crops themselves (with full-image detections for each crop)'
+    )
+    parser.add_argument(
+        '--confidence_threshold',
+        type=float,
+        default=0.1,
+        help='Confidence threshold for detections to be cropped (default: 0.1)'
+    )
+    parser.add_argument(
+        '--expansion',
+        type=int,
+        default=0,
+        help='Number of pixels to expand each crop (default: 0)'
+    )
+    parser.add_argument(
+        '--quality',
+        type=int,
+        default=95,
+        help='JPEG quality for saving crops (default: 95)'
+    )
+    parser.add_argument(
+        '--overwrite',
+        type=str,
+        default='true',
+        choices=['true', 'false'],
+        help="Overwrite existing crop images (default: 'true')"
+    )
+    parser.add_argument(
+        '--n_workers',
+        type=int,
+        default=8,
+        help='Number of concurrent workers (default: 8)'
+    )
+    parser.add_argument(
+        '--pool_type',
+        type=str,
+        default='thread',
+        choices=['thread', 'process'],
+        help="Type of parallelism to use ('thread' or 'process', default: 'thread')"
+    )
+    parser.add_argument(
+        '--category_names',
+        type=str,
+        default=None,
+        help="Comma-separated list of category names to include " + \
+             "(e.g., 'animal,person'). If None (default), all categories are included."
+    )
+
+    args = parser.parse_args()
+
+    options = CreateCropFolderOptions()
+    options.confidence_threshold = args.confidence_threshold
+    options.expansion = args.expansion
+    options.quality = args.quality
+    options.overwrite = (args.overwrite.lower() == 'true')
+    options.n_workers = args.n_workers
+    options.pool_type = args.pool_type
+
+    if args.category_names:
+        options.category_names_to_include = [name.strip() for name in args.category_names.split(',')]
+    else:
+        options.category_names_to_include = None
+
+    print('Starting crop folder creation...')
+    print('Input MD results: {}'.format(args.input_file))
+    print('Input image folder {}'.format(args.input_folder))
+    print('Output crop folder: {}'.format(args.output_folder))
+
+    if args.output_file:
+        print('Modified MD results will be saved to {}'.format(args.output_file))
+    if args.crops_output_file:
+        print('Crops .json output will be saved to {}'.format(args.crops_output_file))
+
+    create_crop_folder(
+        input_file=args.input_file,
+        input_folder=args.input_folder,
+        output_folder=args.output_folder,
+        output_file=args.output_file,
+        crops_output_file=args.crops_output_file,
+        options=options
+    )
+
+if __name__ == '__main__':
+    main()

megadetector/postprocessing/load_api_results.py

@@ -18,7 +18,8 @@ Includes functions to read/write the (very very old) .csv results format.
 import json
 import os
 
-from typing import Mapping, Optional
+from typing import Optional
+from collections.abc import Mapping
 
 import pandas as pd
 
@@ -36,13 +37,13 @@ def load_api_results(api_output_path: str, normalize_paths: bool = True,
     Loads json-formatted MegaDetector results to a Pandas DataFrame.
 
     Args:
-        api_output_path: path to the output json file
-        normalize_paths: whether to apply os.path.normpath to the 'file' field
-            in each image entry in the output file
-        filename_replacements: replace some path tokens to match local paths to
-            the original blob structure
-        force_forward_slashes: whether to convert backslashes to forward slashes
-            in filenames
+        api_output_path (str): path to the output json file
+        normalize_paths (bool, optional): whether to apply os.path.normpath to the 'file'
+            field in each image entry in the output file
+        filename_replacements (dict, optional): replace some path tokens to match local paths
+            to the original file structure
+        force_forward_slashes (bool, optional): whether to convert backslashes to forward
+            slashes in filenames
 
     Returns:
         detection_results: pd.DataFrame, contains at least the columns ['file', 'detections','failure']
@@ -98,6 +99,11 @@ def load_api_results(api_output_path: str, normalize_paths: bool = True,
 def write_api_results(detection_results_table, other_fields, out_path):
     """
     Writes a Pandas DataFrame to the MegaDetector .json format.
+
+    Args:
+        detection_results_table (DataFrame): data to write
+        other_fields (dict): additional fields to include in the output .json
+        out_path (str): output .json filename
     """
 
     print('Writing detection results to {}'.format(out_path))
@@ -144,6 +150,14 @@ def load_api_results_csv(filename, normalize_paths=True, filename_replacements=N
     [DEPRECATED]
 
     Loads .csv-formatted MegaDetector results to a pandas table
+
+    Args:
+        filename (str): path to the csv file to read
+        normalize_paths (bool, optional): whether to apply os.path.normpath to the 'file'
+            field in each image entry in the output file
+        filename_replacements (dict, optional): replace some path tokens to match local paths
+            to the original file structure
+        nrows (int, optional): read only the first N rows of [filename]
     """
 
     if filename_replacements is None:
@@ -192,6 +206,10 @@ def write_api_results_csv(detection_results, filename):
     Writes a Pandas table to csv in a way that's compatible with the .csv output
     format.  Currently just a wrapper around to_csv that forces output writing
     to go through a common code path.
+
+    Args:
+        detection_results (DataFrame): dataframe to write to [filename]
+        filename (str): .csv filename to write
     """
 
     print('Writing detection results to {}'.format(filename))

megadetector/postprocessing/md_to_coco.py

@@ -74,9 +74,9 @@ def md_to_coco(md_results_file,
             data (e.g. EXIF metadata) will be propagated to COCO output
         include_failed_images (bool, optional): if this is True, failed images will be propagated to COCO output
             with a non-empty "failure" field and no other fields, otherwise failed images will be skipped.
-        include_annotations_without_bounding_boxes (bool, optional): if this is True, annotations with
-            only class labels (no bounding boxes) will be included in the output.  If this is False, empty
-            images will be represented with no annotations.
+        include_annotations_without_bounding_boxes (bool, optional): the only time we end up with
+            annotations without bounding boxes is when a detection has the category [empty_category_id];
+            this determines whether those annotations are included in the output.
         empty_category_id (str, optional): category ID reserved for the 'empty' class, should not be
             attached to any bounding boxes
         overwrite_behavior (str, optional): determines behavior if the output file exists ('skip' to skip conversion,
@@ -257,7 +257,7 @@ def md_to_coco(md_results_file,
             else:
 
                 # In very esoteric cases, we use the empty category (0) in MD-formatted output files
-                print('Warning: empty category ({}) used for annotation in file {}'.format(
+                print('Warning: empty category ({}) used for annotation for image {}'.format(
                     empty_category_id,im['file']))
                 pass
 

megadetector/postprocessing/md_to_labelme.py

@@ -108,8 +108,13 @@ def get_labelme_dict_for_image(im,image_base_name=None,category_id_to_name=None,
 # ...def get_labelme_dict_for_image()
 
 
-def _write_output_for_image(im,image_base,extension_prefix,info,
-                            confidence_threshold,category_id_to_name,overwrite,
+def _write_output_for_image(im,
+                            image_base,
+                            extension_prefix,
+                            info,
+                            confidence_threshold,
+                            category_id_to_name,
+                            overwrite,
                             verbose=False):
 
     if 'failure' in im and im['failure'] is not None:
@@ -140,9 +145,14 @@ def _write_output_for_image(im,image_base,extension_prefix,info,
 
 
 
-def md_to_labelme(results_file,image_base,confidence_threshold=None,
-                  overwrite=False,extension_prefix='',n_workers=1,
-                  use_threads=False,bypass_image_size_read=False,
+def md_to_labelme(results_file,
+                  image_base,
+                  confidence_threshold=None,
+                  overwrite=False,
+                  extension_prefix='',
+                  n_workers=1,
+                  use_threads=False,
+                  bypass_image_size_read=False,
                   verbose=False):
     """
     For all the images in [results_file], write a .json file in labelme format alongside the
@@ -153,11 +163,12 @@ def md_to_labelme(results_file,image_base,confidence_threshold=None,
         image_base (str): folder of images; filenames in [results_file] should be relative to
             this folder
         confidence_threshold (float, optional): only detections at or above this confidence threshold
-            will be included in the output dict
+            will be included in the output dict.  If None, no threshold will be applied.
         overwrite (bool, optional): whether to overwrite existing output files; if this is False
             and the output file for an image exists, we'll skip that image
         extension_prefix (str, optional): if non-empty, "extension_prefix" will be inserted before the .json
-            extension
+            extension (typically used to generate multiple copies of labelme files representing different
+            MD thresholds)
         n_workers (int, optional): enables multiprocessing if > 1
         use_threads (bool, optional): if [n_workers] > 1, determines whether we parallelize via threads (True)
             or processes (False)

megadetector/postprocessing/merge_detections.py

@@ -85,6 +85,11 @@ def merge_detections(source_files,target_file,output_file,options=None):
 
     The results are written to [output_file].
 
+    Args:
+        source_files (list of str): list of files to merge into the results in [target_file]
+        target_file (str): filename that is treated as the primary source of results
+        output_file (str): file to which we should write merged results
+        options (MergeDetectionsOptions, optional): see MergeDetectionsOptions
     """
 
     if isinstance(source_files,str):

megadetector/postprocessing/postprocess_batch_results.py

@@ -1001,13 +1001,16 @@ def process_batch_results(options):
             options.separate_detections_by_category = False
 
         ground_truth_indexed_db = IndexedJsonDb(
-            options.ground_truth_json_file, b_normalize_paths=True,
+            options.ground_truth_json_file,
+            b_normalize_paths=True,
             filename_replacements=options.ground_truth_filename_replacements)
 
         # Mark images in the ground truth as positive or negative
         n_negative, n_positive, n_unknown, n_ambiguous = _mark_detection_status(
-            ground_truth_indexed_db, negative_classes=options.negative_classes,
+            ground_truth_indexed_db,
+            negative_classes=options.negative_classes,
             unknown_classes=options.unlabeled_classes)
+
         print(f'Finished loading and indexing ground truth: {n_negative} '
               f'negative, {n_positive} positive, {n_unknown} unknown, '
               f'{n_ambiguous} ambiguous')
@@ -1348,7 +1351,7 @@ def process_batch_results(options):
             # Prepend class name on each line and add to the top
             cm_str_lines = [' ' * 16 + ' '.join(classname_headers)]
             cm_str_lines += ['{:>15}'.format(cn[:15]) + ' ' + cm_line for cn, cm_line in \
-                             zip(classname_list, cm_str.splitlines())]
+                             zip(classname_list, cm_str.splitlines(), strict=True)]
 
             # Print formatted confusion matrix
             if False:

megadetector/postprocessing/remap_detection_categories.py

@@ -11,8 +11,9 @@ Currently only supports remapping detection categories, not classification categ
 
 #%% Constants and imports
 
-import json
 import os
+import json
+import argparse
 
 from tqdm import tqdm
 
@@ -167,4 +168,56 @@ if False:
 
 #%% Command-line driver
 
-# TODO
+def main():
+    """
+    Command-line interface for remapping detection categories in a MegaDetector results file
+    """
+
+    parser = argparse.ArgumentParser(
+        description='Remap detection categories in a MegaDetector .json results file'
+    )
+    parser.add_argument(
+        'input_file',
+        type=str,
+        help='Path to the MegaDetector .json results file to remap'
+    )
+    parser.add_argument(
+        'output_file',
+        type=str,
+        help='Path to save the remapped .json results file'
+    )
+    parser.add_argument(
+        'target_category_map_file',
+        type=str,
+        help="Path to a MegaDetector .json results file from which to take the target 'detection_categories' mapping"
+    )
+    parser.add_argument(
+        '--extra_category_handling',
+        type=str,
+        default='error',
+        choices=['error', 'drop_if_unused'],
+        help="How to handle source categories not in target map (default: 'error')"
+    )
+    parser.add_argument(
+        '--overwrite',
+        type=str,
+        default='false',
+        choices=['true', 'false'],
+        help="Overwrite output file if it exists (default: 'false')."
+    )
+
+    args = parser.parse_args()
+
+    overwrite_bool = (args.overwrite.lower() == 'true')
+
+    print('Starting category remapping...')
+
+    remap_detection_categories(
+        input_file=args.input_file,
+        output_file=args.output_file,
+        target_category_map=args.target_category_map_file, # Pass filename directly
+        extra_category_handling=args.extra_category_handling,
+        overwrite=overwrite_bool)
+
+if __name__ == '__main__':
+    main()

megadetector/postprocessing/render_detection_confusion_matrix.py

@@ -118,25 +118,28 @@ def render_detection_confusion_matrix(ground_truth_file,
             [results_file] should be relative to this folder.
         preview_folder (str): the output folder, i.e. the folder in which we'll create our nifty
             HTML stuff.
-        force_rendering_images (bool, optional): if False, skips images that already exist
+        force_render_images (bool, optional): if False, skips images that already exist
         confidence_thresholds (dict, optional): a dictionary mapping class names to thresholds;
             all classes not explicitly named here will use the threshold for the "default" category.
-        rendering_thresholds (dict, optional): a dictionary mapping class names to thresholds;
+        rendering_confidence_thresholds (dict, optional): a dictionary mapping class names to thresholds;
             all classes not explicitly named here will use the threshold for the "default" category.
         target_image_size (tuple, optional): output image size, as a pair of ints (width,height).  If one
             value is -1 and the other is not, aspect ratio is preserved.  If both are -1, the original image
             sizes are preserved.
         parallelize_rendering (bool, optional): enable (default) or disable parallelization when rendering
-        parallelize_rendering_n_core (int, optional): number of threads or processes to use for rendering, only
+        parallelize_rendering_n_cores (int, optional): number of threads or processes to use for rendering, only
             used if parallelize_rendering is True
-        parallelize_rendering_with_threads: whether to use threads (True) or processes (False) when rendering,
-            only used if parallelize_rendering is True
+        parallelize_rendering_with_threads (bool, optional): whether to use threads (True) or processes (False)
+            when rendering, only used if parallelize_rendering is True
         job_name (str, optional): job name to include in big letters in the output file
-        model_file (str, optional) model filename to include in HTML output
+        model_file (str, optional): model filename to include in HTML output
         empty_category_name (str, optional): special category name that we should treat as empty, typically
             "empty"
         html_image_list_options (dict, optional): options listed passed along to write_html_image_list;
             see write_html_image_list for documentation.
+
+    Returns:
+        dict: confusion matrix information, containing at least the key "html_file"
     """
 
     ##%% Argument and path handling

megadetector/postprocessing/repeat_detection_elimination/repeat_detections_core.py

@@ -1119,8 +1119,8 @@ def find_repeat_detections(input_filename, output_file_name=None, options=None):
         output_file_name (str, optional): the filename to which we should write results
             with repeat detections removed, typically set to None during the first
             part of the RDE process.
-        options (RepeatDetectionOptions): all the interesting options controlling this
-            process; see RepeatDetectionOptions for details.
+        options (RepeatDetectionOptions, optional): all the interesting options controlling
+            this process; see RepeatDetectionOptions for details.
 
     Returns:
         RepeatDetectionResults: results of the RDE process; see RepeatDetectionResults

megadetector/taxonomy_mapping/map_new_lila_datasets.py

@@ -15,10 +15,10 @@ import json
 # Created by get_lila_category_list.py
 input_lila_category_list_file = os.path.expanduser('~/lila/lila_categories_list/lila_dataset_to_categories.json')
 
-output_file = os.path.expanduser('~/lila/lila_additions_2025.03.24.csv')
+output_file = os.path.expanduser('~/lila/lila_additions_2025.06.23.csv')
 
 datasets_to_map = [
-    'UNSW Predators'
+    'Nkhotakota Camera Traps'
     ]
 
 
@@ -140,7 +140,7 @@ if False:
 
     #%%
 
-    q = 'dasyurus maculatus'
+    q = 'animalia'
 
     taxonomy_preference = 'inat'
     m = get_preferred_taxonomic_match(q,taxonomy_preference)
@@ -154,5 +154,4 @@ if False:
             # raise ValueError('')
         print(m.source)
         print(m.taxonomy_string)
-        # print(m.scientific_name); import clipboard; clipboard.copy(m.scientific_name)
         import clipboard; clipboard.copy(m.taxonomy_string)

megadetector/taxonomy_mapping/prepare_lila_taxonomy_release.py

@@ -52,6 +52,8 @@ if False:
 
     df['used'] = False
 
+    n_dropped = 0
+
     # i_row = 0; row = df.iloc[i_row]; row
     for i_row,row in df.iterrows():
         ds_name = row['dataset_name']
@@ -60,8 +62,11 @@ if False:
         if mapping_name in used_category_mappings:
             df.loc[i_row,'used'] = True
         else:
+            n_dropped += 1
             print('Dropping unused mapping {}'.format(mapping_name))
 
+    print('Dropping {} of {} mappings'.format(n_dropped,len(df)))
+
     df = df[df.used]
     df = df.drop('used',axis=1)
 
@@ -71,29 +76,41 @@ if False:
     assert not os.path.isfile(release_taxonomy_file), \
         'File {} exists, delete it manually before proceeding'.format(release_taxonomy_file)
 
-    known_levels = ['stateofmatter', #noqa
-                        'kingdom',
-                        'phylum','subphylum',
-                        'superclass','class','subclass','infraclass',
-                        'superorder','order','parvorder','suborder','infraorder',
-                        'zoosection',
-                        'superfamily','family','subfamily','tribe',
-                        'genus',
-                        'species','subspecies','variety']
-
     levels_to_include = ['kingdom',
-                        'phylum','subphylum',
-                        'superclass','class','subclass','infraclass',
-                        'superorder','order','suborder','infraorder',
-                        'superfamily','family','subfamily','tribe',
-                        'genus',
-                        'species','subspecies','variety']
-
-    levels_to_exclude = ['stateofmatter','zoosection','parvorder','complex','epifamily']
+                         'phylum',
+                         'subphylum',
+                         'superclass',
+                         'class',
+                         'subclass',
+                         'infraclass',
+                         'superorder',
+                         'order',
+                         'suborder',
+                         'infraorder',
+                         'superfamily',
+                         'family',
+                         'subfamily',
+                         'tribe',
+                         'genus',
+                         'subgenus',
+                         'species',
+                         'subspecies',
+                         'variety']
+
+    levels_to_exclude = ['stateofmatter',
+                         'zoosection',
+                         'parvorder',
+                         'complex',
+                         'epifamily']
+
+    for x in [levels_to_include,levels_to_exclude]:
+        assert len(x) == len(set(x))
 
     for s in levels_to_exclude:
         assert s not in levels_to_include
 
+    known_levels = levels_to_include + levels_to_exclude
+
     levels_used = set()
 
     # i_row = 0; row = df.iloc[i_row]; row
@@ -103,17 +120,21 @@ if False:
             assert not isinstance(row['taxonomy_string'],str)
             continue
 
+        # This is a list of length-4 tuples that each look like:
+        #
+        # (41789, 'species', 'taxidea taxus', ['american badger'])
         taxonomic_match = eval(row['taxonomy_string'])
 
         # match_at_level = taxonomic_match[0]
         for match_at_level in taxonomic_match:
             assert len(match_at_level) == 4
+            # E.g. "species"
             levels_used.add(match_at_level[1])
 
     levels_used = [s for s in levels_used if isinstance(s,str)]
 
     for s in levels_used:
-        assert s in levels_to_exclude or s in levels_to_include, 'Unrecognized level {}'.format(s)
+        assert s in known_levels, 'Unrecognized level {}'.format(s)
 
     for s in levels_to_include:
         assert s in levels_used

megadetector/taxonomy_mapping/preview_lila_taxonomy.py

@@ -16,7 +16,7 @@ import os
 import pandas as pd
 
 # lila_taxonomy_file = r"c:\git\agentmorrisprivate\lila-taxonomy\lila-taxonomy-mapping.csv"
-lila_taxonomy_file = os.path.expanduser('~/lila/lila_additions_2025.03.24.csv')
+lila_taxonomy_file = os.path.expanduser('~/lila/lila_additions_2025.06.23.csv')
 
 preview_base = os.path.expanduser('~/lila/lila_taxonomy_preview')
 os.makedirs(preview_base,exist_ok=True)