megadetector 10.0.3__tar.gz → 10.0.5__tar.gz

{megadetector-10.0.3/megadetector.egg-info → megadetector-10.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megadetector
-Version: 10.0.3
+Version: 10.0.5
 Summary: MegaDetector is an AI model that helps conservation folks spend less time doing boring things with camera trap images.
 Author-email: Your friendly neighborhood MegaDetector team <cameratraps@lila.science>
 Maintainer-email: Your friendly neighborhood MegaDetector team <cameratraps@lila.science>
@@ -38,7 +38,7 @@ Requires-Dist: numpy>=1.26.4
 Requires-Dist: Pillow>=9.5
 Requires-Dist: tqdm>=4.64.0
 Requires-Dist: jsonpickle>=3.0.2
-Requires-Dist: humanfriendly>=10.0
+Requires-Dist: humanfriendly>=2.1
 Requires-Dist: matplotlib>=3.8.0
 Requires-Dist: opencv-python>=4.8.0
 Requires-Dist: requests>=2.31.0

{megadetector-10.0.3 → megadetector-10.0.5}/README.md

@@ -274,7 +274,7 @@ Code written on or before April 28, 2023 is [copyright Microsoft](https://github
 
 ## Contributing
 
-This project welcomes contributions, as pull requests, issues, or suggestions by [email](mailto:cameratraps@lila.science).  We have a [list](https://github.com/agentmorris/MegaDetector/issues/84) of issues that we're hoping to address, many of which would be good starting points for new contributors.  We also depend on other open-source tools that help users run MegaDetector (particularly [AddaxAI](https://github.com/PetervanLunteren/AddaxAI) (formerly EcoAssist), and open-source tools that help users work with MegaDetector results (particularly [Timelapse](https://github.com/saulgreenberg/Timelapse)).  If you are looking to get involved in GUI development, reach out to the developers of those tools as well!
+This project welcomes contributions, as pull requests, issues, or suggestions by [email](mailto:cameratraps@lila.science).  We have a [list](https://dmorris.net/task-viewer/?file=https://raw.githubusercontent.com/agentmorris/MegaDetector/refs/heads/main/TODO.md) of issues that we're hoping to address, many of which would be good starting points for new contributors.  We also depend on other open-source tools that help users run MegaDetector (particularly [AddaxAI](https://github.com/PetervanLunteren/AddaxAI) (formerly EcoAssist), and open-source tools that help users work with MegaDetector results (particularly [Timelapse](https://github.com/saulgreenberg/Timelapse)).  If you are looking to get involved in GUI development, reach out to the developers of those tools as well!
 
 If you are interesting in getting involved in the conservation technology space, and MegaDetector just happens to be the first page you landed on, and none of our open issues are getting you fired up, don't fret!  Head over to the [WILDLABS discussion forums](https://wildlabs.net/discussions) and let the community know you're a developer looking to get involved.  Someone needs your help!
 

megadetector-10.0.5/megadetector/data_management/animl_to_md.py

@@ -0,0 +1,158 @@
+"""
+
+animl_to_md.py
+
+Convert a .csv file produced by the Animl package:
+
+https://github.com/conservationtechlab/animl-py
+
+...to a MD results file suitable for import into Timelapse.
+
+Columns are expected to be:
+
+file
+category (MD category identifies: 1==animal, 2==person, 3==vehicle)
+detection_conf
+bbox1,bbox2,bbox3,bbox4
+class
+classification_conf
+
+"""
+
+#%% Imports and constants
+
+import sys
+import argparse
+
+import pandas as pd
+
+from megadetector.utils.ct_utils import write_json
+from megadetector.detection.run_detector import DEFAULT_DETECTOR_LABEL_MAP
+detection_category_id_to_name = DEFAULT_DETECTOR_LABEL_MAP
+
+
+#%% Main function
+
+def animl_results_to_md_results(input_file,output_file=None):
+    """
+    Converts the .csv file [input_file] to the MD-formatted .json file [output_file].
+
+    If [output_file] is None, '.json' will be appended to the input file.
+    """
+
+    if output_file is None:
+        output_file = input_file + '.json'
+
+    df = pd.read_csv(input_file)
+
+    expected_columns = ('file','category','detection_conf',
+                        'bbox1','bbox2','bbox3','bbox4','class','classification_conf')
+
+    for s in expected_columns:
+        assert s in df.columns,\
+            'Expected column {} not found'.format(s)
+
+    classification_category_name_to_id = {}
+    filename_to_results = {}
+
+    # i_row = 0; row = df.iloc[i_row]
+    for i_row,row in df.iterrows():
+
+        # Is this the first detection we've seen for this file?
+        if row['file'] in filename_to_results:
+            im = filename_to_results[row['file']]
+        else:
+            im = {}
+            im['detections'] = []
+            im['file'] = row['file']
+            filename_to_results[im['file']] = im
+
+        assert isinstance(row['category'],int),'Invalid category identifier in row {}'.format(im['file'])
+        detection_category_id = str(row['category'])
+        assert detection_category_id in detection_category_id_to_name,\
+            'Unrecognized detection category ID {}'.format(detection_category_id)
+
+        detection = {}
+        detection['category'] = detection_category_id
+        detection['conf'] = row['detection_conf']
+        bbox = [row['bbox1'],row['bbox2'],row['bbox3'],row['bbox4']]
+        detection['bbox'] = bbox
+        classification_category_name = row['class']
+
+        # Have we seen this classification category before?
+        if classification_category_name in classification_category_name_to_id:
+            classification_category_id = \
+                classification_category_name_to_id[classification_category_name]
+        else:
+            classification_category_id = str(len(classification_category_name_to_id))
+            classification_category_name_to_id[classification_category_name] = \
+                classification_category_id
+
+        classifications = [[classification_category_id,row['classification_conf']]]
+        detection['classifications'] = classifications
+
+        im['detections'].append(detection)
+
+    # ...for each row
+
+    info = {}
+    info['format_version'] = '1.3'
+    info['detector'] = 'Animl'
+    info['classifier'] = 'Animl'
+
+    results = {}
+    results['info'] = info
+    results['detection_categories'] = detection_category_id_to_name
+    results['classification_categories'] = \
+        {v: k for k, v in classification_category_name_to_id.items()}
+    results['images'] = list(filename_to_results.values())
+
+    write_json(output_file,results)
+
+# ...animl_results_to_md_results(...)
+
+
+#%% Interactive driver
+
+if False:
+
+    pass
+
+    #%%
+
+    input_file = r"G:\temp\animl-runs\animl-runs\Coati_v2\manifest.csv"
+    output_file = None
+    animl_results_to_md_results(input_file,output_file)
+
+
+#%% Command-line driver
+
+def main():
+    """
+    Command-line driver for animl_to_md
+    """
+
+    parser = argparse.ArgumentParser(
+        description='Convert an Animl-formatted .csv results file to MD-formatted .json results file')
+
+    parser.add_argument(
+        'input_file',
+        type=str,
+        help='input .csv file')
+
+    parser.add_argument(
+        '--output_file',
+        type=str,
+        default=None,
+        help='output .json file (defaults to input file appended with ".json")')
+
+    if len(sys.argv[1:]) == 0:
+        parser.print_help()
+        parser.exit()
+
+    args = parser.parse_args()
+
+    animl_results_to_md_results(args.input_file,args.output_file)
+
+if __name__ == '__main__':
+    main()

{megadetector-10.0.3 → megadetector-10.0.5}/megadetector/data_management/cct_json_utils.py

@@ -298,6 +298,7 @@ class SequenceOptions:
     """
 
     def __init__(self):
+
         #: Images separated by <= this duration will be grouped into the same sequence.
         self.episode_interval_seconds = 60.0
 

{megadetector-10.0.3 → megadetector-10.0.5}/megadetector/data_management/speciesnet_to_md.py

@@ -3,7 +3,7 @@
 speciesnet_to_md.py
 
 Converts the WI (SpeciesNet) predictions.json format to MD .json format.  This is just a
-command-line wrapper around utils.wi_utils.generate_md_results_from_predictions_json.
+command-line wrapper around utils.wi_taxonomy_utils.generate_md_results_from_predictions_json.
 
 """
 
@@ -11,7 +11,7 @@ command-line wrapper around utils.wi_utils.generate_md_results_from_predictions_
 
 import sys
 import argparse
-from megadetector.utils.wi_utils import generate_md_results_from_predictions_json
+from megadetector.utils.wi_taxonomy_utils import generate_md_results_from_predictions_json
 
 
 #%% Command-line driver

megadetector-10.0.5/megadetector/data_management/zamba_to_md.py

@@ -0,0 +1,188 @@
+"""
+
+zamba_to_md.py
+
+Convert a labels.csv file produced by Zamba Cloud to a MD results file suitable
+for import into Timelapse.
+
+Columns are expected to be:
+
+video_uuid (not used)
+original_filename (assumed to be a relative path name)
+top_k_label,top_k_probability, for k = 1..N
+[category name 1],[category name 2],...
+corrected_label
+
+Because the MD results file fundamentally stores detections, what we'll
+actually do is create bogus detections that fill the entire image.
+
+There is no special handling of empty/blank categories; because these results are
+based on a classifier, rather than a detector (where "blank" would be the absence of
+all other categories), "blank" can be queried in Timelapse just like any other class.
+
+"""
+
+#%% Imports and constants
+
+import sys
+import argparse
+
+import pandas as pd
+
+from megadetector.utils.ct_utils import write_json
+
+
+#%% Main function
+
+def zamba_results_to_md_results(input_file,output_file=None):
+    """
+    Converts the .csv file [input_file] to the MD-formatted .json file [output_file].
+
+    If [output_file] is None, '.json' will be appended to the input file.
+
+    Args:
+        input_file (str): the .csv file to convert
+        output_file (str, optional): the output .json file (defaults to
+            [input_file].json)
+    """
+
+    if output_file is None:
+        output_file = input_file + '.json'
+
+    df = pd.read_csv(input_file)
+
+    expected_columns = ('video_uuid','corrected_label','original_filename')
+    for s in expected_columns:
+        assert s in df.columns,\
+            'Expected column {} not found, are you sure this is a Zamba results .csv file?'.format(
+                s)
+
+    # How many results are included per file?
+    assert 'top_1_probability' in df.columns and 'top_1_label' in df.columns
+    top_k = 2
+    while(True):
+        p_string = 'top_' + str(top_k) + '_probability'
+        label_string = 'top_' + str(top_k) + '_label'
+
+        if p_string in df.columns:
+            assert label_string in df.columns,\
+                'Oops, {} is a column but {} is not'.format(
+                    p_string,label_string)
+            top_k += 1
+            continue
+        else:
+            assert label_string not in df.columns,\
+                'Oops, {} is a column but {} is not'.format(
+                    label_string,p_string)
+            top_k -= 1
+            break
+
+    print('Found {} probability column pairs'.format(top_k))
+
+    # Category names start after the fixed columns and the probability columns
+    category_names = []
+    column_names = list(df.columns)
+    first_category_name_index = 0
+    while('top_' in column_names[first_category_name_index] or \
+          column_names[first_category_name_index] in expected_columns):
+        first_category_name_index += 1
+
+    i_column = first_category_name_index
+    while( (i_column < len(column_names)) and (column_names[i_column] != 'corrected_label') ):
+        category_names.append(column_names[i_column])
+        i_column += 1
+
+    print('Found {} categories:\n'.format(len(category_names)))
+
+    for s in category_names:
+        print(s)
+
+    info = {}
+    info['format_version'] = '1.3'
+    info['detector'] = 'Zamba Cloud'
+    info['classifier'] = 'Zamba Cloud'
+
+    detection_category_id_to_name = {}
+    for category_id,category_name in enumerate(category_names):
+        detection_category_id_to_name[str(category_id)] = category_name
+    detection_category_name_to_id = {v: k for k, v in detection_category_id_to_name.items()}
+
+    images = []
+
+    # i_row = 0; row = df.iloc[i_row]
+    for i_row,row in df.iterrows():
+
+        im = {}
+        images.append(im)
+        im['file'] = row['original_filename']
+
+        detections = []
+
+        # k = 1
+        for k in range(1,top_k+1):
+            label = row['top_{}_label'.format(k)]
+            confidence = row['top_{}_probability'.format(k)]
+            det = {}
+            det['category'] = detection_category_name_to_id[label]
+            det['conf'] = confidence
+            det['bbox'] = [0,0,1.0,1.0]
+            detections.append(det)
+
+        im['detections'] = detections
+
+    # ...for each row
+
+    results = {}
+    results['info'] = info
+    results['detection_categories'] = detection_category_id_to_name
+    results['images'] = images
+
+    write_json(output_file,results)
+
+# ...zamba_results_to_md_results(...)
+
+
+#%% Interactive driver
+
+if False:
+
+    pass
+
+    #%%
+
+    input_file = r"G:\temp\labels-job-b95a4b76-e332-4e17-ab40-03469392d36a-2023-11-04_16-28-50.060130.csv"
+    output_file = None
+    zamba_results_to_md_results(input_file,output_file)
+
+
+#%% Command-line driver
+
+def main():
+    """
+    Command-line driver for zamba_to_md
+    """
+
+    parser = argparse.ArgumentParser(
+        description='Convert a Zamba-formatted .csv results file to a MD-formatted .json results file')
+
+    parser.add_argument(
+        'input_file',
+        type=str,
+        help='input .csv file')
+
+    parser.add_argument(
+        '--output_file',
+        type=str,
+        default=None,
+        help='output .json file (defaults to input file appended with ".json")')
+
+    if len(sys.argv[1:]) == 0:
+        parser.print_help()
+        parser.exit()
+
+    args = parser.parse_args()
+
+    zamba_results_to_md_results(args.input_file,args.output_file)
+
+if __name__ == '__main__':
+    main()

{megadetector-10.0.3 → megadetector-10.0.5}/megadetector/detection/process_video.py

@@ -22,11 +22,16 @@ from megadetector.detection import run_detector_batch
 from megadetector.utils.ct_utils import args_to_object
 from megadetector.utils.ct_utils import dict_to_kvp_list, parse_kvp_list
 from megadetector.detection.video_utils import _filename_to_frame_number
+from megadetector.detection.video_utils import find_videos
 from megadetector.detection.video_utils import run_callback_on_frames_for_folder
 from megadetector.detection.run_detector import load_detector
 from megadetector.postprocessing.validate_batch_results import \
         ValidateBatchResultsOptions, validate_batch_results
 
+# Notes to self re: upcoming work on checkpointing
+from megadetector.utils.ct_utils import split_list_into_fixed_size_chunks # noqa
+from megadetector.detection.run_detector_batch import write_checkpoint, load_checkpoint # noqa
+
 
 #%% Classes
 
@@ -68,16 +73,6 @@ class ProcessVideoOptions:
         #: Sample frames every N seconds.  Mutually exclusive with [frame_sample]
         self.time_sample = None
 
-        #: Number of workers to use for parallelization; set to <= 1 to disable parallelization
-        self.n_cores = 1
-
-        #: For debugging only, stop processing after a certain number of frames.
-        self.debug_max_frames = -1
-
-        #: File containing non-standard categories, typically only used if you're running a non-MD
-        #: detector.
-        self.class_mapping_filename = None
-
         #: Run the model at this image size (don't mess with this unless you know what you're
         #: getting into)... if you just want to pass smaller frames to MD, use max_width
         self.image_size = None
@@ -92,6 +87,17 @@ class ProcessVideoOptions:
         #: Detector-specific options
         self.detector_options = None
 
+        #: Write a checkpoint file (to resume processing later) every N videos;
+        #: set to -1 (default) to disable checkpointing
+        self.checkpoint_frequency = -1
+
+        #: Path to checkpoint file; None (default) for auto-generation based on output filename
+        self.checkpoint_path = None
+
+        #: Resume from a checkpoint file, or "auto" to use the most recent checkpoint in the
+        #: output directory
+        self.resume_from_checkpoint = None
+
 # ...class ProcessVideoOptions
 
 
@@ -137,7 +143,7 @@ def process_videos(options):
         every_n_frames_param = options.frame_sample
 
     if options.verbose:
-        print('Running MegaDetector for folder {}'.format(options.input_video_file))
+        print('Processing videos from input source {}'.format(options.input_video_file))
 
     detector = load_detector(options.model_file,detector_options=options.detector_options)
 
@@ -145,7 +151,9 @@ def process_videos(options):
         return detector.generate_detections_one_image(image_np,
                                                       image_id,
                                                       detection_threshold=options.json_confidence_threshold,
-                                                      augment=options.augment)
+                                                      augment=options.augment,
+                                                      image_size=options.image_size,
+                                                      verbose=options.verbose)
 
     """
     [md_results] will be dict with keys 'video_filenames' (list of str), 'frame_rates' (list of floats),
@@ -164,7 +172,8 @@ def process_videos(options):
                                                        frame_callback=frame_callback,
                                                        every_n_frames=every_n_frames_param,
                                                        verbose=options.verbose,
-                                                       files_to_process_relative=[video_bn])
+                                                       files_to_process_relative=[video_bn],
+                                                       allow_empty_videos=options.allow_empty_videos)
 
     else:
 
@@ -172,10 +181,15 @@ def process_videos(options):
             '{} is neither a file nor a folder'.format(options.input_video_file)
 
         video_folder = options.input_video_file
+
         md_results = run_callback_on_frames_for_folder(input_video_folder=options.input_video_file,
                                                        frame_callback=frame_callback,
                                                        every_n_frames=every_n_frames_param,
-                                                       verbose=options.verbose)
+                                                       verbose=options.verbose,
+                                                       recursive=options.recursive,
+                                                       allow_empty_videos=options.allow_empty_videos)
+
+    # ...whether we're processing a file or a folder
 
     print('Finished running MD on videos')
 
@@ -231,12 +245,12 @@ def process_videos(options):
 
         # ...was this a failed video?
 
+        im['frames_processed'] = sorted(im['frames_processed'])
+
         video_list_md_format.append(im)
 
     # ...for each video
 
-    im['frames_processed'] = sorted(im['frames_processed'])
-
     run_detector_batch.write_results_to_file(
         video_list_md_format,
         options.output_json_file,
@@ -276,14 +290,8 @@ def options_to_command(options):
         cmd += ' --output_json_file' + ' "' + options.output_json_file + '"'
     if options.json_confidence_threshold is not None:
         cmd += ' --json_confidence_threshold ' + str(options.json_confidence_threshold)
-    if options.n_cores is not None:
-        cmd += ' --n_cores ' + str(options.n_cores)
     if options.frame_sample is not None:
         cmd += ' --frame_sample ' + str(options.frame_sample)
-    if options.debug_max_frames is not None:
-        cmd += ' --debug_max_frames ' + str(options.debug_max_frames)
-    if options.class_mapping_filename is not None:
-        cmd += ' --class_mapping_filename ' + str(options.class_mapping_filename)
     if options.verbose:
         cmd += ' --verbose'
     if options.detector_options is not None and len(options.detector_options) > 0:
@@ -384,13 +392,6 @@ def main(): # noqa
                             'below this threshold (default {})'.format(
                                 default_options.json_confidence_threshold))
 
-    parser.add_argument('--n_cores', type=int,
-                        default=default_options.n_cores,
-                        help='Number of cores to use for frame separation and detection. '\
-                            'If using a GPU, this option will be respected for frame separation but '\
-                            'ignored for detection.  Only relevant to frame separation when processing '\
-                            'a folder.  Default {}.'.format(default_options.n_cores))
-
     parser.add_argument('--frame_sample', type=int,
                         default=None, help='process every Nth frame (defaults to every frame), mutually exclusive '\
                             'with --time_sample.')
@@ -400,17 +401,6 @@ def main(): # noqa
                             'frame sampling rate, so it may not be exactly the requested interval in seconds. '\
                             'mutually exclusive with --frame_sample')
 
-    parser.add_argument('--debug_max_frames', type=int,
-                        default=-1, help='Trim to N frames for debugging (impacts model execution, '\
-                            'not frame rendering)')
-
-    parser.add_argument('--class_mapping_filename',
-                        type=str,
-                        default=None, help='Use a non-default class mapping, supplied in a .json file '\
-                            'with a dictionary mapping int-strings to strings.  This will also disable '\
-                            'the addition of "1" to all category IDs, so your class mapping should start '\
-                            'at zero.')
-
     parser.add_argument('--verbose', action='store_true',
                         help='Enable additional debug output')
 
@@ -435,6 +425,28 @@ def main(): # noqa
         default='',
         help='Detector-specific options, as a space-separated list of key-value pairs')
 
+    parser.add_argument(
+        '--checkpoint_frequency',
+        type=int,
+        default=default_options.checkpoint_frequency,
+        help='Write a checkpoint file (to resume processing later) every N videos; ' + \
+             'set to -1 to disable checkpointing (default {})'.format(
+                 default_options.checkpoint_frequency))
+
+    parser.add_argument(
+        '--checkpoint_path',
+        type=str,
+        default=None,
+        help='Path to checkpoint file; defaults to a file in the same directory ' + \
+             'as the output file')
+
+    parser.add_argument(
+        '--resume_from_checkpoint',
+        type=str,
+        default=None,
+        help='Resume from a specific checkpoint file, or "auto" to resume from the ' + \
+             'most recent checkpoint in the output directory')
+
     if len(sys.argv[1:]) == 0:
         parser.print_help()
         parser.exit()