megadetector 10.0.4__tar.gz → 10.0.6__tar.gz

{megadetector-10.0.4/megadetector.egg-info → megadetector-10.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megadetector
-Version: 10.0.4
+Version: 10.0.6
 Summary: MegaDetector is an AI model that helps conservation folks spend less time doing boring things with camera trap images.
 Author-email: Your friendly neighborhood MegaDetector team <cameratraps@lila.science>
 Maintainer-email: Your friendly neighborhood MegaDetector team <cameratraps@lila.science>
@@ -38,7 +38,7 @@ Requires-Dist: numpy>=1.26.4
 Requires-Dist: Pillow>=9.5
 Requires-Dist: tqdm>=4.64.0
 Requires-Dist: jsonpickle>=3.0.2
-Requires-Dist: humanfriendly>=10.0
+Requires-Dist: humanfriendly>=2.1
 Requires-Dist: matplotlib>=3.8.0
 Requires-Dist: opencv-python>=4.8.0
 Requires-Dist: requests>=2.31.0

{megadetector-10.0.4 → megadetector-10.0.6}/megadetector/data_management/cct_json_utils.py

@@ -298,6 +298,7 @@ class SequenceOptions:
     """
 
     def __init__(self):
+
         #: Images separated by <= this duration will be grouped into the same sequence.
         self.episode_interval_seconds = 60.0
 

{megadetector-10.0.4 → megadetector-10.0.6}/megadetector/data_management/speciesnet_to_md.py

@@ -3,7 +3,7 @@
 speciesnet_to_md.py
 
 Converts the WI (SpeciesNet) predictions.json format to MD .json format.  This is just a
-command-line wrapper around utils.wi_utils.generate_md_results_from_predictions_json.
+command-line wrapper around utils.wi_taxonomy_utils.generate_md_results_from_predictions_json.
 
 """
 
@@ -11,7 +11,7 @@ command-line wrapper around utils.wi_utils.generate_md_results_from_predictions_
 
 import sys
 import argparse
-from megadetector.utils.wi_utils import generate_md_results_from_predictions_json
+from megadetector.utils.wi_taxonomy_utils import generate_md_results_from_predictions_json
 
 
 #%% Command-line driver

{megadetector-10.0.4 → megadetector-10.0.6}/megadetector/detection/process_video.py

@@ -81,8 +81,8 @@ class ProcessVideoOptions:
         self.augment = False
 
         #: By default, a video with no frames (or no frames retrievable with the current parameters)
-        #: is treated as a failure; this causes it to be treated as a video with no detections.
-        self.allow_empty_videos = False
+        #: is silently stored as a failure; this causes it to halt execution.
+        self.exit_on_empty_video = False
 
         #: Detector-specific options
         self.detector_options = None
@@ -134,6 +134,14 @@ def process_videos(options):
     # Check for incompatible options
     _validate_video_options(options)
 
+    if options.output_json_file is None:
+        video_file = options.input_video_file.replace('\\','/')
+        if video_file.endswith('/'):
+            video_file = video_file[:-1]
+        options.output_json_file = video_file + '.json'
+        print('Output file not specified, defaulting to {}'.format(
+            options.output_json_file))
+
     assert options.output_json_file.endswith('.json'), \
         'Illegal output file {}'.format(options.output_json_file)
 
@@ -173,7 +181,7 @@ def process_videos(options):
                                                        every_n_frames=every_n_frames_param,
                                                        verbose=options.verbose,
                                                        files_to_process_relative=[video_bn],
-                                                       allow_empty_videos=options.allow_empty_videos)
+                                                       error_on_empty_video=options.exit_on_empty_video)
 
     else:
 
@@ -187,7 +195,7 @@ def process_videos(options):
                                                        every_n_frames=every_n_frames_param,
                                                        verbose=options.verbose,
                                                        recursive=options.recursive,
-                                                       allow_empty_videos=options.allow_empty_videos)
+                                                       error_on_empty_video=options.exit_on_empty_video)
 
     # ...whether we're processing a file or a folder
 
@@ -414,9 +422,10 @@ def main(): # noqa
                         action='store_true',
                         help='Enable image augmentation')
 
-    parser.add_argument('--allow_empty_videos',
+    parser.add_argument('--exit_on_empty_video',
                         action='store_true',
-                        help='By default, videos with no retrievable frames cause an error, this makes it a warning')
+                        help=('By default, videos with no retrievable frames are stored as failures; this' \
+                              'causes them to halt execution'))
 
     parser.add_argument(
         '--detector_options',

{megadetector-10.0.4 → megadetector-10.0.6}/megadetector/detection/video_utils.py

@@ -101,6 +101,99 @@ def find_videos(dirname,
     return find_video_strings(files)
 
 
+#%% Shared function for opening videos
+
+DEFAULT_BACKEND = -1
+
+# This is the order in which we'll try to open backends.
+#
+# In general, the defaults are as follows, though they vary depending
+# on what's installed:
+#
+# Windows: CAP_DSHOW or CAP_MSMF
+# Linux: CAP_FFMPEG
+# macOS: CAP_AVFOUNDATION
+#
+# Technically if the default fails, we may try the same backend again, but this
+# is rare, and it's not worth the complexity of figuring out what the system
+# default is.
+backend_id_to_name = {
+    DEFAULT_BACKEND:'default',
+    cv2.CAP_FFMPEG: 'CAP_FFMPEG',
+    cv2.CAP_DSHOW: 'CAP_DSHOW',
+    cv2.CAP_MSMF: 'CAP_MSMF',
+    cv2.CAP_AVFOUNDATION: 'CAP_AVFOUNDATION',
+    cv2.CAP_GSTREAMER: 'CAP_GSTREAMER'
+}
+
+def open_video(video_path,verbose=False):
+    """
+    Open the video at [video_path], trying multiple OpenCV backends if necessary.
+
+    Args:
+        video_path (str): the file to open
+        verbose (bool, optional): enable additional debug output
+
+    Returns:
+        (cv2.VideoCapture,image): a tuple containing (a) the open video capture device
+        (or None if no backends succeeded) and (b) the first frame of the video (or None)
+    """
+
+    if not os.path.isfile(video_path):
+        print('Video file {} not found'.format(video_path))
+        return None,None
+
+    backend_ids = backend_id_to_name.keys()
+
+    for backend_id in backend_ids:
+
+        backend_name = backend_id_to_name[backend_id]
+        if verbose:
+            print('Trying backend {}'.format(backend_name))
+
+        try:
+            if backend_id == DEFAULT_BACKEND:
+                vidcap = cv2.VideoCapture(video_path)
+            else:
+                vidcap = cv2.VideoCapture(video_path, backend_id)
+        except Exception as e:
+            if verbose:
+                print('Warning: error opening {} with backend {}: {}'.format(
+                    video_path,backend_name,str(e)))
+            continue
+
+        if not vidcap.isOpened():
+            if verbose:
+                print('Warning: isOpened() is False for {} with backend {}'.format(
+                    video_path,backend_name))
+            try:
+                vidcap.release()
+            except Exception:
+                pass
+            continue
+
+        success, image = vidcap.read()
+        if success and (image is not None):
+            if verbose:
+                print('Successfully opened {} with backend: {}'.format(
+                    video_path,backend_name))
+            return vidcap,image
+
+        print('Warning: failed to open {} with backend {}'.format(
+            video_path,backend_name))
+        try:
+            vidcap.release()
+        except Exception:
+            pass
+
+    # ...for each backend
+
+    print('Error: failed to open {} with any backend'.format(video_path))
+    return None,None
+
+# ...def open_video(...)
+
+
 #%% Functions for rendering frames to video and vice-versa
 
 # http://tsaith.github.io/combine-images-into-a-video-with-python-3-and-opencv-3.html
@@ -146,21 +239,32 @@ def frames_to_video(images, fs, output_file_name, codec_spec=default_fourcc):
     cv2.destroyAllWindows()
 
 
-def get_video_fs(input_video_file):
+def get_video_fs(input_video_file,verbose=False):
     """
     Retrieves the frame rate of [input_video_file].
 
     Args:
         input_video_file (str): video file for which we want the frame rate
+        verbose (bool, optional): enable additional debug output
 
     Returns:
-        float: the frame rate of [input_video_file]
+        float: the frame rate of [input_video_file], or None if no frame
+            rate could be extracted
     """
 
-    assert os.path.isfile(input_video_file), 'File {} not found'.format(input_video_file)
-    vidcap = cv2.VideoCapture(input_video_file)
+    assert os.path.isfile(input_video_file), \
+        'File {} not found'.format(input_video_file)
+    vidcap,_ = open_video(input_video_file,verbose=verbose)
+    if vidcap is None:
+        if verbose:
+            print('Failed to get frame rate for {}'.format(input_video_file))
+        return None
     fs = vidcap.get(cv2.CAP_PROP_FPS)
-    vidcap.release()
+    try:
+        vidcap.release()
+    except Exception as e:
+        print('Warning: error closing video handle for {}: {}'.format(
+            input_video_file,str(e)))
     return fs
 
 
@@ -249,7 +353,7 @@ def run_callback_on_frames(input_video_file,
             of the video, no frames are extracted.  Can also be a single int, specifying
             a single frame number.
         allow_empty_videos (bool, optional): Just print a warning if a video appears to have no
-            frames (by default, this is an error).
+            frames (by default, this raises an Exception).
 
     Returns:
         dict: dict with keys 'frame_filenames' (list), 'frame_rate' (float), 'results' (list).
@@ -271,7 +375,7 @@ def run_callback_on_frames(input_video_file,
 
     try:
 
-        vidcap = cv2.VideoCapture(input_video_file)
+        vidcap,image = open_video(input_video_file,verbose=verbose)
         n_frames = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
         frame_rate = vidcap.get(cv2.CAP_PROP_FPS)
 
@@ -300,7 +404,11 @@ def run_callback_on_frames(input_video_file,
         # frame_number = 0
         for frame_number in range(0,n_frames):
 
-            success,image = vidcap.read()
+            # We've already read the first frame, when we opened the video
+            if frame_number != 0:
+                success,image = vidcap.read()
+            else:
+                success = True
 
             if not success:
                 assert image is None
@@ -363,9 +471,9 @@ def run_callback_on_frames_for_folder(input_video_folder,
                                       frame_callback,
                                       every_n_frames=None,
                                       verbose=False,
-                                      allow_empty_videos=False,
                                       recursive=True,
-                                      files_to_process_relative=None):
+                                      files_to_process_relative=None,
+                                      error_on_empty_video=False):
     """
     Calls the function frame_callback(np.array,image_id) on all (or selected) frames in
     all videos in [input_video_folder].
@@ -382,10 +490,10 @@ def run_callback_on_frames_for_folder(input_video_folder,
             interpreted as a sampling rate in seconds, which is rounded to the nearest frame
             sampling rate.
         verbose (bool, optional): enable additional debug console output
-        allow_empty_videos (bool, optional): Just print a warning if a video appears to have no
-            frames (by default, this is an error).
         recursive (bool, optional): recurse into [input_video_folder]
         files_to_process_relative (list, optional): only process specific relative paths
+        error_on_empty_video (bool, optional): by default, videos with errors or no valid frames
+            are silently stored as failures; this turns them into exceptions
 
     Returns:
         dict: dict with keys 'video_filenames' (list of str), 'frame_rates' (list of floats),
@@ -440,18 +548,21 @@ def run_callback_on_frames_for_folder(input_video_folder,
                                                    every_n_frames=every_n_frames,
                                                    verbose=verbose,
                                                    frames_to_process=None,
-                                                   allow_empty_videos=allow_empty_videos)
+                                                   allow_empty_videos=False)
 
         except Exception as e:
 
-            print('Warning: error processing video {}: {}'.format(
-                video_fn_abs,str(e)
-            ))
-            to_return['frame_rates'].append(-1.0)
-            failure_result = {}
-            failure_result['failure'] = 'Failure processing video: {}'.format(str(e))
-            to_return['results'].append(failure_result)
-            continue
+            if (not error_on_empty_video):
+                print('Warning: error processing video {}: {}'.format(
+                    video_fn_abs,str(e)
+                ))
+                to_return['frame_rates'].append(-1.0)
+                failure_result = {}
+                failure_result['failure'] = 'Failure processing video: {}'.format(str(e))
+                to_return['results'].append(failure_result)
+                continue
+            else:
+                raise
 
         # ...try/except
 

{megadetector-10.0.4 → megadetector-10.0.6}/megadetector/postprocessing/classification_postprocessing.py

@@ -25,14 +25,14 @@ from megadetector.utils.ct_utils import sort_dictionary_by_value
 from megadetector.utils.ct_utils import sort_dictionary_by_key
 from megadetector.utils.ct_utils import invert_dictionary
 
-from megadetector.utils.wi_utils import clean_taxonomy_string
-from megadetector.utils.wi_utils import taxonomy_level_index
-from megadetector.utils.wi_utils import taxonomy_level_string_to_index
+from megadetector.utils.wi_taxonomy_utils import clean_taxonomy_string
+from megadetector.utils.wi_taxonomy_utils import taxonomy_level_index
+from megadetector.utils.wi_taxonomy_utils import taxonomy_level_string_to_index
 
-from megadetector.utils.wi_utils import non_taxonomic_prediction_strings
-from megadetector.utils.wi_utils import human_prediction_string
-from megadetector.utils.wi_utils import animal_prediction_string
-from megadetector.utils.wi_utils import blank_prediction_string # noqa
+from megadetector.utils.wi_taxonomy_utils import non_taxonomic_prediction_strings
+from megadetector.utils.wi_taxonomy_utils import human_prediction_string
+from megadetector.utils.wi_taxonomy_utils import animal_prediction_string
+from megadetector.utils.wi_taxonomy_utils import blank_prediction_string # noqa
 
 
 #%% Options classes
@@ -1100,7 +1100,8 @@ def restrict_to_taxa_list(taxa_list,
                           input_file,
                           output_file,
                           allow_walk_down=False,
-                          add_pre_filtering_description=True):
+                          add_pre_filtering_description=True,
+                          allow_redundant_latin_names=False):
     """
     Given a prediction file in MD .json format, likely without having had
     a geofence applied, apply a custom taxa list.
@@ -1123,6 +1124,10 @@ def restrict_to_taxa_list(taxa_list,
         add_pre_filtering_description (bool, optional): should we add a new metadata
             field that summarizes each image's classifications prior to taxonomic
             restriction?
+        allow_redundant_latin_names (bool, optional): if False, we'll raise an Exception
+            if the same latin name appears twice in the taxonomy list; if True, we'll
+            just print a warning and ignore all entries other than the first for this
+            latin name
     """
 
     ##%% Read target taxa list
@@ -1137,11 +1142,14 @@ def restrict_to_taxa_list(taxa_list,
     taxa_list = [s for s in taxa_list if len(s) > 0]
 
     target_latin_to_common = {}
+
     for s in taxa_list:
+
         if s.strip().startswith('#'):
             continue
         tokens = s.split(',')
-        assert len(tokens) <= 2
+        # We allow additional columns now
+        # assert len(tokens) <= 2
         binomial_name = tokens[0]
         assert len(binomial_name.split(' ')) in (1,2,3), \
             'Illegal binomial name in species list: {}'.format(binomial_name)
@@ -1149,9 +1157,17 @@ def restrict_to_taxa_list(taxa_list,
             common_name = tokens[1].strip().lower()
         else:
             common_name = None
-        assert binomial_name not in target_latin_to_common
+        if binomial_name in target_latin_to_common:
+            error_string = 'scientific name {} appears multiple times in the taxonomy list'.format(
+                    binomial_name)
+            if allow_redundant_latin_names:
+                print('Warning: {}'.format(error_string))
+            else:
+                raise ValueError(error_string)
         target_latin_to_common[binomial_name] = common_name
 
+    # ...for each line in the taxonomy file
+
 
     ##%% Read taxonomy file
 

{megadetector-10.0.4 → megadetector-10.0.6}/megadetector/postprocessing/generate_csv_report.py

@@ -42,7 +42,7 @@ import pandas as pd
 
 from copy import deepcopy
 
-from megadetector.utils.wi_utils import load_md_or_speciesnet_file
+from megadetector.utils.wi_taxonomy_utils import load_md_or_speciesnet_file
 from megadetector.utils.ct_utils import get_max_conf
 from megadetector.utils.ct_utils import is_list_sorted
 from megadetector.detection.run_detector import \

{megadetector-10.0.4 → megadetector-10.0.6}/megadetector/postprocessing/load_api_results.py

@@ -24,7 +24,7 @@ from collections.abc import Mapping
 import pandas as pd
 
 from megadetector.utils import ct_utils
-from megadetector.utils.wi_utils import load_md_or_speciesnet_file
+from megadetector.utils.wi_taxonomy_utils import load_md_or_speciesnet_file
 
 
 #%% Functions for loading .json results into a Pandas DataFrame, and writing back to .json

{megadetector-10.0.4 → megadetector-10.0.6}/megadetector/postprocessing/md_to_wi.py

@@ -10,7 +10,7 @@ Converts the MD .json format to the WI predictions.json format.
 
 import sys
 import argparse
-from megadetector.utils.wi_utils import generate_predictions_json_from_md_results
+from megadetector.utils.wi_taxonomy_utils import generate_predictions_json_from_md_results
 
 
 #%% Command-line driver

{megadetector-10.0.4 → megadetector-10.0.6}/megadetector/postprocessing/postprocess_batch_results.py

@@ -47,7 +47,7 @@ from tqdm import tqdm
 from megadetector.visualization import visualization_utils as vis_utils
 from megadetector.visualization import plot_utils
 from megadetector.utils.write_html_image_list import write_html_image_list
-from megadetector.utils.wi_utils import load_md_or_speciesnet_file
+from megadetector.utils.wi_taxonomy_utils import load_md_or_speciesnet_file
 from megadetector.utils import path_utils
 from megadetector.utils.ct_utils import args_to_object
 from megadetector.utils.ct_utils import sets_overlap

{megadetector-10.0.4 → megadetector-10.0.6}/megadetector/postprocessing/repeat_detection_elimination/repeat_detections_core.py

@@ -637,7 +637,7 @@ def _find_matches_in_directory(dir_name_and_rows, options):
 
         if 'max_detection_conf' not in row or 'detections' not in row or \
             row['detections'] is None:
-            print('Skipping row {}'.format(i_directory_row))
+            # print('Skipping row {}'.format(i_directory_row))
             continue
 
         # Don't bother checking images with no detections above threshold

{megadetector-10.0.4 → megadetector-10.0.6}/megadetector/utils/ct_utils.py

@@ -850,6 +850,24 @@ def isnan(v):
         return False
 
 
+def compare_values_nan_equal(v0,v1):
+    """
+    Utility function for comparing two values when we want to return True if both
+    values are NaN.
+
+    Args:
+        v0 (object): the first value to compare
+        v1 (object): the second value to compare
+
+    Returns:
+        bool: True if v0 == v1, or if both v0 and v1 are NaN
+    """
+
+    if isinstance(v0,float) and isinstance(v1,float) and np.isnan(v0) and np.isnan(v1):
+        return True
+    return v0 == v1
+
+
 def sets_overlap(set1, set2):
     """
     Determines whether two sets overlap.