megadetector 10.0.9__py3-none-any.whl → 10.0.10__py3-none-any.whl

megadetector/data_management/read_exif.py

@@ -210,6 +210,8 @@ def read_pil_exif(im,options=None):
     if exif_info is None:
         return exif_tags
 
+    # Read all standard EXIF tags; if necessary, we'll filter later to a restricted
+    # list of tags.
     for k, v in exif_info.items():
         assert isinstance(k,str) or isinstance(k,int), \
             'Invalid EXIF key {}'.format(str(k))
@@ -221,6 +223,7 @@ def read_pil_exif(im,options=None):
 
     exif_ifd_tags = _get_exif_ifd(exif_info)
 
+    # Read tags that are only available via offset
     for k in exif_ifd_tags.keys():
         v = exif_ifd_tags[k]
         if k in exif_tags:
@@ -266,7 +269,7 @@ def read_pil_exif(im,options=None):
 
             # Convert to strings, e.g. 'GPSTimeStamp'
             gps_info = {}
-            for int_tag,v in enumerate(gps_info_raw.keys()):
+            for int_tag,v in gps_info_raw.items():
                 assert isinstance(int_tag,int)
                 if int_tag in ExifTags.GPSTAGS:
                     gps_info[ExifTags.GPSTAGS[int_tag]] = v
@@ -276,11 +279,15 @@ def read_pil_exif(im,options=None):
             exif_tags['GPSInfo'] = gps_info
 
         except Exception as e:
+
             if options.verbose:
                 print('Warning: error reading GPS info: {}'.format(str(e)))
 
     # ...if we think there might be GPS tags in this image
 
+    # Filter tags if necessary
+    exif_tags = _filter_tags(exif_tags,options)
+
     return exif_tags
 
 # ...read_pil_exif()
@@ -337,10 +344,16 @@ def _filter_tags(tags,options):
     if options.tags_to_include is None and options.tags_to_exclude is None:
         return tags
     if options.tags_to_include is not None:
+        if isinstance(options.tags_to_include,str):
+            if options.tags_to_include == 'all':
+                return tags
         assert options.tags_to_exclude is None, "tags_to_include and tags_to_exclude are incompatible"
+        tags_to_include = options.tags_to_include.split(',')
+        # Case-insensitive matching
+        tags_to_include = [s.lower() for s in tags_to_include]
         tags_to_return = {}
         for tag_name in tags.keys():
-            if tag_name in options.tags_to_include:
+            if str(tag_name).lower() in tags_to_include:
                 tags_to_return[tag_name] = tags[tag_name]
         return tags_to_return
     if options.tags_to_exclude is not None:

megadetector/detection/run_detector_batch.py

@@ -94,10 +94,10 @@ max_queue_size = 10
 # How often should we print progress when using the image queue?
 n_queue_print = 1000
 
-# Only used if --include_exif_data or --include_image_timestamp are supplied
-exif_options = read_exif.ReadExifOptions()
-exif_options.processing_library = 'pil'
-exif_options.byte_handling = 'convert_to_string'
+# Only used if --include_exif_tags or --include_image_timestamp are supplied
+exif_options_base = read_exif.ReadExifOptions()
+exif_options_base.processing_library = 'pil'
+exif_options_base.byte_handling = 'convert_to_string'
 
 # Only relevant when we're running our test harness; because bugs in batch
 # inference are dependent on batch grouping, we randomize batch grouping
@@ -208,7 +208,7 @@ def _consumer_func(q,
                    image_size=None,
                    include_image_size=False,
                    include_image_timestamp=False,
-                   include_exif_data=False,
+                   include_exif_tags=None,
                    augment=False,
                    detector_options=None,
                    preprocess_on_image_queue=default_preprocess_on_image_queue,
@@ -232,7 +232,7 @@ def _consumer_func(q,
         image_size (int, optional): image size to use for inference
         include_image_size (bool, optional): include image dimensions in output
         include_image_timestamp (bool, optional): include image timestamps in output
-        include_exif_data (bool, optional): include EXIF data in output
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): enable image augmentation
         detector_options (dict, optional): key/value pairs that are interpreted differently
             by different detectors
@@ -334,7 +334,7 @@ def _consumer_func(q,
                                                        image_size=image_size,
                                                        include_image_size=include_image_size,
                                                        include_image_timestamp=include_image_timestamp,
-                                                       include_exif_data=include_exif_data,
+                                                       include_exif_tags=include_exif_tags,
                                                        augment=augment)
                         results.extend(batch_results)
 
@@ -411,7 +411,7 @@ def _consumer_func(q,
                                                    image_size=image_size,
                                                    include_image_size=include_image_size,
                                                    include_image_timestamp=include_image_timestamp,
-                                                   include_exif_data=include_exif_data,
+                                                   include_exif_tags=include_exif_tags,
                                                    augment=augment)
                     results.extend(batch_results)
 
@@ -431,7 +431,7 @@ def _consumer_func(q,
                                         image_size=image_size,
                                         include_image_size=include_image_size,
                                         include_image_timestamp=include_image_timestamp,
-                                        include_exif_data=include_exif_data,
+                                        include_exif_tags=include_exif_tags,
                                         augment=augment)
                 results.append(result)
                 n_images_processed += 1
@@ -464,7 +464,7 @@ def _run_detector_with_image_queue(image_files,
                                    image_size=None,
                                    include_image_size=False,
                                    include_image_timestamp=False,
-                                   include_exif_data=False,
+                                   include_exif_tags=None,
                                    augment=False,
                                    detector_options=None,
                                    loader_workers=default_loaders,
@@ -487,7 +487,7 @@ def _run_detector_with_image_queue(image_files,
             doing
         include_image_size (bool, optional): should we include image size in the output for each image?
         include_image_timestamp (bool, optional): should we include image timestamps in the output for each image?
-        include_exif_data (bool, optional): should we include EXIF data in the output for each image?
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): enable image augmentation
         detector_options (dict, optional): key/value pairs that are interpreted differently
             by different detectors
@@ -562,7 +562,7 @@ def _run_detector_with_image_queue(image_files,
                                                           image_size,
                                                           include_image_size,
                                                           include_image_timestamp,
-                                                          include_exif_data,
+                                                          include_exif_tags,
                                                           augment,
                                                           detector_options,
                                                           preprocess_on_image_queue,
@@ -579,7 +579,7 @@ def _run_detector_with_image_queue(image_files,
                                                            image_size,
                                                            include_image_size,
                                                            include_image_timestamp,
-                                                           include_exif_data,
+                                                           include_exif_tags,
                                                            augment,
                                                            detector_options,
                                                            preprocess_on_image_queue,
@@ -598,7 +598,7 @@ def _run_detector_with_image_queue(image_files,
                        image_size,
                        include_image_size,
                        include_image_timestamp,
-                       include_exif_data,
+                       include_exif_tags,
                        augment,
                        detector_options,
                        preprocess_on_image_queue,
@@ -680,7 +680,7 @@ def _process_batch(image_items_batch,
                    image_size=None,
                    include_image_size=False,
                    include_image_timestamp=False,
-                   include_exif_data=False,
+                   include_exif_tags=None,
                    augment=False):
     """
     Process a batch of images using generate_detections_one_batch().  Does not necessarily return
@@ -695,7 +695,7 @@ def _process_batch(image_items_batch,
         image_size (int, optional): image size override
         include_image_size (bool, optional): include image dimensions in results
         include_image_timestamp (bool, optional): include image timestamps in results
-        include_exif_data (bool, optional): include EXIF data in results
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): whether to use image augmentation
 
     Returns:
@@ -762,12 +762,13 @@ def _process_batch(image_items_batch,
                     image_result['detections'] = \
                         [det for det in image_result['detections'] if det['conf'] >= confidence_threshold]
 
-                    if include_image_size or include_image_timestamp or include_exif_data:
+                    if include_image_size or include_image_timestamp or (include_exif_tags is not None):
 
                         image = valid_images[i_valid_image]
 
                         # If this was preprocessed by the producer thread, pull out the PIL version
                         if isinstance(image,dict):
+
                             image = image['img_original_pil']
 
                         if include_image_size:
@@ -779,9 +780,12 @@ def _process_batch(image_items_batch,
 
                             image_result['datetime'] = get_image_datetime(image)
 
-                        if include_exif_data:
+                        if include_exif_tags is not None:
 
-                            image_result['exif_metadata'] = read_exif.read_pil_exif(image,exif_options)
+                            exif_options = copy.copy(exif_options_base)
+                            exif_options.tags_to_include = include_exif_tags
+                            image_result['exif_metadata'] = read_exif.read_pil_exif(
+                                image,exif_options)
 
                     # ...if we need to store metadata
 
@@ -834,7 +838,7 @@ def _process_images(im_files,
                    checkpoint_queue=None,
                    include_image_size=False,
                    include_image_timestamp=False,
-                   include_exif_data=False,
+                   include_exif_tags=None,
                    augment=False,
                    detector_options=None,
                    loader_workers=default_loaders,
@@ -856,7 +860,7 @@ def _process_images(im_files,
         checkpoint_queue (Queue, optional): internal parameter used to pass image queues around
         include_image_size (bool, optional): should we include image size in the output for each image?
         include_image_timestamp (bool, optional): should we include image timestamps in the output for each image?
-        include_exif_data (bool, optional): should we include EXIF data in the output for each image?
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): enable image augmentation
         detector_options (dict, optional): key/value pairs that are interpreted differently
             by different detectors
@@ -890,7 +894,7 @@ def _process_images(im_files,
                                       image_size=image_size,
                                       include_image_size=include_image_size,
                                       include_image_timestamp=include_image_timestamp,
-                                      include_exif_data=include_exif_data,
+                                      include_exif_tags=include_exif_tags,
                                       augment=augment,
                                       detector_options=detector_options,
                                       loader_workers=loader_workers,
@@ -907,7 +911,7 @@ def _process_images(im_files,
                                    image_size=image_size,
                                    include_image_size=include_image_size,
                                    include_image_timestamp=include_image_timestamp,
-                                   include_exif_data=include_exif_data,
+                                   include_exif_tags=include_exif_tags,
                                    augment=augment)
 
             if checkpoint_queue is not None:
@@ -929,7 +933,7 @@ def _process_image(im_file,
                    image_size=None,
                    include_image_size=False,
                    include_image_timestamp=False,
-                   include_exif_data=False,
+                   include_exif_tags=False,
                    augment=False):
     """
     Runs a detector (typically MegaDetector) on a single image file.
@@ -947,7 +951,7 @@ def _process_image(im_file,
             doing
         include_image_size (bool, optional): should we include image size in the output for each image?
         include_image_timestamp (bool, optional): should we include image timestamps in the output for each image?
-        include_exif_data (bool, optional): should we include EXIF data in the output for each image?
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): enable image augmentation
 
     Returns:
@@ -1000,7 +1004,9 @@ def _process_image(im_file,
     if include_image_timestamp:
         result['datetime'] = get_image_datetime(image)
 
-    if include_exif_data:
+    if include_exif_tags is not None:
+        exif_options = copy.copy(exif_options_base)
+        exif_options.tags_to_include = include_exif_tags
         result['exif_metadata'] = read_exif.read_pil_exif(image,exif_options)
 
     return result
@@ -1055,7 +1061,7 @@ def load_and_run_detector_batch(model_file,
                                 class_mapping_filename=None,
                                 include_image_size=False,
                                 include_image_timestamp=False,
-                                include_exif_data=False,
+                                include_exif_tags=None,
                                 augment=False,
                                 force_model_download=False,
                                 detector_options=None,
@@ -1088,7 +1094,7 @@ def load_and_run_detector_batch(model_file,
             file or YOLOv5 dataset.yaml file
         include_image_size (bool, optional): should we include image size in the output for each image?
         include_image_timestamp (bool, optional): should we include image timestamps in the output for each image?
-        include_exif_data (bool, optional): should we include EXIF data in the output for each image?
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): enable image augmentation
         force_model_download (bool, optional): force downloading the model file if
             a named model (e.g. "MDV5A") is supplied, even if the local file already
@@ -1207,7 +1213,7 @@ def load_and_run_detector_batch(model_file,
                           image_size=image_size,
                           include_image_size=include_image_size,
                           include_image_timestamp=include_image_timestamp,
-                          include_exif_data=include_exif_data,
+                          include_exif_tags=include_exif_tags,
                           augment=augment,
                           detector_options=detector_options,
                           loader_workers=loader_workers,
@@ -1269,7 +1275,7 @@ def load_and_run_detector_batch(model_file,
                                                image_size=image_size,
                                                include_image_size=include_image_size,
                                                include_image_timestamp=include_image_timestamp,
-                                               include_exif_data=include_exif_data,
+                                               include_exif_tags=include_exif_tags,
                                                augment=augment)
 
                 results.extend(batch_results)
@@ -1295,7 +1301,7 @@ def load_and_run_detector_batch(model_file,
                                        image_size=image_size,
                                        include_image_size=include_image_size,
                                        include_image_timestamp=include_image_timestamp,
-                                       include_exif_data=include_exif_data,
+                                       include_exif_tags=include_exif_tags,
                                        augment=augment)
                 results.append(result)
 
@@ -1354,7 +1360,7 @@ def load_and_run_detector_batch(model_file,
                                  checkpoint_queue=checkpoint_queue,
                                  include_image_size=include_image_size,
                                  include_image_timestamp=include_image_timestamp,
-                                 include_exif_data=include_exif_data,
+                                 include_exif_tags=include_exif_tags,
                                  augment=augment,
                                  detector_options=detector_options),
                                  image_chunks)
@@ -1374,7 +1380,7 @@ def load_and_run_detector_batch(model_file,
                                                image_size=image_size,
                                                include_image_size=include_image_size,
                                                include_image_timestamp=include_image_timestamp,
-                                               include_exif_data=include_exif_data,
+                                               include_exif_tags=include_exif_tags,
                                                augment=augment,
                                                detector_options=detector_options),
                                                image_chunks)
@@ -1495,7 +1501,7 @@ def get_image_datetime(image):
         returns None if EXIF datetime is not available.
     """
 
-    exif_tags = read_exif.read_pil_exif(image,exif_options)
+    exif_tags = read_exif.read_pil_exif(image,exif_options_base)
 
     try:
         datetime_str = exif_tags['DateTimeOriginal']
@@ -1654,7 +1660,7 @@ if False:
     class_mapping_filename = None
     include_image_size = True
     include_image_timestamp = True
-    include_exif_data = True
+    include_exif_tags = None
     overwrite_handling = None
 
     # Generate a command line
@@ -1689,8 +1695,8 @@ if False:
         cmd += ' --include_image_size'
     if include_image_timestamp:
         cmd += ' --include_image_timestamp'
-    if include_exif_data:
-        cmd += ' --include_exif_data'
+    if include_exif_tags is not None:
+        cmd += ' --include_exif_tags "{}"'.format(include_exif_tags)
     if overwrite_handling is not None:
         cmd += ' --overwrite_handling {}'.format(overwrite_handling)
 
@@ -1837,9 +1843,10 @@ def main(): # noqa
         help='Include image datetime (if available) in output file'
     )
     parser.add_argument(
-        '--include_exif_data',
-        action='store_true',
-        help='Include available EXIF data in output file'
+        '--include_exif_tags',
+        type=str,
+        default=None,
+        help='Command-separated list of EXIF tags to include in output, or "all" to include all tags'
     )
     parser.add_argument(
         '--overwrite_handling',
@@ -1878,6 +1885,12 @@ def main(): # noqa
         action='store_true',
         help=argparse.SUPPRESS)
 
+    # This argument is deprecated in favor use --include_exif_tags
+    parser.add_argument(
+        '--include_exif_data',
+        action='store_true',
+        help=argparse.SUPPRESS)
+
     if len(sys.argv[1:]) == 0:
         parser.print_help()
         parser.exit()
@@ -1888,6 +1901,10 @@ def main(): # noqa
     if args.use_threads_for_queue:
         use_threads_for_queue = True
 
+    # Support the legacy --include_exif_data flag
+    if args.include_exif_data and (args.include_exif_tags is None):
+        args.include_exif_tags = 'all'
+
     detector_options = parse_kvp_list(args.detector_options)
 
     # If the specified detector file is really the name of a known model, find
@@ -2094,7 +2111,7 @@ def main(): # noqa
                                           class_mapping_filename=args.class_mapping_filename,
                                           include_image_size=args.include_image_size,
                                           include_image_timestamp=args.include_image_timestamp,
-                                          include_exif_data=args.include_exif_data,
+                                          include_exif_tags=args.include_exif_tags,
                                           augment=args.augment,
                                           # Don't download the model *again*
                                           force_model_download=False,

megadetector/detection/run_md_and_speciesnet.py

@@ -61,7 +61,18 @@ DEFAULT_DETECTION_CONFIDENCE_THRESHOLD_FOR_OUTPUT = DEFAULT_OUTPUT_CONFIDENCE_TH
 DEFAULT_DETECTOR_BATCH_SIZE = 1
 DEFAULT_CLASSIFIER_BATCH_SIZE = 8
 DEFAULT_LOADER_WORKERS = 4
-MAX_QUEUE_SIZE_IMAGES_PER_WORKER = 10
+
+# This determines the maximum number of images that can get read from disk
+# on each of the producer workers before blocking.  The actual size of the queue
+# will be MAX_IMAGE_QUEUE_SIZE_PER_WORKER * n_workers.  This is only used for
+# the classification step.
+MAX_IMAGE_QUEUE_SIZE_PER_WORKER = 10
+
+# This determines the maximum number of crops that can accumulate in the queue
+# used to communicate between the producers (which read and crop images) and the
+# consumer (which runs the classifier).  This is only used for the classification step.
+MAX_BATCH_QUEUE_SIZE = 300
+
 DEAFULT_SECONDS_PER_VIDEO_FRAME = 1.0
 
 # Max number of classification scores to include per detection
@@ -71,6 +82,11 @@ DEFAULT_TOP_N_SCORES = 2
 # cumulative confidence is above this value
 ROLLUP_TARGET_CONFIDENCE = 0.5
 
+# When the called supplies an existing MD results file, should we validate it before
+# starting classification?  This tends
+VALIDATE_DETECTION_FILE = False
+
+
 verbose = False
 
 
@@ -109,10 +125,10 @@ class CropBatch:
     """
 
     def __init__(self):
-        # List of preprocessed images
+        #: List of preprocessed images
         self.crops = []
 
-        # List of CropMetadata objects
+        #: List of CropMetadata objects
         self.metadata = []
 
     def add_crop(self, crop_data, metadata):
@@ -192,6 +208,7 @@ def _process_image_detections(file_path: str,
 
         # Preprocess the crop
         try:
+
             preprocessed_crop = classifier.preprocess(
                 image,
                 bboxes=[speciesnet_bbox],
@@ -199,6 +216,7 @@ def _process_image_detections(file_path: str,
             )
 
             if preprocessed_crop is not None:
+
                 metadata = CropMetadata(
                     image_file=file_path,
                     detection_index=detection_index,
@@ -207,10 +225,11 @@ def _process_image_detections(file_path: str,
                     original_height=original_height
                 )
 
-                # Send individual crop immediately to consumer
+                # Send individual crop to the consumer
                 batch_queue.put(('crop', preprocessed_crop, metadata))
 
         except Exception as e:
+
             print('Warning: failed to preprocess crop from {}, detection {}: {}'.format(
                 file_path, detection_index, str(e)))
 
@@ -226,6 +245,8 @@ def _process_image_detections(file_path: str,
                              'Failed to preprocess crop: {}'.format(str(e)),
                              failure_metadata))
 
+        # ...try/except
+
     # ...for each detection in this image
 
 # ...def _process_image_detections(...)
@@ -256,6 +277,7 @@ def _process_video_detections(file_path: str,
     frame_to_detections = {}
 
     for detection_index, detection in enumerate(detections):
+
         conf = detection['conf']
         if conf < detection_confidence_threshold:
             continue
@@ -267,6 +289,8 @@ def _process_video_detections(file_path: str,
             frame_to_detections[frame_number] = []
         frame_to_detections[frame_number].append((detection_index, detection))
 
+    # ...for each detection in this video
+
     if len(frames_with_detections) == 0:
         return
 
@@ -290,6 +314,7 @@ def _process_video_detections(file_path: str,
             return
         frame_number = int(match.group(1))
 
+        # Only process frames for which we have detection results
         if frame_number not in frame_to_detections:
             return
 
@@ -360,13 +385,16 @@ def _process_video_detections(file_path: str,
 
     # Process the video frames
     try:
+
         run_callback_on_frames(
             input_video_file=absolute_file_path,
             frame_callback=frame_callback,
             frames_to_process=frames_to_process,
             verbose=verbose
         )
+
     except Exception as e:
+
         print('Warning: failed to process video {}: {}'.format(file_path, str(e)))
 
         # Send failure information to consumer for the whole video
@@ -448,6 +476,7 @@ def _crop_producer_func(image_queue: JoinableQueue,
         is_video = is_video_file(file_path)
 
         if is_video:
+
             # Process video
             _process_video_detections(
                 file_path=file_path,
@@ -457,7 +486,9 @@ def _crop_producer_func(image_queue: JoinableQueue,
                 detection_confidence_threshold=detection_confidence_threshold,
                 batch_queue=batch_queue
             )
+
         else:
+
             # Process image
             _process_image_detections(
                 file_path=file_path,
@@ -571,9 +602,9 @@ def _crop_consumer_func(batch_queue: Queue,
         item_type, data, metadata = item
 
         if metadata.image_file not in all_results:
-                all_results[metadata.image_file] = {}
+            all_results[metadata.image_file] = {}
 
-        # We should never be processing the same detetion twice
+        # We should never be processing the same detection twice
         assert metadata.detection_index not in all_results[metadata.image_file]
 
         if item_type == 'failure':
@@ -601,6 +632,7 @@ def _crop_consumer_func(batch_queue: Queue,
 
     # ...while (we have items to process)
 
+    # Send all the results at once back to the main process
     results_queue.put(all_results)
 
     if verbose:
@@ -828,7 +860,7 @@ def _run_detection_step(source_folder: str,
             batch_size=detector_batch_size,
             include_image_size=False,
             include_image_timestamp=False,
-            include_exif_data=False,
+            include_exif_tags=None,
             loader_workers=detector_worker_threads,
             preprocess_on_image_queue=True
         )
@@ -914,9 +946,11 @@ def _run_classification_step(detector_results_file: str,
         top_n_scores (int, optional): maximum number of scores to include for each detection
     """
 
-    print('Starting SpeciesNet classification step...')
+    print('Starting classification step...')
 
     # Load MegaDetector results
+    print('Reading detection results from {}'.format(detector_results_file))
+
     with open(detector_results_file, 'r') as f:
         detector_results = json.load(f)
 
@@ -936,10 +970,22 @@ def _run_classification_step(detector_results_file: str,
         print('Set multiprocessing start method to spawn (was {})'.format(
             original_start_method))
 
-    # Set up multiprocessing queues
-    max_queue_size = classifier_worker_threads * MAX_QUEUE_SIZE_IMAGES_PER_WORKER
-    image_queue = JoinableQueue(max_queue_size)
-    batch_queue = Queue()
+    ## Set up multiprocessing queues
+
+    # This queue receives lists of image filenames (and associated detection results)
+    # from the "main" thread (the one you're reading right now).  Items are pulled off
+    # of this queue by producer workers (on _crop_producer_func), where the corresponding
+    # images are loaded from disk and preprocessed into crops.
+    image_queue = JoinableQueue(maxsize= \
+                                classifier_worker_threads * MAX_IMAGE_QUEUE_SIZE_PER_WORKER)
+
+    # This queue receives cropped images from producers (on _crop_producer_func); those
+    # crops are pulled off of this queue by the consumer (on _crop_consumer_func).
+    batch_queue = Queue(maxsize=MAX_BATCH_QUEUE_SIZE)
+
+    # This is not really used as a queue, rather it's just used to send all the results
+    # at once from the consumer process to the main process (the one you're reading right
+    # now).
     results_queue = Queue()
 
     # Start producer workers
@@ -951,7 +997,9 @@ def _run_classification_step(detector_results_file: str,
         p.start()
         producers.append(p)
 
-    # Start consumer worker
+
+    ## Start consumer worker
+
     consumer = Process(target=_crop_consumer_func,
                        args=(batch_queue, results_queue, classifier_model,
                              classifier_batch_size, classifier_worker_threads,
@@ -974,16 +1022,23 @@ def _run_classification_step(detector_results_file: str,
 
     print('Finished waiting for input queue')
 
-    # Wait for results
+
+    ## Wait for results
+
     classification_results = results_queue.get()
 
-    # Clean up processes
+
+    ## Clean up processes
+
     for p in producers:
         p.join()
     consumer.join()
 
     print('Finished waiting for workers')
 
+
+    ## Format results and write output
+
     class CategoryState:
         """
         Helper class to manage classification category IDs.
@@ -1257,15 +1312,18 @@ def main():
     print('Intermediate files: {}'.format(temp_folder))
 
     # Determine detector output file path
-    if args.detections_file:
+    if args.detections_file is not None:
         detector_output_file = args.detections_file
-        print('Using existing detections file: {}'.format(detector_output_file))
-        validation_options = ValidateBatchResultsOptions()
-        validation_options.check_image_existence = True
-        validation_options.relative_path_base = args.source
-        validation_options.raise_errors = True
-        validate_batch_results(detector_output_file,options=validation_options)
-        print('Validated detections file')
+        if VALIDATE_DETECTION_FILE:
+            print('Using existing detections file: {}'.format(detector_output_file))
+            validation_options = ValidateBatchResultsOptions()
+            validation_options.check_image_existence = True
+            validation_options.relative_path_base = args.source
+            validation_options.raise_errors = True
+            validate_batch_results(detector_output_file,options=validation_options)
+            print('Validated detections file')
+        else:
+            print('Bypassing validation of {}'.format(args.detections_file))
     else:
         detector_output_file = os.path.join(temp_folder, 'detector_output.json')
 

megadetector/detection/tf_detector.py

@@ -138,8 +138,8 @@ class TFDetector:
                                       image_id,
                                       detection_threshold,
                                       image_size=None,
-                                      skip_image_resizing=False,
-                                      augment=False):
+                                      augment=False,
+                                      verbose=False):
         """
         Runs the detector on an image.
 
@@ -152,10 +152,9 @@ class TFDetector:
             image_size (tuple, optional): image size to use for inference, only mess with this
                 if (a) you're using a model other than MegaDetector or (b) you know what you're
                 doing
-            skip_image_resizing (bool, optional): whether to skip internal image resizing (and rely on external
-                resizing).  Not currently supported, but included here for compatibility with PTDetector.
             augment (bool, optional): enable image augmentation.  Not currently  supported, but included
                 here for compatibility with PTDetector.
+            verbose (bool, optional): enable additional debug output
 
         Returns:
             dict: a dictionary with the following fields:
@@ -166,7 +165,6 @@ class TFDetector:
         """
 
         assert image_size is None, 'Image sizing not supported for TF detectors'
-        assert not skip_image_resizing, 'Image sizing not supported for TF detectors'
         assert not augment, 'Image augmentation is not supported for TF detectors'
 
         if detection_threshold is None:

megadetector/postprocessing/classification_postprocessing.py

@@ -1168,7 +1168,7 @@ def restrict_to_taxa_list(taxa_list,
     # Convert all NaN values in the "common" column to empty strings
     taxa_list_df['common'] = taxa_list_df['common'].fillna('')
 
-    # Create a dictionary mapping latin names to common names
+    # Create a dictionary mapping source Latin names to target common names
     target_latin_to_common = {}
 
     for i_row,row in taxa_list_df.iterrows():
@@ -1332,7 +1332,7 @@ def restrict_to_taxa_list(taxa_list,
         _insert_taxonomy_string(new_taxon_string)
 
 
-    ##%% Make sure all species on the allow-list are in the taxonomy
+    ##%% Make sure all taxa on the allow-list are in the taxonomy
 
     n_failed_mappings = 0
 
@@ -1498,7 +1498,8 @@ def restrict_to_taxa_list(taxa_list,
         if (protected_common_names is not None) and \
             (common_name in protected_common_names):
             if verbose:
-                print('Not messing with protected category {}'.format(common_name))
+                print('Not messing with protected category {}:\n{}'.format(
+                    common_name,input_taxon_string))
             input_category_id_to_output_taxon_string[input_category_id] = \
                 input_taxon_string
             continue
@@ -1578,12 +1579,13 @@ def restrict_to_taxa_list(taxa_list,
             output_taxon_string = speciesnet_latin_name_to_taxon_string[target_taxon]
         input_category_id_to_output_taxon_string[input_category_id] = output_taxon_string
 
-    # ...for each category
+    # ...for each category (mapping input category IDs to output taxon strings)
 
 
-    ##%% Build the new tables
+    ##%% Map input category IDs to output category IDs
 
-    speciesnet_taxon_string_to_latin_name = invert_dictionary(speciesnet_latin_name_to_taxon_string)
+    speciesnet_taxon_string_to_latin_name = \
+        invert_dictionary(speciesnet_latin_name_to_taxon_string)
 
     input_category_id_to_output_category_id = {}
     output_taxon_string_to_category_id = {}
@@ -1604,7 +1606,8 @@ def restrict_to_taxa_list(taxa_list,
             if speciesnet_latin_name in speciesnet_latin_name_to_output_common_name:
                 custom_common_name = speciesnet_latin_name_to_output_common_name[speciesnet_latin_name]
                 if custom_common_name != output_common_name:
-                    print('Substituting common name {} for {}'.format(custom_common_name,output_common_name))
+                    if verbose:
+                        print('Substituting common name {} for {}'.format(custom_common_name,output_common_name))
                     output_common_name = custom_common_name
 
         # Do we need to create a new output category?
@@ -1625,20 +1628,16 @@ def restrict_to_taxa_list(taxa_list,
         if False:
             original_common_name = \
               input_category_id_to_common_name[input_category_id]
-
             original_taxon_string = \
                 input_category_id_to_taxonomy_string[input_category_id]
-
             print('Mapping {} ({}) to:\n{} ({})\n'.format(
                 original_common_name,original_taxon_string,
                 output_common_name,output_taxon_string))
-            print('Mapping {} to {}'.format(
-                original_common_name,output_common_name,))
 
-    # ...for each category
+    # ...for each category (mapping input category IDs to output category IDs)
 
 
-    #%% Remap all category labels
+    ##%% Remap all category labels
 
     assert len(set(output_taxon_string_to_category_id.keys())) == \
            len(set(output_taxon_string_to_category_id.values())), \

megadetector/utils/directory_listing.py

@@ -129,6 +129,9 @@ def create_html_index(dir,
         recursive (bool, optional): recurse into subfolders
     """
 
+    if template_fun is None:
+        template_fun = _create_plain_index
+
     print('Traversing {}'.format(dir))
 
     # Make sure we remove the trailing /

megadetector/utils/path_utils.py

@@ -1046,6 +1046,73 @@ def parallel_copy_files(input_file_to_output_file,
 # ...def parallel_copy_files(...)
 
 
+#%% File deletion functions
+
+def delete_file(input_file, verbose=False):
+    """
+    Deletes a single file.
+
+    Args:
+        input_file (str): file to delete
+        verbose (bool, optional): enable additional debug console output
+
+    Returns:
+        bool: True if file was deleted successfully, False otherwise
+    """
+
+    try:
+        if verbose:
+            print('Deleting file {}'.format(input_file))
+
+        if os.path.isfile(input_file):
+            os.remove(input_file)
+            return True
+        else:
+            if verbose:
+                print('File {} does not exist'.format(input_file))
+            return False
+
+    except Exception as e:
+        if verbose:
+            print('Error deleting file {}: {}'.format(input_file, str(e)))
+        return False
+
+# ...def delete_file(...)
+
+
+def parallel_delete_files(input_files,
+                          max_workers=16,
+                          use_threads=True,
+                          verbose=False):
+    """
+    Deletes one or more files in parallel.
+
+    Args:
+        input_files (list): list of files to delete
+        max_workers (int, optional): number of concurrent workers, set to <= 1 to disable parallelism
+        use_threads (bool, optional): whether to use threads (True) or processes (False); ignored if
+            max_workers <= 1
+        verbose (bool, optional): enable additional debug console output
+    """
+
+    if len(input_files) == 0:
+        return
+
+    n_workers = min(max_workers, len(input_files))
+
+    if use_threads:
+        pool = ThreadPool(n_workers)
+    else:
+        pool = Pool(n_workers)
+
+    with tqdm(total=len(input_files)) as pbar:
+        for i, _ in enumerate(pool.imap_unordered(partial(delete_file, verbose=verbose),
+                                                  input_files)):
+            pbar.update()
+
+# ...def parallel_delete_files(...)
+
+
 #%% File size functions
 
 def get_file_sizes(base_dir, convert_slashes=True):

megadetector/utils/string_utils.py

@@ -34,6 +34,27 @@ def is_float(s):
     return True
 
 
+def is_int(s):
+    """
+    Checks whether [s] is an object (typically a string) that can be cast to a int
+
+    Args:
+        s (object): object to evaluate
+
+    Returns:
+        bool: True if s successfully casts to a int, otherwise False
+    """
+
+    if s is None:
+        return False
+
+    try:
+        _ = int(s)
+    except ValueError:
+        return False
+    return True
+
+
 def human_readable_to_bytes(size):
     """
     Given a human-readable byte string (e.g. 2G, 10GB, 30MB, 20KB),

megadetector/utils/wi_platform_utils.py

@@ -24,10 +24,14 @@ from multiprocessing.pool import Pool, ThreadPool
 from functools import partial
 
 from megadetector.utils.path_utils import insert_before_extension
+from megadetector.utils.path_utils import path_join
+
 from megadetector.utils.ct_utils import split_list_into_n_chunks
 from megadetector.utils.ct_utils import invert_dictionary
 from megadetector.utils.ct_utils import compare_values_nan_equal
 
+from megadetector.utils.string_utils import is_int
+
 from megadetector.utils.wi_taxonomy_utils import is_valid_prediction_string
 from megadetector.utils.wi_taxonomy_utils import no_cv_result_prediction_string
 from megadetector.utils.wi_taxonomy_utils import blank_prediction_string
@@ -68,7 +72,7 @@ def read_sequences_from_download_bundle(download_folder):
     assert len(sequence_list_files) == 1, \
         'Could not find sequences.csv in {}'.format(download_folder)
 
-    sequence_list_file = os.path.join(download_folder,sequence_list_files[0])
+    sequence_list_file = path_join(download_folder,sequence_list_files[0])
 
     df = pd.read_csv(sequence_list_file)
     sequence_records = df.to_dict('records')
@@ -104,7 +108,7 @@ def read_images_from_download_bundle(download_folder):
     image_list_files = \
         [fn for fn in image_list_files if fn.startswith('images_') and fn.endswith('.csv')]
     image_list_files = \
-        [os.path.join(download_folder,fn) for fn in image_list_files]
+        [path_join(download_folder,fn) for fn in image_list_files]
     print('Found {} image list files'.format(len(image_list_files)))
 
 
@@ -118,7 +122,7 @@ def read_images_from_download_bundle(download_folder):
         print('Reading images from list file {}'.format(
             os.path.basename(image_list_file)))
 
-        df = pd.read_csv(image_list_file)
+        df = pd.read_csv(image_list_file,low_memory=False)
 
         # i_row = 0; row = df.iloc[i_row]
         for i_row,row in tqdm(df.iterrows(),total=len(df)):
@@ -203,11 +207,91 @@ def find_images_in_identify_tab(download_folder_with_identify,download_folder_ex
 # ...def find_images_in_identify_tab(...)
 
 
-def write_download_commands(image_records_to_download,
+def write_prefix_download_command(image_records,
+                                  download_dir_base,
+                                  force_download=False,
+                                  download_command_file=None):
+    """
+    Write a .sh script to download all images (using gcloud) from the longest common URL
+    prefix in the images represented in [image_records].
+
+    Args:
+        image_records (list of dict): list of dicts with at least the field 'location'.
+            Can also be a dict whose values are lists of record dicts.
+        download_dir_base (str): local destination folder
+        force_download (bool, optional): overwrite existing files
+        download_command_file (str, optional): path of the .sh script we should write, defaults
+            to "download_wi_images_with_prefix.sh" in the destination folder.
+    """
+
+    ##%% Input validation
+
+    # If a dict is provided, assume it maps image GUIDs to lists of records, flatten to a list
+    if isinstance(image_records,dict):
+        all_image_records = []
+        for k in image_records:
+            records_this_image = image_records[k]
+            all_image_records.extend(records_this_image)
+        image_records = all_image_records
+
+    assert isinstance(image_records,list), \
+        'Illegal image record list format {}'.format(type(image_records))
+    assert isinstance(image_records[0],dict), \
+        'Illegal image record format {}'.format(type(image_records[0]))
+
+    urls = [r['location'] for r in image_records]
+
+    # "urls" is a list of URLs starting with gs://.  Find the highest-level folder
+    # that is common to all URLs in the list.  For example, if the list is:
+    #
+    # gs://a/b/c
+    # gs://a/b/d
+    #
+    # The result should be:
+    #
+    # gs://a/b
+    common_prefix = os.path.commonprefix(urls)
+
+    # Remove the gs:// prefix if it's still there
+    if common_prefix.startswith('gs://'):
+        common_prefix = common_prefix[len('gs://'):]
+
+    # Ensure the common prefix ends with a '/' if it's not empty
+    if (len(common_prefix) > 0) and (not common_prefix.endswith('/')):
+        common_prefix = os.path.dirname(common_prefix) + '/'
+
+    print('Longest common prefix: {}'.format(common_prefix))
+
+    if download_command_file is None:
+        download_command_file = \
+            path_join(download_dir_base,'download_wi_images_with_prefix.sh')
+
+    os.makedirs(download_dir_base,exist_ok=True)
+
+    with open(download_command_file,'w',newline='\n') as f:
+        # The --no-clobber flag prevents overwriting existing files
+        # The -r flag is for recursive download
+        # The gs:// prefix is added back for the gcloud command
+        no_clobber_string = ''
+        if not force_download:
+            no_clobber_string = '--no-clobber'
+
+        cmd = 'gcloud storage cp -r {} "gs://{}" "{}"'.format(
+            no_clobber_string,common_prefix,download_dir_base)
+        print('Writing download command:\n{}'.format(cmd))
+        f.write(cmd + '\n')
+
+    print('Download script written to {}'.format(download_command_file))
+
+# ...def write_prefix_download_command(...)
+
+
+def write_download_commands(image_records,
                             download_dir_base,
                             force_download=False,
                             n_download_workers=25,
-                            download_command_file_base=None):
+                            download_command_file_base=None,
+                            image_flattening='deployment'):
     """
     Given a list of dicts with at least the field 'location' (a gs:// URL), prepare a set of "gcloud
     storage" commands to download images, and write those to a series of .sh scripts, along with one
@@ -215,10 +299,9 @@ def write_download_commands(image_records_to_download,
 
     gcloud commands will use relative paths.
 
-    image_records_to_download can also be a dict mapping IDs to lists of records.
-
     Args:
-        image_records_to_download (list of dict): list of dicts with at least the field 'location'
+        image_records (list of dict): list of dicts with at least the field 'location'.
+            Can also be a dict whose values are lists of record dicts.
         download_dir_base (str): local destination folder
         force_download (bool, optional): include gs commands even if the target file exists
         n_download_workers (int, optional): number of scripts to write (that's our hacky way
@@ -226,42 +309,103 @@ def write_download_commands(image_records_to_download,
         download_command_file_base (str, optional): path of the .sh script we should write, defaults
             to "download_wi_images.sh" in the destination folder.  Individual worker scripts will
             have a number added, e.g. download_wi_images_00.sh.
+        image_flattening (str, optional): if 'none', relative paths will be preserved
+            representing the entire URL for each image.  Can be 'guid' (just download to
+            [GUID].JPG) or 'deployment' (download to [deployment]/[GUID].JPG).
     """
 
-    if isinstance(image_records_to_download,dict):
+    ##%% Input validation
 
+    # If a dict is provided, assume it maps image GUIDs to lists of records, flatten to a list
+    if isinstance(image_records,dict):
         all_image_records = []
-        for k in image_records_to_download:
-            records_this_image = image_records_to_download[k]
+        for k in image_records:
+            records_this_image = image_records[k]
             all_image_records.extend(records_this_image)
-        return write_download_commands(all_image_records,
-                                       download_dir_base=download_dir_base,
-                                       force_download=force_download,
-                                       n_download_workers=n_download_workers,
-                                       download_command_file_base=download_command_file_base)
+        image_records = all_image_records
+
+    assert isinstance(image_records,list), \
+        'Illegal image record list format {}'.format(type(image_records))
+    assert isinstance(image_records[0],dict), \
+        'Illegal image record format {}'.format(type(image_records[0]))
+
+
+    ##%% Map URLs to relative paths
+
+    # URLs look like:
+    #
+    # gs://145625555_2004881_2323_name__main/deployment/2241000/prod/directUpload/5fda0ddd-511e-46ca-95c1-302b3c71f8ea.JPG
+    if image_flattening is None:
+        image_flattening = 'none'
+    image_flattening = image_flattening.lower().strip()
+
+    assert image_flattening in ('none','guid','deployment'), \
+        'Illegal image flattening strategy {}'.format(image_flattening)
+
+    url_to_relative_path = {}
+
+    for image_record in image_records:
+
+        url = image_record['location']
+        assert url.startswith('gs://'), 'Illegal URL {}'.format(url)
+
+        relative_path = None
+
+        if image_flattening == 'none':
+            relative_path = url.replace('gs://','')
+        elif image_flattening == 'guid':
+            relative_path = url.split('/')[-1]
+        else:
+            assert image_flattening == 'deployment'
+            tokens = url.split('/')
+            found_deployment_id = False
+            for i_token,token in enumerate(tokens):
+                if token == 'deployment':
+                    assert i_token < (len(tokens)-1)
+                    deployment_id_string = tokens[i_token + 1]
+                    deployment_id_string = deployment_id_string.replace('_thumb','')
+                    assert is_int(deployment_id_string), \
+                        'Illegal deployment ID {}'.format(deployment_id_string)
+                    image_id = url.split('/')[-1]
+                    relative_path = deployment_id_string + '/' + image_id
+                    found_deployment_id = True
+                    break
+            assert found_deployment_id, \
+                'Could not find deployment ID in record {}'.format(str(image_record))
+
+        assert relative_path is not None
+
+        if url in url_to_relative_path:
+            assert url_to_relative_path[url] == relative_path, \
+                'URL path mapping error'
+        else:
+            url_to_relative_path[url] = relative_path
+
+    # ...for each image record
+
 
     ##%% Make list of gcloud storage commands
 
     if download_command_file_base is None:
-        download_command_file_base = os.path.join(download_dir_base,'download_wi_images.sh')
+        download_command_file_base = path_join(download_dir_base,'download_wi_images.sh')
 
     commands = []
     skipped_urls = []
     downloaded_urls = set()
 
-    # image_record = image_records_to_download[0]
-    for image_record in tqdm(image_records_to_download):
+    # image_record = image_records[0]
+    for image_record in tqdm(image_records):
 
         url = image_record['location']
         if url in downloaded_urls:
             continue
 
-        assert url.startswith('gs://')
+        assert url.startswith('gs://'), 'Illegal URL {}'.format(url)
 
-        relative_path = url.replace('gs://','')
-        abs_path = os.path.join(download_dir_base,relative_path)
+        relative_path = url_to_relative_path[url]
+        abs_path = path_join(download_dir_base,relative_path)
 
-        # Skip files that already exist
+        # Optionally skip files that already exist
         if (not force_download) and (os.path.isfile(abs_path)):
             skipped_urls.append(url)
             continue
@@ -271,7 +415,7 @@ def write_download_commands(image_records_to_download,
         commands.append(command)
 
     print('Generated {} commands for {} image records'.format(
-        len(commands),len(image_records_to_download)))
+        len(commands),len(image_records)))
 
     print('Skipped {} URLs'.format(len(skipped_urls)))
 

megadetector/utils/wi_taxonomy_utils.py

@@ -754,6 +754,7 @@ def generate_instances_json_from_folder(folder,
 
     assert os.path.isdir(folder)
 
+    print('Enumerating images in {}'.format(folder))
     image_files_abs = find_images(folder,recursive=True,return_relative_paths=False)
 
     if tokens_to_ignore is not None:

megadetector/visualization/visualize_detector_output.py

@@ -428,6 +428,7 @@ def main(): # noqa
         category_names_to_blur=category_names_to_blur)
 
     if (args.html_output_file is not None) and args.open_html_output_file:
+        print('Opening output file {}'.format(args.html_output_file))
         open_file(args.html_output_file)
 
 if __name__ == '__main__':

{megadetector-10.0.9.dist-info → megadetector-10.0.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: megadetector
-Version: 10.0.9
+Version: 10.0.10
 Summary: MegaDetector is an AI model that helps conservation folks spend less time doing boring things with camera trap images.
 Author-email: Your friendly neighborhood MegaDetector team <cameratraps@lila.science>
 Maintainer-email: Your friendly neighborhood MegaDetector team <cameratraps@lila.science>

{megadetector-10.0.9.dist-info → megadetector-10.0.10.dist-info}/RECORD

@@ -43,7 +43,7 @@ megadetector/data_management/labelme_to_coco.py,sha256=SO6DMfJ9WNlMUHF9EUYWjSNye
 megadetector/data_management/labelme_to_yolo.py,sha256=bsqpNUsnDJucJ60wSQD_yvq_tWiots1u4tSFNiHeaYA,12769
 megadetector/data_management/mewc_to_md.py,sha256=09XHEykIG-whGkgEIkho7xfVuPlic1TYTKGAufv_tto,12637
 megadetector/data_management/ocr_tools.py,sha256=aYpULCPn_tHaqatOd8qjEpKJ7MksRZS0o1kqQF04IE0,31389
-megadetector/data_management/read_exif.py,sha256=M_8492al57kWgZ-0gNWLNdzpm442zPCC8J2DtgzHAyA,29646
+megadetector/data_management/read_exif.py,sha256=80L_P2mGRKGA02X7jaGje27pwxrZ7mfIPo5IHKpYihc,30166
 megadetector/data_management/remap_coco_categories.py,sha256=DT4Rdt7Y1IdhbO2TZiBhQDESdit-l_-b_Hw0tbJ2Nuw,7090
 megadetector/data_management/remove_exif.py,sha256=5JHGWMIeXqB2PE2ZwIMJOEtNYopxknNDwynQAuJCLvw,4031
 megadetector/data_management/rename_images.py,sha256=iHkdQ_c1G9Oc8C4wcnPLmhKv0S9i9g7ppbytfBBqn2Y,6516
@@ -75,16 +75,16 @@ megadetector/detection/change_detection.py,sha256=Ne3GajbH_0KPBU8ruHp4Rkr0uKd5oK
 megadetector/detection/process_video.py,sha256=kuQHrpOC3LQo9ecqJPpzkds9fZVnoLmrfJw_yh-oxi8,17890
 megadetector/detection/pytorch_detector.py,sha256=-TvtDcX2Hh_CgBEz7Eg2NzyEts8DjOgY0mE_fle6zkM,60705
 megadetector/detection/run_detector.py,sha256=JWTIYsk5aCgW9PBCGnAECe31JwKHhkfp6zKsSDqfrsA,46831
-megadetector/detection/run_detector_batch.py,sha256=Ah-LSsA73Io-GH0BpI8qopGRI5eUKWhhIR19lC6s41A,90602
+megadetector/detection/run_detector_batch.py,sha256=gc3T_h61tRk705Lxbi4BXboc-BmaitiNd-5cR5MzHC0,91423
 megadetector/detection/run_inference_with_yolov5_val.py,sha256=dJXh3BwKOQQ4OA-Mq_heEb7AfBAk7qKUAagnIGuFtaU,53689
-megadetector/detection/run_md_and_speciesnet.py,sha256=Dp_SpJZp0pX9jzFtxM6zPCyBNq49uyQpMDAdNDLVorM,50280
+megadetector/detection/run_md_and_speciesnet.py,sha256=LMpQVqpLqju2WFznHGKEyc9QCgU4pHSPyfDnf0HL9GA,52179
 megadetector/detection/run_tiled_inference.py,sha256=v_wL4uZfYdswJdYXBoGci62UvVprBD9OHKkrw2g-G5M,41081
-megadetector/detection/tf_detector.py,sha256=3b2MiqgMw8KBDzHQliUSDXWrmKpa9iZnfe6EgYpMcYo,8398
+megadetector/detection/tf_detector.py,sha256=E0PZ1jHAv31kvsdaUD8IjuPmdLulkKwx6s2wbPMK-WQ,8151
 megadetector/detection/video_utils.py,sha256=M7yje6XeOnR_QwDyuG1o6bwTKvRysoA2NiOK2MSi98E,53943
 megadetector/postprocessing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megadetector/postprocessing/add_max_conf.py,sha256=9MYtsH2mwkiaZb7Qcor5J_HskfAj7d9srp8G_Qldpk0,1722
 megadetector/postprocessing/categorize_detections_by_size.py,sha256=DpZpRNFlyeOfWuOc6ICuENgIWDCEtiErJ_frBZp9lYM,5382
-megadetector/postprocessing/classification_postprocessing.py,sha256=OoPVr34vXyLykB42SplcSKo9cj7dgf8Yju_DCDhd6_k,68574
+megadetector/postprocessing/classification_postprocessing.py,sha256=WtZqgY43-KdbVmuG_PPh_5HK3J7-Q-laKQtQmbhdvI0,68701
 megadetector/postprocessing/combine_batch_outputs.py,sha256=BEP8cVa0sMIPg7tkWQc_8vOEPnbmWjOsQdVJHe61uz8,8468
 megadetector/postprocessing/compare_batch_results.py,sha256=QbdegGZkgVLZdO5Vjm3aTAQS5VzP9_tX0PKwCSkHKhw,85009
 megadetector/postprocessing/convert_output_format.py,sha256=3KLO6NqddofgIEYjV8_iZIf0iXaplFN2AroUq5i4R7k,14472
@@ -121,27 +121,27 @@ megadetector/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 megadetector/tests/test_nms_synthetic.py,sha256=oY6xmT1sLSSN7weQJ8TPTaZgAiSiZ6s43EffUhwLWIw,14707
 megadetector/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megadetector/utils/ct_utils.py,sha256=IiZV8dWtJamveINv_joATMgMPHeDkZ8l82jDEQcLgQg,60502
-megadetector/utils/directory_listing.py,sha256=0-VMuQWo6rETIKERqfX6Zn7pRp_GJ4JiFiWvsw9PQcU,6500
+megadetector/utils/directory_listing.py,sha256=ZTwryqP_02XSHlsaVNCo7qyLGM4EKB-2NYxcmQpQEwM,6573
 megadetector/utils/extract_frames_from_video.py,sha256=vjSVgxtb5z2syHCVYWc2KdNUpc-O6yY8nkbj_wqsIvY,12255
 megadetector/utils/gpu_test.py,sha256=5zUfAVeSjH8I08eCqayFmMxL-0mix8SjJJTe5ORABvU,3544
 megadetector/utils/md_tests.py,sha256=Iup4KjyIpLUpZ4TzzwEyGK61rg6aH7NrEQsdQ-ov51I,80300
-megadetector/utils/path_utils.py,sha256=tV8eh77m_uS8YYpOQZO8GUKR6l5sZrSSIkApqgi_DmY,101030
+megadetector/utils/path_utils.py,sha256=RDz3-Cts6NG118ll2WXzSThNZe5rJUxWZdxeIbjUMcg,102900
 megadetector/utils/process_utils.py,sha256=gQcpH9WYvGPUs0FhtJ5_Xvl6JsvoGz8_mnDQk0PbTRM,5673
 megadetector/utils/split_locations_into_train_val.py,sha256=fd_6pj1aWY6hybwaXvBn9kBcOHjI90U-OsTmEAGpeu8,10297
-megadetector/utils/string_utils.py,sha256=r2Maw3zbzk3EyaZcNkdqr96yP_8m4ey6v0WxlemEY9U,6155
+megadetector/utils/string_utils.py,sha256=OejBfVWdmc-uHaCTfQN5PChsd1tMuiRJVRHQV0xZWt8,6533
 megadetector/utils/url_utils.py,sha256=PzqN-VquAZFBRin2ZaYi5U2WCsMYSwvM0X-NN45Fdh4,28448
-megadetector/utils/wi_platform_utils.py,sha256=8CGpiox_aL6RVZKfJqPVwpW4_6Cjku0HIajJPcmeNpE,32019
-megadetector/utils/wi_taxonomy_utils.py,sha256=o4AvY5gZXfk69pPckdGxgIPhqsH2-hJQucavSRsUnoc,66513
+megadetector/utils/wi_platform_utils.py,sha256=TSckCBi6yIe6VAJhZtbf7aiZ_U0ipBzr_RU9OEqneaU,37688
+megadetector/utils/wi_taxonomy_utils.py,sha256=yd8C38DJWUfhSTaO5Zr_YoDqqR02OA-ZbrSHCbU1utI,66566
 megadetector/utils/write_html_image_list.py,sha256=6Tbe5wyUxoBYJgH9yVrxxKCeWF2BVre_wQMEOQJ-ZIU,9068
 megadetector/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megadetector/visualization/plot_utils.py,sha256=uDDlOhdaJ3V8sGj2kS9b0cgszKc8WCq2_ofl6TW_XUs,10727
 megadetector/visualization/render_images_with_thumbnails.py,sha256=-XX4PG4wnrFjFTIwd0sMxXxKMxPuu0SZ_TfK3dI1x8Y,8425
 megadetector/visualization/visualization_utils.py,sha256=E5uvysS3F1S_yiPFxZty3U2f6cjuE8zG6XWggYOu-5o,75921
 megadetector/visualization/visualize_db.py,sha256=8YDWSR0eMehXYdPtak9z8UUw35xV7hu-0eCuzgSLjWc,25558
-megadetector/visualization/visualize_detector_output.py,sha256=HpWh7ugwo51YBHsFi40iAp9G-uRAMMjgsm8H_uBolBs,20295
+megadetector/visualization/visualize_detector_output.py,sha256=nfB4JtfNU5PgFzuWxXSUSfCib29DWSNPhIf9drtD9Qs,20365
 megadetector/visualization/visualize_video_output.py,sha256=ibMGB5ynMwNXmaMlY8h8tURb-Lyvuxs1EB08x_jvev0,20606
-megadetector-10.0.9.dist-info/licenses/LICENSE,sha256=RMa3qq-7Cyk7DdtqRj_bP1oInGFgjyHn9-PZ3PcrqIs,1100
-megadetector-10.0.9.dist-info/METADATA,sha256=s8q_fi96c5kt67xihApa5_Lko4voT5lH0swUawbUgzE,6486
-megadetector-10.0.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-megadetector-10.0.9.dist-info/top_level.txt,sha256=wf9DXa8EwiOSZ4G5IPjakSxBPxTDjhYYnqWRfR-zS4M,13
-megadetector-10.0.9.dist-info/RECORD,,
+megadetector-10.0.10.dist-info/licenses/LICENSE,sha256=RMa3qq-7Cyk7DdtqRj_bP1oInGFgjyHn9-PZ3PcrqIs,1100
+megadetector-10.0.10.dist-info/METADATA,sha256=-4OEXDgVH3BLpXqLN2a3BKeVBI4wDbOQTnfzRs2dggw,6487
+megadetector-10.0.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+megadetector-10.0.10.dist-info/top_level.txt,sha256=wf9DXa8EwiOSZ4G5IPjakSxBPxTDjhYYnqWRfR-zS4M,13
+megadetector-10.0.10.dist-info/RECORD,,