megadetector 10.0.9__py3-none-any.whl → 10.0.11__py3-none-any.whl

megadetector/detection/run_detector.py

@@ -13,8 +13,7 @@ check out run_detector_batch.py**.
 That said, this script (run_detector.py) is a good way to test our detector on a handful of images
 and get super-satisfying, graphical results.
 
-If you would like to *not* use the GPU on the machine, set the environment
-variable CUDA_VISIBLE_DEVICES to "-1".
+If you would like to *not* use the GPU, set the environment variable CUDA_VISIBLE_DEVICES to "-1".
 
 This script will only consider detections with > 0.005 confidence at all times.
 The threshold you provide is only for rendering the results. If you need to
@@ -79,6 +78,10 @@ DEFAULT_DETECTOR_LABEL_MAP = {
 # models other than MegaDetector.
 USE_MODEL_NATIVE_CLASSES = False
 
+# Detection threshold to recommend to callers when all other mechanisms for choosing
+# a model-specific threshold fail
+fallback_detection_threshold = 0.2
+
 # Maps a variety of strings that might occur in filenames to canonical version numbers.
 #
 # Order matters here.
@@ -215,7 +218,8 @@ known_models = {
     {
         'url':model_url_base + 'md_v1000.0.0-redwood.pt',
         'normalized_typical_inference_speed':1.0,
-        'md5':'74474b3aec9cf1a990da38b37ddf9197'
+        'md5':'74474b3aec9cf1a990da38b37ddf9197',
+        'typical_detection_threshold':0.3
     },
     'v1000.0.0-spruce':
     {
@@ -515,19 +519,34 @@ def get_typical_confidence_threshold_from_results(results):
         with open(results,'r') as f:
             results = json.load(f)
 
+    default_threshold = None
+
+    # Best case: the .json file tells us the default threshold
     if 'detector_metadata' in results['info'] and \
         'typical_detection_threshold' in results['info']['detector_metadata']:
         default_threshold = results['info']['detector_metadata']['typical_detection_threshold']
+    # Worst case: we don't even know what detector this is
     elif ('detector' not in results['info']) or (results['info']['detector'] is None):
         print('Warning: detector version not available in results file, using MDv5 defaults')
         detector_metadata = get_detector_metadata_from_version_string('v5a.0.0')
         default_threshold = detector_metadata['typical_detection_threshold']
+    # We know what detector this is, but it doesn't have a default threshold
+    # in the .json file
     else:
         print('Warning: detector metadata not available in results file, inferring from MD version')
-        detector_filename = results['info']['detector']
-        detector_version = get_detector_version_from_filename(detector_filename)
-        detector_metadata = get_detector_metadata_from_version_string(detector_version)
-        default_threshold = detector_metadata['typical_detection_threshold']
+        try:
+            detector_filename = results['info']['detector']
+            detector_version = get_detector_version_from_filename(detector_filename)
+            detector_metadata = get_detector_metadata_from_version_string(detector_version)
+            if 'typical_detection_threshold' in detector_metadata:
+                default_threshold = detector_metadata['typical_detection_threshold']
+        except Exception:
+            pass
+
+    if default_threshold is None:
+        print('Could not determine threshold, using fallback threshold of {}'.format(
+            fallback_detection_threshold))
+        default_threshold = fallback_detection_threshold
 
     return default_threshold
 
@@ -546,7 +565,7 @@ def is_gpu_available(model_file):
     """
 
     if model_file.endswith('.pb'):
-        import tensorflow.compat.v1 as tf
+        import tensorflow.compat.v1 as tf # type: ignore
         gpu_available = tf.test.is_gpu_available()
         print('TensorFlow version:', tf.__version__)
         print('tf.test.is_gpu_available:', gpu_available)
@@ -741,8 +760,9 @@ def load_and_run_detector(model_file,
         fn = '{}{}{}'.format(name, DETECTION_FILENAME_INSERT, '.jpg')
         if fn in output_filename_collision_counts:
             n_collisions = output_filename_collision_counts[fn]
+            fn_original = fn
             fn = '{:0>4d}'.format(n_collisions) + '_' + fn
-            output_filename_collision_counts[fn] += 1
+            output_filename_collision_counts[fn_original] += 1
         else:
             output_filename_collision_counts[fn] = 0
         fn = os.path.join(output_dir, fn)
@@ -847,7 +867,11 @@ def _validate_zip_file(file_path, file_description='file'):
     """
     try:
         with zipfile.ZipFile(file_path, 'r') as zipf:
-            zipf.testzip()
+            corrupt_file = zipf.testzip()
+            if corrupt_file is not None:
+                print('{} {} contains at least one corrupt file: {}'.format(
+                    file_description.capitalize(), file_path, corrupt_file))
+                return False
         return True
     except (zipfile.BadZipFile, zipfile.LargeZipFile) as e:
         print('{} {} appears to be corrupted (bad zip): {}'.format(

megadetector/detection/run_detector_batch.py

@@ -94,10 +94,10 @@ max_queue_size = 10
 # How often should we print progress when using the image queue?
 n_queue_print = 1000
 
-# Only used if --include_exif_data or --include_image_timestamp are supplied
-exif_options = read_exif.ReadExifOptions()
-exif_options.processing_library = 'pil'
-exif_options.byte_handling = 'convert_to_string'
+# Only used if --include_exif_tags or --include_image_timestamp are supplied
+exif_options_base = read_exif.ReadExifOptions()
+exif_options_base.processing_library = 'pil'
+exif_options_base.byte_handling = 'convert_to_string'
 
 # Only relevant when we're running our test harness; because bugs in batch
 # inference are dependent on batch grouping, we randomize batch grouping
@@ -208,7 +208,7 @@ def _consumer_func(q,
                    image_size=None,
                    include_image_size=False,
                    include_image_timestamp=False,
-                   include_exif_data=False,
+                   include_exif_tags=None,
                    augment=False,
                    detector_options=None,
                    preprocess_on_image_queue=default_preprocess_on_image_queue,
@@ -232,7 +232,7 @@ def _consumer_func(q,
         image_size (int, optional): image size to use for inference
         include_image_size (bool, optional): include image dimensions in output
         include_image_timestamp (bool, optional): include image timestamps in output
-        include_exif_data (bool, optional): include EXIF data in output
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): enable image augmentation
         detector_options (dict, optional): key/value pairs that are interpreted differently
             by different detectors
@@ -334,7 +334,7 @@ def _consumer_func(q,
                                                        image_size=image_size,
                                                        include_image_size=include_image_size,
                                                        include_image_timestamp=include_image_timestamp,
-                                                       include_exif_data=include_exif_data,
+                                                       include_exif_tags=include_exif_tags,
                                                        augment=augment)
                         results.extend(batch_results)
 
@@ -411,7 +411,7 @@ def _consumer_func(q,
                                                    image_size=image_size,
                                                    include_image_size=include_image_size,
                                                    include_image_timestamp=include_image_timestamp,
-                                                   include_exif_data=include_exif_data,
+                                                   include_exif_tags=include_exif_tags,
                                                    augment=augment)
                     results.extend(batch_results)
 
@@ -431,7 +431,7 @@ def _consumer_func(q,
                                         image_size=image_size,
                                         include_image_size=include_image_size,
                                         include_image_timestamp=include_image_timestamp,
-                                        include_exif_data=include_exif_data,
+                                        include_exif_tags=include_exif_tags,
                                         augment=augment)
                 results.append(result)
                 n_images_processed += 1
@@ -464,7 +464,7 @@ def _run_detector_with_image_queue(image_files,
                                    image_size=None,
                                    include_image_size=False,
                                    include_image_timestamp=False,
-                                   include_exif_data=False,
+                                   include_exif_tags=None,
                                    augment=False,
                                    detector_options=None,
                                    loader_workers=default_loaders,
@@ -487,7 +487,7 @@ def _run_detector_with_image_queue(image_files,
             doing
         include_image_size (bool, optional): should we include image size in the output for each image?
         include_image_timestamp (bool, optional): should we include image timestamps in the output for each image?
-        include_exif_data (bool, optional): should we include EXIF data in the output for each image?
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): enable image augmentation
         detector_options (dict, optional): key/value pairs that are interpreted differently
             by different detectors
@@ -562,7 +562,7 @@ def _run_detector_with_image_queue(image_files,
                                                           image_size,
                                                           include_image_size,
                                                           include_image_timestamp,
-                                                          include_exif_data,
+                                                          include_exif_tags,
                                                           augment,
                                                           detector_options,
                                                           preprocess_on_image_queue,
@@ -579,7 +579,7 @@ def _run_detector_with_image_queue(image_files,
                                                            image_size,
                                                            include_image_size,
                                                            include_image_timestamp,
-                                                           include_exif_data,
+                                                           include_exif_tags,
                                                            augment,
                                                            detector_options,
                                                            preprocess_on_image_queue,
@@ -598,7 +598,7 @@ def _run_detector_with_image_queue(image_files,
                        image_size,
                        include_image_size,
                        include_image_timestamp,
-                       include_exif_data,
+                       include_exif_tags,
                        augment,
                        detector_options,
                        preprocess_on_image_queue,
@@ -680,7 +680,7 @@ def _process_batch(image_items_batch,
                    image_size=None,
                    include_image_size=False,
                    include_image_timestamp=False,
-                   include_exif_data=False,
+                   include_exif_tags=None,
                    augment=False):
     """
     Process a batch of images using generate_detections_one_batch().  Does not necessarily return
@@ -695,7 +695,7 @@ def _process_batch(image_items_batch,
         image_size (int, optional): image size override
         include_image_size (bool, optional): include image dimensions in results
         include_image_timestamp (bool, optional): include image timestamps in results
-        include_exif_data (bool, optional): include EXIF data in results
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): whether to use image augmentation
 
     Returns:
@@ -762,12 +762,13 @@ def _process_batch(image_items_batch,
                     image_result['detections'] = \
                         [det for det in image_result['detections'] if det['conf'] >= confidence_threshold]
 
-                    if include_image_size or include_image_timestamp or include_exif_data:
+                    if include_image_size or include_image_timestamp or (include_exif_tags is not None):
 
                         image = valid_images[i_valid_image]
 
                         # If this was preprocessed by the producer thread, pull out the PIL version
                         if isinstance(image,dict):
+
                             image = image['img_original_pil']
 
                         if include_image_size:
@@ -779,9 +780,12 @@ def _process_batch(image_items_batch,
 
                             image_result['datetime'] = get_image_datetime(image)
 
-                        if include_exif_data:
+                        if include_exif_tags is not None:
 
-                            image_result['exif_metadata'] = read_exif.read_pil_exif(image,exif_options)
+                            exif_options = copy.copy(exif_options_base)
+                            exif_options.tags_to_include = include_exif_tags
+                            image_result['exif_metadata'] = read_exif.read_pil_exif(
+                                image,exif_options)
 
                     # ...if we need to store metadata
 
@@ -834,7 +838,7 @@ def _process_images(im_files,
                    checkpoint_queue=None,
                    include_image_size=False,
                    include_image_timestamp=False,
-                   include_exif_data=False,
+                   include_exif_tags=None,
                    augment=False,
                    detector_options=None,
                    loader_workers=default_loaders,
@@ -856,7 +860,7 @@ def _process_images(im_files,
         checkpoint_queue (Queue, optional): internal parameter used to pass image queues around
         include_image_size (bool, optional): should we include image size in the output for each image?
         include_image_timestamp (bool, optional): should we include image timestamps in the output for each image?
-        include_exif_data (bool, optional): should we include EXIF data in the output for each image?
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): enable image augmentation
         detector_options (dict, optional): key/value pairs that are interpreted differently
             by different detectors
@@ -883,18 +887,19 @@ def _process_images(im_files,
 
     if use_image_queue:
 
-        _run_detector_with_image_queue(im_files,
+        results = _run_detector_with_image_queue(im_files,
                                       detector,
                                       confidence_threshold,
                                       quiet=quiet,
                                       image_size=image_size,
                                       include_image_size=include_image_size,
                                       include_image_timestamp=include_image_timestamp,
-                                      include_exif_data=include_exif_data,
+                                      include_exif_tags=include_exif_tags,
                                       augment=augment,
                                       detector_options=detector_options,
                                       loader_workers=loader_workers,
                                       preprocess_on_image_queue=preprocess_on_image_queue)
+        return results
 
     else:
 
@@ -907,7 +912,7 @@ def _process_images(im_files,
                                    image_size=image_size,
                                    include_image_size=include_image_size,
                                    include_image_timestamp=include_image_timestamp,
-                                   include_exif_data=include_exif_data,
+                                   include_exif_tags=include_exif_tags,
                                    augment=augment)
 
             if checkpoint_queue is not None:
@@ -929,7 +934,7 @@ def _process_image(im_file,
                    image_size=None,
                    include_image_size=False,
                    include_image_timestamp=False,
-                   include_exif_data=False,
+                   include_exif_tags=False,
                    augment=False):
     """
     Runs a detector (typically MegaDetector) on a single image file.
@@ -947,7 +952,7 @@ def _process_image(im_file,
             doing
         include_image_size (bool, optional): should we include image size in the output for each image?
         include_image_timestamp (bool, optional): should we include image timestamps in the output for each image?
-        include_exif_data (bool, optional): should we include EXIF data in the output for each image?
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): enable image augmentation
 
     Returns:
@@ -1000,7 +1005,9 @@ def _process_image(im_file,
     if include_image_timestamp:
         result['datetime'] = get_image_datetime(image)
 
-    if include_exif_data:
+    if include_exif_tags is not None:
+        exif_options = copy.copy(exif_options_base)
+        exif_options.tags_to_include = include_exif_tags
         result['exif_metadata'] = read_exif.read_pil_exif(image,exif_options)
 
     return result
@@ -1055,7 +1062,7 @@ def load_and_run_detector_batch(model_file,
                                 class_mapping_filename=None,
                                 include_image_size=False,
                                 include_image_timestamp=False,
-                                include_exif_data=False,
+                                include_exif_tags=None,
                                 augment=False,
                                 force_model_download=False,
                                 detector_options=None,
@@ -1088,7 +1095,7 @@ def load_and_run_detector_batch(model_file,
             file or YOLOv5 dataset.yaml file
         include_image_size (bool, optional): should we include image size in the output for each image?
         include_image_timestamp (bool, optional): should we include image timestamps in the output for each image?
-        include_exif_data (bool, optional): should we include EXIF data in the output for each image?
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): enable image augmentation
         force_model_download (bool, optional): force downloading the model file if
             a named model (e.g. "MDV5A") is supplied, even if the local file already
@@ -1207,7 +1214,7 @@ def load_and_run_detector_batch(model_file,
                           image_size=image_size,
                           include_image_size=include_image_size,
                           include_image_timestamp=include_image_timestamp,
-                          include_exif_data=include_exif_data,
+                          include_exif_tags=include_exif_tags,
                           augment=augment,
                           detector_options=detector_options,
                           loader_workers=loader_workers,
@@ -1269,7 +1276,7 @@ def load_and_run_detector_batch(model_file,
                                                image_size=image_size,
                                                include_image_size=include_image_size,
                                                include_image_timestamp=include_image_timestamp,
-                                               include_exif_data=include_exif_data,
+                                               include_exif_tags=include_exif_tags,
                                                augment=augment)
 
                 results.extend(batch_results)
@@ -1295,7 +1302,7 @@ def load_and_run_detector_batch(model_file,
                                        image_size=image_size,
                                        include_image_size=include_image_size,
                                        include_image_timestamp=include_image_timestamp,
-                                       include_exif_data=include_exif_data,
+                                       include_exif_tags=include_exif_tags,
                                        augment=augment)
                 results.append(result)
 
@@ -1354,7 +1361,7 @@ def load_and_run_detector_batch(model_file,
                                  checkpoint_queue=checkpoint_queue,
                                  include_image_size=include_image_size,
                                  include_image_timestamp=include_image_timestamp,
-                                 include_exif_data=include_exif_data,
+                                 include_exif_tags=include_exif_tags,
                                  augment=augment,
                                  detector_options=detector_options),
                                  image_chunks)
@@ -1374,7 +1381,7 @@ def load_and_run_detector_batch(model_file,
                                                image_size=image_size,
                                                include_image_size=include_image_size,
                                                include_image_timestamp=include_image_timestamp,
-                                               include_exif_data=include_exif_data,
+                                               include_exif_tags=include_exif_tags,
                                                augment=augment,
                                                detector_options=detector_options),
                                                image_chunks)
@@ -1495,7 +1502,7 @@ def get_image_datetime(image):
         returns None if EXIF datetime is not available.
     """
 
-    exif_tags = read_exif.read_pil_exif(image,exif_options)
+    exif_tags = read_exif.read_pil_exif(image,exif_options_base)
 
     try:
         datetime_str = exif_tags['DateTimeOriginal']
@@ -1654,7 +1661,7 @@ if False:
     class_mapping_filename = None
     include_image_size = True
     include_image_timestamp = True
-    include_exif_data = True
+    include_exif_tags = None
     overwrite_handling = None
 
     # Generate a command line
@@ -1689,8 +1696,8 @@ if False:
         cmd += ' --include_image_size'
     if include_image_timestamp:
         cmd += ' --include_image_timestamp'
-    if include_exif_data:
-        cmd += ' --include_exif_data'
+    if include_exif_tags is not None:
+        cmd += ' --include_exif_tags "{}"'.format(include_exif_tags)
     if overwrite_handling is not None:
         cmd += ' --overwrite_handling {}'.format(overwrite_handling)
 
@@ -1837,9 +1844,10 @@ def main(): # noqa
         help='Include image datetime (if available) in output file'
     )
     parser.add_argument(
-        '--include_exif_data',
-        action='store_true',
-        help='Include available EXIF data in output file'
+        '--include_exif_tags',
+        type=str,
+        default=None,
+        help='Command-separated list of EXIF tags to include in output, or "all" to include all tags'
     )
     parser.add_argument(
         '--overwrite_handling',
@@ -1878,6 +1886,12 @@ def main(): # noqa
         action='store_true',
         help=argparse.SUPPRESS)
 
+    # This argument is deprecated in favor use --include_exif_tags
+    parser.add_argument(
+        '--include_exif_data',
+        action='store_true',
+        help=argparse.SUPPRESS)
+
     if len(sys.argv[1:]) == 0:
         parser.print_help()
         parser.exit()
@@ -1888,6 +1902,10 @@ def main(): # noqa
     if args.use_threads_for_queue:
         use_threads_for_queue = True
 
+    # Support the legacy --include_exif_data flag
+    if args.include_exif_data and (args.include_exif_tags is None):
+        args.include_exif_tags = 'all'
+
     detector_options = parse_kvp_list(args.detector_options)
 
     # If the specified detector file is really the name of a known model, find
@@ -2094,7 +2112,7 @@ def main(): # noqa
                                           class_mapping_filename=args.class_mapping_filename,
                                           include_image_size=args.include_image_size,
                                           include_image_timestamp=args.include_image_timestamp,
-                                          include_exif_data=args.include_exif_data,
+                                          include_exif_tags=args.include_exif_tags,
                                           augment=args.augment,
                                           # Don't download the model *again*
                                           force_model_download=False,

megadetector/detection/run_inference_with_yolov5_val.py

@@ -351,7 +351,9 @@ def run_inference_with_yolo_val(options):
         else:
             raise ValueError('Unknown output handling method {}'.format(options.overwrite_handling))
 
-    os.makedirs(os.path.dirname(options.output_file),exist_ok=True)
+    output_dir = os.path.dirname(options.output_file)
+    if len(output_dir) > 0:
+        os.makedirs(output_dir, exist_ok=True)
 
     if options.input_folder is not None:
         options.input_folder = options.input_folder.replace('\\','/')