megadetector 10.0.8__py3-none-any.whl → 10.0.10__py3-none-any.whl

megadetector/data_management/read_exif.py

@@ -210,6 +210,8 @@ def read_pil_exif(im,options=None):
     if exif_info is None:
         return exif_tags
 
+    # Read all standard EXIF tags; if necessary, we'll filter later to a restricted
+    # list of tags.
     for k, v in exif_info.items():
         assert isinstance(k,str) or isinstance(k,int), \
             'Invalid EXIF key {}'.format(str(k))
@@ -221,6 +223,7 @@ def read_pil_exif(im,options=None):
 
     exif_ifd_tags = _get_exif_ifd(exif_info)
 
+    # Read tags that are only available via offset
     for k in exif_ifd_tags.keys():
         v = exif_ifd_tags[k]
         if k in exif_tags:
@@ -266,7 +269,7 @@ def read_pil_exif(im,options=None):
 
             # Convert to strings, e.g. 'GPSTimeStamp'
             gps_info = {}
-            for int_tag,v in enumerate(gps_info_raw.keys()):
+            for int_tag,v in gps_info_raw.items():
                 assert isinstance(int_tag,int)
                 if int_tag in ExifTags.GPSTAGS:
                     gps_info[ExifTags.GPSTAGS[int_tag]] = v
@@ -276,11 +279,15 @@ def read_pil_exif(im,options=None):
             exif_tags['GPSInfo'] = gps_info
 
         except Exception as e:
+
             if options.verbose:
                 print('Warning: error reading GPS info: {}'.format(str(e)))
 
     # ...if we think there might be GPS tags in this image
 
+    # Filter tags if necessary
+    exif_tags = _filter_tags(exif_tags,options)
+
     return exif_tags
 
 # ...read_pil_exif()
@@ -337,10 +344,16 @@ def _filter_tags(tags,options):
     if options.tags_to_include is None and options.tags_to_exclude is None:
         return tags
     if options.tags_to_include is not None:
+        if isinstance(options.tags_to_include,str):
+            if options.tags_to_include == 'all':
+                return tags
         assert options.tags_to_exclude is None, "tags_to_include and tags_to_exclude are incompatible"
+        tags_to_include = options.tags_to_include.split(',')
+        # Case-insensitive matching
+        tags_to_include = [s.lower() for s in tags_to_include]
         tags_to_return = {}
         for tag_name in tags.keys():
-            if tag_name in options.tags_to_include:
+            if str(tag_name).lower() in tags_to_include:
                 tags_to_return[tag_name] = tags[tag_name]
         return tags_to_return
     if options.tags_to_exclude is not None:

megadetector/detection/pytorch_detector.py

@@ -859,6 +859,9 @@ class PTDetector:
             except AttributeError:
                 pass
 
+        # AddaxAI depends on this printout, don't remove it
+        print('PTDetector using device {}'.format(str(self.device).lower()))
+
         try:
             self.model = PTDetector._load_model(model_path,
                                                 device=self.device,

megadetector/detection/run_detector.py

@@ -596,8 +596,7 @@ def load_detector(model_file,
     model_file = try_download_known_detector(model_file,
                                              force_download=force_model_download)
 
-    if verbose:
-        print('GPU available: {}'.format(is_gpu_available(model_file)))
+    print('GPU available: {}'.format(is_gpu_available(model_file)))
 
     start_time = time.time()
 

megadetector/detection/run_detector_batch.py

@@ -94,20 +94,29 @@ max_queue_size = 10
 # How often should we print progress when using the image queue?
 n_queue_print = 1000
 
-# TODO: it's a little sloppy that these are module-level globals, but in practice it
+# Only used if --include_exif_tags or --include_image_timestamp are supplied
+exif_options_base = read_exif.ReadExifOptions()
+exif_options_base.processing_library = 'pil'
+exif_options_base.byte_handling = 'convert_to_string'
+
+# Only relevant when we're running our test harness; because bugs in batch
+# inference are dependent on batch grouping, we randomize batch grouping
+# during testing to maximize the probability that latent bugs come up
+# eventually.
+randomize_batch_order_during_testing = True
+
+# TODO: it's a little sloppy that the following are module-level globals, but in practice it
 # doesn't really matter, so I'm not in a big rush to move these to options until I do
 # a larger cleanup of all the long argument lists in this module.
-#
+
 # Should the consumer loop run on its own process, or here in the main process?
 run_separate_consumer_process = False
-use_threads_for_queue = False
-verbose = False
 
-exif_options = read_exif.ReadExifOptions()
-exif_options.processing_library = 'pil'
-exif_options.byte_handling = 'convert_to_string'
+# Should we use threads (rather than processes) for the data loading workers?
+use_threads_for_queue = False
 
-randomize_batch_order_during_testing = True
+# Enable additional debug output
+verbose = False
 
 
 #%% Support functions for multiprocessing
@@ -199,7 +208,7 @@ def _consumer_func(q,
                    image_size=None,
                    include_image_size=False,
                    include_image_timestamp=False,
-                   include_exif_data=False,
+                   include_exif_tags=None,
                    augment=False,
                    detector_options=None,
                    preprocess_on_image_queue=default_preprocess_on_image_queue,
@@ -223,7 +232,7 @@ def _consumer_func(q,
         image_size (int, optional): image size to use for inference
         include_image_size (bool, optional): include image dimensions in output
         include_image_timestamp (bool, optional): include image timestamps in output
-        include_exif_data (bool, optional): include EXIF data in output
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): enable image augmentation
         detector_options (dict, optional): key/value pairs that are interpreted differently
             by different detectors
@@ -325,7 +334,7 @@ def _consumer_func(q,
                                                        image_size=image_size,
                                                        include_image_size=include_image_size,
                                                        include_image_timestamp=include_image_timestamp,
-                                                       include_exif_data=include_exif_data,
+                                                       include_exif_tags=include_exif_tags,
                                                        augment=augment)
                         results.extend(batch_results)
 
@@ -402,7 +411,7 @@ def _consumer_func(q,
                                                    image_size=image_size,
                                                    include_image_size=include_image_size,
                                                    include_image_timestamp=include_image_timestamp,
-                                                   include_exif_data=include_exif_data,
+                                                   include_exif_tags=include_exif_tags,
                                                    augment=augment)
                     results.extend(batch_results)
 
@@ -422,7 +431,7 @@ def _consumer_func(q,
                                         image_size=image_size,
                                         include_image_size=include_image_size,
                                         include_image_timestamp=include_image_timestamp,
-                                        include_exif_data=include_exif_data,
+                                        include_exif_tags=include_exif_tags,
                                         augment=augment)
                 results.append(result)
                 n_images_processed += 1
@@ -455,7 +464,7 @@ def _run_detector_with_image_queue(image_files,
                                    image_size=None,
                                    include_image_size=False,
                                    include_image_timestamp=False,
-                                   include_exif_data=False,
+                                   include_exif_tags=None,
                                    augment=False,
                                    detector_options=None,
                                    loader_workers=default_loaders,
@@ -478,7 +487,7 @@ def _run_detector_with_image_queue(image_files,
             doing
         include_image_size (bool, optional): should we include image size in the output for each image?
         include_image_timestamp (bool, optional): should we include image timestamps in the output for each image?
-        include_exif_data (bool, optional): should we include EXIF data in the output for each image?
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): enable image augmentation
         detector_options (dict, optional): key/value pairs that are interpreted differently
             by different detectors
@@ -553,7 +562,7 @@ def _run_detector_with_image_queue(image_files,
                                                           image_size,
                                                           include_image_size,
                                                           include_image_timestamp,
-                                                          include_exif_data,
+                                                          include_exif_tags,
                                                           augment,
                                                           detector_options,
                                                           preprocess_on_image_queue,
@@ -570,7 +579,7 @@ def _run_detector_with_image_queue(image_files,
                                                            image_size,
                                                            include_image_size,
                                                            include_image_timestamp,
-                                                           include_exif_data,
+                                                           include_exif_tags,
                                                            augment,
                                                            detector_options,
                                                            preprocess_on_image_queue,
@@ -589,7 +598,7 @@ def _run_detector_with_image_queue(image_files,
                        image_size,
                        include_image_size,
                        include_image_timestamp,
-                       include_exif_data,
+                       include_exif_tags,
                        augment,
                        detector_options,
                        preprocess_on_image_queue,
@@ -671,7 +680,7 @@ def _process_batch(image_items_batch,
                    image_size=None,
                    include_image_size=False,
                    include_image_timestamp=False,
-                   include_exif_data=False,
+                   include_exif_tags=None,
                    augment=False):
     """
     Process a batch of images using generate_detections_one_batch().  Does not necessarily return
@@ -686,7 +695,7 @@ def _process_batch(image_items_batch,
         image_size (int, optional): image size override
         include_image_size (bool, optional): include image dimensions in results
         include_image_timestamp (bool, optional): include image timestamps in results
-        include_exif_data (bool, optional): include EXIF data in results
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): whether to use image augmentation
 
     Returns:
@@ -736,7 +745,9 @@ def _process_batch(image_items_batch,
         try:
 
             batch_detections = \
-                detector.generate_detections_one_batch(valid_images, valid_image_filenames, verbose=verbose)
+                detector.generate_detections_one_batch(valid_images,
+                                                       valid_image_filenames,
+                                                       verbose=verbose)
 
             assert len(batch_detections) == len(valid_images)
 
@@ -751,12 +762,13 @@ def _process_batch(image_items_batch,
                     image_result['detections'] = \
                         [det for det in image_result['detections'] if det['conf'] >= confidence_threshold]
 
-                    if include_image_size or include_image_timestamp or include_exif_data:
+                    if include_image_size or include_image_timestamp or (include_exif_tags is not None):
 
                         image = valid_images[i_valid_image]
 
                         # If this was preprocessed by the producer thread, pull out the PIL version
                         if isinstance(image,dict):
+
                             image = image['img_original_pil']
 
                         if include_image_size:
@@ -768,9 +780,12 @@ def _process_batch(image_items_batch,
 
                             image_result['datetime'] = get_image_datetime(image)
 
-                        if include_exif_data:
+                        if include_exif_tags is not None:
 
-                            image_result['exif_metadata'] = read_exif.read_pil_exif(image,exif_options)
+                            exif_options = copy.copy(exif_options_base)
+                            exif_options.tags_to_include = include_exif_tags
+                            image_result['exif_metadata'] = read_exif.read_pil_exif(
+                                image,exif_options)
 
                     # ...if we need to store metadata
 
@@ -823,7 +838,7 @@ def _process_images(im_files,
                    checkpoint_queue=None,
                    include_image_size=False,
                    include_image_timestamp=False,
-                   include_exif_data=False,
+                   include_exif_tags=None,
                    augment=False,
                    detector_options=None,
                    loader_workers=default_loaders,
@@ -845,7 +860,7 @@ def _process_images(im_files,
         checkpoint_queue (Queue, optional): internal parameter used to pass image queues around
         include_image_size (bool, optional): should we include image size in the output for each image?
         include_image_timestamp (bool, optional): should we include image timestamps in the output for each image?
-        include_exif_data (bool, optional): should we include EXIF data in the output for each image?
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): enable image augmentation
         detector_options (dict, optional): key/value pairs that are interpreted differently
             by different detectors
@@ -879,7 +894,7 @@ def _process_images(im_files,
                                       image_size=image_size,
                                       include_image_size=include_image_size,
                                       include_image_timestamp=include_image_timestamp,
-                                      include_exif_data=include_exif_data,
+                                      include_exif_tags=include_exif_tags,
                                       augment=augment,
                                       detector_options=detector_options,
                                       loader_workers=loader_workers,
@@ -896,7 +911,7 @@ def _process_images(im_files,
                                    image_size=image_size,
                                    include_image_size=include_image_size,
                                    include_image_timestamp=include_image_timestamp,
-                                   include_exif_data=include_exif_data,
+                                   include_exif_tags=include_exif_tags,
                                    augment=augment)
 
             if checkpoint_queue is not None:
@@ -918,7 +933,7 @@ def _process_image(im_file,
                    image_size=None,
                    include_image_size=False,
                    include_image_timestamp=False,
-                   include_exif_data=False,
+                   include_exif_tags=False,
                    augment=False):
     """
     Runs a detector (typically MegaDetector) on a single image file.
@@ -936,7 +951,7 @@ def _process_image(im_file,
             doing
         include_image_size (bool, optional): should we include image size in the output for each image?
         include_image_timestamp (bool, optional): should we include image timestamps in the output for each image?
-        include_exif_data (bool, optional): should we include EXIF data in the output for each image?
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): enable image augmentation
 
     Returns:
@@ -989,7 +1004,9 @@ def _process_image(im_file,
     if include_image_timestamp:
         result['datetime'] = get_image_datetime(image)
 
-    if include_exif_data:
+    if include_exif_tags is not None:
+        exif_options = copy.copy(exif_options_base)
+        exif_options.tags_to_include = include_exif_tags
         result['exif_metadata'] = read_exif.read_pil_exif(image,exif_options)
 
     return result
@@ -1044,13 +1061,14 @@ def load_and_run_detector_batch(model_file,
                                 class_mapping_filename=None,
                                 include_image_size=False,
                                 include_image_timestamp=False,
-                                include_exif_data=False,
+                                include_exif_tags=None,
                                 augment=False,
                                 force_model_download=False,
                                 detector_options=None,
                                 loader_workers=default_loaders,
                                 preprocess_on_image_queue=default_preprocess_on_image_queue,
-                                batch_size=1):
+                                batch_size=1,
+                                verbose_output=False):
     """
     Load a model file and run it on a list of images.
 
@@ -1076,7 +1094,7 @@ def load_and_run_detector_batch(model_file,
             file or YOLOv5 dataset.yaml file
         include_image_size (bool, optional): should we include image size in the output for each image?
         include_image_timestamp (bool, optional): should we include image timestamps in the output for each image?
-        include_exif_data (bool, optional): should we include EXIF data in the output for each image?
+        include_exif_tags (str, optional): comma-separated list of EXIF tags to include in output
         augment (bool, optional): enable image augmentation
         force_model_download (bool, optional): force downloading the model file if
             a named model (e.g. "MDV5A") is supplied, even if the local file already
@@ -1087,6 +1105,7 @@ def load_and_run_detector_batch(model_file,
         preprocess_on_image_queue (bool, optional): if the image queue is enabled, should it handle
             image loading and preprocessing (True), or just image loading (False)?
         batch_size (int, optional): batch size for GPU processing, automatically set to 1 for CPU processing
+        verbose_output (bool, optional): enable additional debug output
 
     Returns:
         results: list of dicts; each dict represents detections on one image
@@ -1109,6 +1128,11 @@ def load_and_run_detector_batch(model_file,
     if class_mapping_filename is not None:
         _load_custom_class_mapping(class_mapping_filename)
 
+    global verbose
+    if verbose_output:
+        print('Enabling verbose output')
+        verbose = True
+
     # Handle the case where image_file_names is not yet actually a list
     if isinstance(image_file_names,str):
 
@@ -1189,7 +1213,7 @@ def load_and_run_detector_batch(model_file,
                           image_size=image_size,
                           include_image_size=include_image_size,
                           include_image_timestamp=include_image_timestamp,
-                          include_exif_data=include_exif_data,
+                          include_exif_tags=include_exif_tags,
                           augment=augment,
                           detector_options=detector_options,
                           loader_workers=loader_workers,
@@ -1251,7 +1275,7 @@ def load_and_run_detector_batch(model_file,
                                                image_size=image_size,
                                                include_image_size=include_image_size,
                                                include_image_timestamp=include_image_timestamp,
-                                               include_exif_data=include_exif_data,
+                                               include_exif_tags=include_exif_tags,
                                                augment=augment)
 
                 results.extend(batch_results)
@@ -1277,7 +1301,7 @@ def load_and_run_detector_batch(model_file,
                                        image_size=image_size,
                                        include_image_size=include_image_size,
                                        include_image_timestamp=include_image_timestamp,
-                                       include_exif_data=include_exif_data,
+                                       include_exif_tags=include_exif_tags,
                                        augment=augment)
                 results.append(result)
 
@@ -1336,7 +1360,7 @@ def load_and_run_detector_batch(model_file,
                                  checkpoint_queue=checkpoint_queue,
                                  include_image_size=include_image_size,
                                  include_image_timestamp=include_image_timestamp,
-                                 include_exif_data=include_exif_data,
+                                 include_exif_tags=include_exif_tags,
                                  augment=augment,
                                  detector_options=detector_options),
                                  image_chunks)
@@ -1356,7 +1380,7 @@ def load_and_run_detector_batch(model_file,
                                                image_size=image_size,
                                                include_image_size=include_image_size,
                                                include_image_timestamp=include_image_timestamp,
-                                               include_exif_data=include_exif_data,
+                                               include_exif_tags=include_exif_tags,
                                                augment=augment,
                                                detector_options=detector_options),
                                                image_chunks)
@@ -1477,7 +1501,7 @@ def get_image_datetime(image):
         returns None if EXIF datetime is not available.
     """
 
-    exif_tags = read_exif.read_pil_exif(image,exif_options)
+    exif_tags = read_exif.read_pil_exif(image,exif_options_base)
 
     try:
         datetime_str = exif_tags['DateTimeOriginal']
@@ -1636,7 +1660,7 @@ if False:
     class_mapping_filename = None
     include_image_size = True
     include_image_timestamp = True
-    include_exif_data = True
+    include_exif_tags = None
     overwrite_handling = None
 
     # Generate a command line
@@ -1671,8 +1695,8 @@ if False:
         cmd += ' --include_image_size'
     if include_image_timestamp:
         cmd += ' --include_image_timestamp'
-    if include_exif_data:
-        cmd += ' --include_exif_data'
+    if include_exif_tags is not None:
+        cmd += ' --include_exif_tags "{}"'.format(include_exif_tags)
     if overwrite_handling is not None:
         cmd += ' --overwrite_handling {}'.format(overwrite_handling)
 
@@ -1819,9 +1843,10 @@ def main(): # noqa
         help='Include image datetime (if available) in output file'
     )
     parser.add_argument(
-        '--include_exif_data',
-        action='store_true',
-        help='Include available EXIF data in output file'
+        '--include_exif_tags',
+        type=str,
+        default=None,
+        help='Command-separated list of EXIF tags to include in output, or "all" to include all tags'
     )
     parser.add_argument(
         '--overwrite_handling',
@@ -1860,20 +1885,26 @@ def main(): # noqa
         action='store_true',
         help=argparse.SUPPRESS)
 
+    # This argument is deprecated in favor use --include_exif_tags
+    parser.add_argument(
+        '--include_exif_data',
+        action='store_true',
+        help=argparse.SUPPRESS)
+
     if len(sys.argv[1:]) == 0:
         parser.print_help()
         parser.exit()
 
     args = parser.parse_args()
 
-    global verbose
     global use_threads_for_queue
-
-    if args.verbose:
-        verbose = True
     if args.use_threads_for_queue:
         use_threads_for_queue = True
 
+    # Support the legacy --include_exif_data flag
+    if args.include_exif_data and (args.include_exif_tags is None):
+        args.include_exif_tags = 'all'
+
     detector_options = parse_kvp_list(args.detector_options)
 
     # If the specified detector file is really the name of a known model, find
@@ -2080,14 +2111,15 @@ def main(): # noqa
                                           class_mapping_filename=args.class_mapping_filename,
                                           include_image_size=args.include_image_size,
                                           include_image_timestamp=args.include_image_timestamp,
-                                          include_exif_data=args.include_exif_data,
+                                          include_exif_tags=args.include_exif_tags,
                                           augment=args.augment,
                                           # Don't download the model *again*
                                           force_model_download=False,
                                           detector_options=detector_options,
                                           loader_workers=args.loader_workers,
                                           preprocess_on_image_queue=args.preprocess_on_image_queue,
-                                          batch_size=args.batch_size)
+                                          batch_size=args.batch_size,
+                                          verbose_output=args.verbose)
 
     elapsed = time.time() - start_time
     images_per_second = len(results) / elapsed