megadetector 5.0.27__py3-none-any.whl → 5.0.29__py3-none-any.whl

megadetector/data_management/remap_coco_categories.py

@@ -25,24 +25,24 @@ def remap_coco_categories(input_data,
     """
     Given a COCO-formatted dataset, remap the categories to a new categories mapping, optionally
     writing the results to a new file.
-    
+
     Args:
-        input_data (str or dict): a COCO-formatted dict or a filename.  If it's a dict, it will 
+        input_data (str or dict): a COCO-formatted dict or a filename.  If it's a dict, it will
             be copied, not modified in place.
-        output_category_name_to_id (dict) a dict mapping strings to ints.  Categories not in 
+        output_category_name_to_id (dict) a dict mapping strings to ints.  Categories not in
             this dict will be ignored or will result in errors, depending on allow_unused_categories.
-        input_category_name_to_output_category_name: a dict mapping strings to strings.  
-            Annotations using categories not in this dict will be omitted or will result in 
+        input_category_name_to_output_category_name: a dict mapping strings to strings.
+            Annotations using categories not in this dict will be omitted or will result in
             errors, depending on allow_unused_categories.
         output_file (str, optional): output file to which we should write remapped COCO data
         allow_unused_categories (bool, optional): should we ignore categories not present in the
             input/output mappings?  If this is False and we encounter an unmapped category, we'll
             error.
-    
+
     Returns:
         dict: COCO-formatted dict
     """
-    
+
     if isinstance(input_data,str):
         assert os.path.isfile(input_data), "Can't find file {}".format(input_data)
         with open(input_data,'r') as f:
@@ -51,45 +51,45 @@ def remap_coco_categories(input_data,
     else:
         assert isinstance(input_data,dict), 'Illegal COCO input data'
         input_data = deepcopy(input_data)
-    
+
     # It's safe to modify in-place now
     output_data = input_data
-    
+
     # Read input name --> ID mapping
     input_category_name_to_input_category_id = {}
     for c in input_data['categories']:
         input_category_name_to_input_category_id[c['name']] = c['id']
     input_category_id_to_input_category_name = \
         invert_dictionary(input_category_name_to_input_category_id)
-        
+
     # Map input IDs --> output IDs
     input_category_id_to_output_category_id = {}
     input_category_names = list(input_category_name_to_output_category_name.keys())
-    
+
     # input_name = input_category_names[0]
     for input_name in input_category_names:
-        
+
         output_name = input_category_name_to_output_category_name[input_name]
         assert output_name in output_category_name_to_id, \
             'No output ID for {} --> {}'.format(input_name,output_name)
         input_id = input_category_name_to_input_category_id[input_name]
         output_id = output_category_name_to_id[output_name]
         input_category_id_to_output_category_id[input_id] = output_id
-    
+
     # ...for each category we want to keep
-        
+
     printed_unused_category_warnings = set()
-    
+
     valid_annotations = []
-    
+
     # Map annotations
     for ann in output_data['annotations']:
-        
+
         input_category_id = ann['category_id']
         if input_category_id not in input_category_id_to_output_category_id:
             if allow_unused_categories:
                 if input_category_id not in printed_unused_category_warnings:
-                    printed_unused_category_warnings.add(input_category_id)            
+                    printed_unused_category_warnings.add(input_category_id)
                     input_category_name = \
                         input_category_id_to_input_category_name[input_category_id]
                     s = 'Skipping unmapped category ID {} ({})'.format(
@@ -98,31 +98,31 @@ def remap_coco_categories(input_data,
                 continue
             else:
                 s = 'Unmapped category ID {}'.format(input_category_id)
-                raise ValueError(s)        
+                raise ValueError(s)
         output_category_id = input_category_id_to_output_category_id[input_category_id]
         ann['category_id'] = output_category_id
-        valid_annotations.append(ann)        
-        
+        valid_annotations.append(ann)
+
     # ...for each annotation
-    
+
     # The only reason annotations should get excluded is the case where we allow
     # unused categories
     if not allow_unused_categories:
         assert len(valid_annotations) == len(output_data['annotations'])
-    
+
     output_data['annotations'] = valid_annotations
-    
+
     # Update the category list
     output_categories = []
     for output_name in output_category_name_to_id:
         category = {'name':output_name,'id':output_category_name_to_id[output_name]}
         output_categories.append(category)
     output_data['categories'] = output_categories
-        
+
     if output_file is not None:
         with open(output_file,'w') as f:
             json.dump(output_data,f,indent=1)
-            
+
     return input_data
 
 # ...def remap_coco_categories(...)

megadetector/data_management/remove_exif.py

@@ -2,7 +2,7 @@
 
 remove_exif.py
 
-Removes all EXIF/IPTC/XMP metadata from a folder of images, without making 
+Removes all EXIF/IPTC/XMP metadata from a folder of images, without making
 backup copies, using pyexiv2.  Ignores non-jpeg images.
 
 This module is rarely used, and pyexiv2 is not thread-safe, so pyexiv2 is not
@@ -21,22 +21,26 @@ from tqdm import tqdm
 
 #%% Support functions
 
-# Pyexif2 is not thread safe, do not call this function in parallel within a process
-#
-# Parallelizing across processes is fine.
 def remove_exif_from_image(fn):
+    """
+    Remove EXIF information from a single image
+
+    pyexiv2 is not thread safe, do not call this function in parallel within a process.
+
+    Parallelizing across processes is fine.
+    """
+
+    import pyexiv2 # type: ignore
 
-    import pyexiv2
-    
     try:
         img = pyexiv2.Image(fn)
         img.clear_exif()
         img.clear_iptc()
         img.clear_xmp()
-        img.close()        
+        img.close()
     except Exception as e:
         print('EXIF error on {}: {}'.format(fn,str(e)))
-    
+
     return True
 
 
@@ -44,22 +48,23 @@ def remove_exif_from_image(fn):
 
 def remove_exif(image_base_folder,recursive=True,n_processes=1):
     """
-    Removes all EXIF/IPTC/XMP metadata from a folder of images, without making 
+    Removes all EXIF/IPTC/XMP metadata from a folder of images, without making
     backup copies, using pyexiv2.  Ignores non-jpeg images.
-    
+
     Args:
         image_base_folder (str): the folder from which we should remove EXIF data
         recursive (bool, optional): whether to process [image_base_folder] recursively
         n_processes (int, optional): number of concurrent workers.  Because pyexiv2 is not
-            thread-safe, only process-based parallelism is supported.        
+            thread-safe, only process-based parallelism is supported.
     """
+
     try:
-        import pyexiv2 #noqa
+        import pyexiv2 # type: ignore #noqa
     except:
         print('pyexiv2 not available; try "pip install pyexiv2"')
         raise
 
-        
+
     ##%% List files
 
     assert os.path.isdir(image_base_folder), \
@@ -67,22 +72,28 @@ def remove_exif(image_base_folder,recursive=True,n_processes=1):
     all_files = [f for f in glob.glob(image_base_folder+ "*/**", recursive=recursive)]
     image_files = [s for s in all_files if \
                    (s.lower().endswith('.jpg') or s.lower().endswith('.jpeg'))]
-        
+
 
     ##%% Remove EXIF data (execution)
 
     if n_processes == 1:
-        
+
         # fn = image_files[0]
         for fn in tqdm(image_files):
             remove_exif_from_image(fn)
-            
+
     else:
         # pyexiv2 is not thread-safe, so we need to use processes
-        print('Starting parallel process pool with {} workers'.format(n_processes))
-        pool = Pool(n_processes)
-        _ = list(tqdm(pool.imap(remove_exif_from_image,image_files),total=len(image_files)))
-            
+        pool = None
+        try:
+            print('Starting parallel process pool with {} workers'.format(n_processes))
+            pool = Pool(n_processes)
+            _ = list(tqdm(pool.imap(remove_exif_from_image,image_files),total=len(image_files)))
+        finally:
+            pool.close()
+            pool.join()
+            print("Pool closed and joined for EXIF removal")
+
 # ...remove_exif(...)
 
 

megadetector/data_management/rename_images.py

@@ -1,187 +1,187 @@
-"""
-
-rename_images.py
-
-Copies images from a possibly-nested folder structure to a flat folder structure, including EXIF
-timestamps in each filename.  Loosely equivalent to camtrapR's imageRename() function.
-
-"""
-
-#%% Imports and constants
-
-import os
-
-from megadetector.utils.path_utils import \
-    find_images, insert_before_extension, parallel_copy_files
-from megadetector.data_management.read_exif import \
-    ReadExifOptions, read_exif_from_folder
-
-
-#%% Functions
-
-def rename_images(input_folder,
-                  output_folder,
-                  dry_run=False,
-                  verbose=False,
-                  read_exif_options=None,
-                  n_copy_workers=8):
-    """
-    For the given image struct in COCO format and associated list of annotations, reformats the 
-    detections into labelme format.  
-    
-    Args:
-        input_folder: the folder to search for images, always recursive
-        output_folder: the folder to which we will copy images; cannot be the
-            same as [input_folder]
-        dry_run: only map images, don't actually copy
-        verbose (bool, optional): enable additional debug output
-        read_exif_options (ReadExifOptions, optional): parameters controlling the reading of
-            EXIF information
-        n_copy_workers (int, optional): number of parallel threads to use for copying
-                    
-    Returns:
-        dict: a dict mapping relative filenames in the input folder to relative filenames in the output 
-        folder
-    """
-    
-    assert os.path.isdir(input_folder), 'Input folder {} does not exist'.format(
-        input_folder)
-    
-    if not dry_run:
-        os.makedirs(output_folder,exist_ok=True)
-    
-    # Read exif information
-    if read_exif_options is None:
-        read_exif_options = ReadExifOptions()
-   
-    read_exif_options.tags_to_include = ['DateTime','Model','Make','ExifImageWidth','ExifImageHeight','DateTime',
-                                         'DateTimeOriginal']    
-    read_exif_options.verbose = False
-    
-    exif_info = read_exif_from_folder(input_folder=input_folder,
-                                      output_file=None,
-                                      options=read_exif_options,
-                                      filenames=None,recursive=True)
-    
-    print('Read EXIF information for {} images'.format(len(exif_info)))
-    
-    filename_to_exif_info = {info['file_name']:info for info in exif_info}
-    
-    image_files = find_images(input_folder,return_relative_paths=True,convert_slashes=True,recursive=True)
-    
-    for fn in image_files:
-        assert fn in filename_to_exif_info, 'No EXIF info available for {}'.format(fn)
-    
-    input_fn_relative_to_output_fn_relative = {}
-    
-    # fn_relative = image_files[0]
-    for fn_relative in image_files:
-        
-        input_fn_abs = os.path.join(input_folder,fn_relative)
-        image_exif_info = filename_to_exif_info[fn_relative]
-        if 'exif_tags' in image_exif_info:
-            image_exif_info = image_exif_info['exif_tags']
-        
-        if image_exif_info is None or \
-            'DateTimeOriginal' not in image_exif_info or \
-            image_exif_info['DateTimeOriginal'] is None:
-                
-            dt_tag = 'unknown_datetime'
-            print('Warning: no datetime for {}'.format(fn_relative))
-            
-        else:
-            
-            dt_tag = str(image_exif_info['DateTimeOriginal']).replace(':','-').replace(' ','_').strip()            
-        
-        flat_filename = fn_relative.replace('\\','/').replace('/','_')
-        
-        output_fn_relative = insert_before_extension(flat_filename,dt_tag)
-        
-        input_fn_relative_to_output_fn_relative[fn_relative] = output_fn_relative
-        
-    if not dry_run:
-        
-        input_fn_abs_to_output_fn_abs = {}
-        for input_fn_relative in input_fn_relative_to_output_fn_relative:
-            output_fn_relative = input_fn_relative_to_output_fn_relative[input_fn_relative]
-            input_fn_abs = os.path.join(input_folder,input_fn_relative)
-            output_fn_abs = os.path.join(output_folder,output_fn_relative)
-            input_fn_abs_to_output_fn_abs[input_fn_abs] = output_fn_abs
-            
-            parallel_copy_files(input_file_to_output_file=input_fn_abs_to_output_fn_abs,
-                                max_workers=n_copy_workers, 
-                                use_threads=True, 
-                                overwrite=True, 
-                                verbose=verbose)
-            
-    return input_fn_relative_to_output_fn_relative
-
-# ...def rename_images()
-
-
-#%% Interactive driver
-
-if False:
-    
-    pass
-
-    #%% Configure options
-    
-    input_folder = r'G:\camera_traps\camera_trap_videos\2024.05.25\cam3'
-    output_folder = r'G:\camera_traps\camera_trap_videos\2024.05.25\cam3_flat'
-    dry_run = False
-    verbose = True
-    read_exif_options = ReadExifOptions()
-    read_exif_options.tags_to_include = ['DateTime','Model','Make','ExifImageWidth','ExifImageHeight','DateTime',
-                               'DateTimeOriginal']    
-    read_exif_options.n_workers = 8
-    read_exif_options.verbose = verbose    
-    n_copy_workers = 8
-    
-    
-    #%% Programmatic execution
-    
-    input_fn_relative_to_output_fn_relative = rename_images(input_folder,
-                                                            output_folder,
-                                                            dry_run=dry_run,
-                                                            verbose=verbose,
-                                                            read_exif_options=read_exif_options,
-                                                            n_copy_workers=n_copy_workers)
-    
-
-#%% Command-line driver
-
-import sys,argparse
-
-def main():
-
-    parser = argparse.ArgumentParser(
-        description='Copies images from a possibly-nested folder structure to a flat folder structure, ' + \
-            'adding datetime information from EXIF to each filename')
-    
-    parser.add_argument(
-        'input_folder',
-        type=str,
-        help='The folder to search for images, always recursive')
-    
-    parser.add_argument(
-        'output_folder',
-        type=str,
-        help='The folder to which we should write the flattened image structure')
-    
-    parser.add_argument(
-        '--dry_run',
-        action='store_true',
-        help="Only map images, don't actually copy")
-
-    if len(sys.argv[1:]) == 0:
-        parser.print_help()
-        parser.exit()
-
-    args = parser.parse_args()
-
-    rename_images(args.input_folder,args.output_folder,dry_run=args.dry_run,
-                  verbose=True,read_exif_options=None)
-
-if __name__ == '__main__':
-    main()
+"""
+
+rename_images.py
+
+Copies images from a possibly-nested folder structure to a flat folder structure, including EXIF
+timestamps in each filename.  Loosely equivalent to camtrapR's imageRename() function.
+
+"""
+
+#%% Imports and constants
+
+import os
+import sys
+import argparse
+
+from megadetector.utils.path_utils import \
+    find_images, insert_before_extension, parallel_copy_files
+from megadetector.data_management.read_exif import \
+    ReadExifOptions, read_exif_from_folder
+
+
+#%% Functions
+
+def rename_images(input_folder,
+                  output_folder,
+                  dry_run=False,
+                  verbose=False,
+                  read_exif_options=None,
+                  n_copy_workers=8):
+    """
+    For the given image struct in COCO format and associated list of annotations, reformats the
+    detections into labelme format.
+
+    Args:
+        input_folder: the folder to search for images, always recursive
+        output_folder: the folder to which we will copy images; cannot be the
+            same as [input_folder]
+        dry_run: only map images, don't actually copy
+        verbose (bool, optional): enable additional debug output
+        read_exif_options (ReadExifOptions, optional): parameters controlling the reading of
+            EXIF information
+        n_copy_workers (int, optional): number of parallel threads to use for copying
+
+    Returns:
+        dict: a dict mapping relative filenames in the input folder to relative filenames in the output
+        folder
+    """
+
+    assert os.path.isdir(input_folder), 'Input folder {} does not exist'.format(
+        input_folder)
+
+    if not dry_run:
+        os.makedirs(output_folder,exist_ok=True)
+
+    # Read exif information
+    if read_exif_options is None:
+        read_exif_options = ReadExifOptions()
+
+    read_exif_options.tags_to_include = ['DateTime','Model','Make','ExifImageWidth','ExifImageHeight','DateTime',
+                                         'DateTimeOriginal']
+    read_exif_options.verbose = False
+
+    exif_info = read_exif_from_folder(input_folder=input_folder,
+                                      output_file=None,
+                                      options=read_exif_options,
+                                      filenames=None,recursive=True)
+
+    print('Read EXIF information for {} images'.format(len(exif_info)))
+
+    filename_to_exif_info = {info['file_name']:info for info in exif_info}
+
+    image_files = find_images(input_folder,return_relative_paths=True,convert_slashes=True,recursive=True)
+
+    for fn in image_files:
+        assert fn in filename_to_exif_info, 'No EXIF info available for {}'.format(fn)
+
+    input_fn_relative_to_output_fn_relative = {}
+
+    # fn_relative = image_files[0]
+    for fn_relative in image_files:
+
+        input_fn_abs = os.path.join(input_folder,fn_relative)
+        image_exif_info = filename_to_exif_info[fn_relative]
+        if 'exif_tags' in image_exif_info:
+            image_exif_info = image_exif_info['exif_tags']
+
+        if image_exif_info is None or \
+            'DateTimeOriginal' not in image_exif_info or \
+            image_exif_info['DateTimeOriginal'] is None:
+
+            dt_tag = 'unknown_datetime'
+            print('Warning: no datetime for {}'.format(fn_relative))
+
+        else:
+
+            dt_tag = str(image_exif_info['DateTimeOriginal']).replace(':','-').replace(' ','_').strip()
+
+        flat_filename = fn_relative.replace('\\','/').replace('/','_')
+
+        output_fn_relative = insert_before_extension(flat_filename,dt_tag)
+
+        input_fn_relative_to_output_fn_relative[fn_relative] = output_fn_relative
+
+    if not dry_run:
+
+        input_fn_abs_to_output_fn_abs = {}
+        for input_fn_relative in input_fn_relative_to_output_fn_relative:
+            output_fn_relative = input_fn_relative_to_output_fn_relative[input_fn_relative]
+            input_fn_abs = os.path.join(input_folder,input_fn_relative)
+            output_fn_abs = os.path.join(output_folder,output_fn_relative)
+            input_fn_abs_to_output_fn_abs[input_fn_abs] = output_fn_abs
+
+            parallel_copy_files(input_file_to_output_file=input_fn_abs_to_output_fn_abs,
+                                max_workers=n_copy_workers,
+                                use_threads=True,
+                                overwrite=True,
+                                verbose=verbose)
+
+    return input_fn_relative_to_output_fn_relative
+
+# ...def rename_images()
+
+
+#%% Interactive driver
+
+if False:
+
+    pass
+
+    #%% Configure options
+
+    input_folder = r'G:\camera_traps\camera_trap_videos\2024.05.25\cam3'
+    output_folder = r'G:\camera_traps\camera_trap_videos\2024.05.25\cam3_flat'
+    dry_run = False
+    verbose = True
+    read_exif_options = ReadExifOptions()
+    read_exif_options.tags_to_include = ['DateTime','Model','Make','ExifImageWidth','ExifImageHeight','DateTime',
+                               'DateTimeOriginal']
+    read_exif_options.n_workers = 8
+    read_exif_options.verbose = verbose
+    n_copy_workers = 8
+
+
+    #%% Programmatic execution
+
+    input_fn_relative_to_output_fn_relative = rename_images(input_folder,
+                                                            output_folder,
+                                                            dry_run=dry_run,
+                                                            verbose=verbose,
+                                                            read_exif_options=read_exif_options,
+                                                            n_copy_workers=n_copy_workers)
+
+
+#%% Command-line driver
+
+def main(): # noqa
+
+    parser = argparse.ArgumentParser(
+        description='Copies images from a possibly-nested folder structure to a flat folder structure, ' + \
+            'adding datetime information from EXIF to each filename')
+
+    parser.add_argument(
+        'input_folder',
+        type=str,
+        help='The folder to search for images, always recursive')
+
+    parser.add_argument(
+        'output_folder',
+        type=str,
+        help='The folder to which we should write the flattened image structure')
+
+    parser.add_argument(
+        '--dry_run',
+        action='store_true',
+        help="Only map images, don't actually copy")
+
+    if len(sys.argv[1:]) == 0:
+        parser.print_help()
+        parser.exit()
+
+    args = parser.parse_args()
+
+    rename_images(args.input_folder,args.output_folder,dry_run=args.dry_run,
+                  verbose=True,read_exif_options=None)
+
+if __name__ == '__main__':
+    main()