megadetector 5.0.14__py3-none-any.whl → 5.0.16__py3-none-any.whl

megadetector/data_management/importers/import_desert_lion_conservation_camera_traps.py

@@ -0,0 +1,387 @@
+"""
+
+import_desert_lion_conservation_camera_traps.py
+
+Prepare the Desert Lion Conservation Camera Traps dataset for release on LILA.
+
+"""
+
+#%% Imports and constants
+
+import os
+import json
+
+input_base_folder = r'i:/data/desert-lion'
+assert os.path.isdir(input_base_folder)
+
+# md_results_file = r'i:/data/desert-lion/desert-lion-camera-traps-2024-07-14-v5a.0.0_detections-all.json'
+md_results_file = r'i:/data/desert-lion/desert-lion-camera-traps-2024-07-14-v5a.0.0_detections.json'
+assert os.path.isfile(md_results_file)
+
+export_base = os.path.join(input_base_folder,'annotated-imgs')
+assert os.path.isdir(export_base)
+
+preview_dir = r'g:\temp\desert-lion-viz'
+output_file = os.path.join(input_base_folder,'desert_lion_camera_traps.json')
+output_zipfile = os.path.join(input_base_folder,'desert-lion-camera-traps-images.zip')
+
+exif_cache_file_post_exif_removal = os.path.join(input_base_folder,'exif_data_post_exif_removal.json')
+exif_cache_file = os.path.join(input_base_folder,'exif_data.json')
+
+
+#%% Find images and videos
+
+from megadetector.detection.video_utils import find_videos
+from megadetector.utils.path_utils import find_images
+
+video_files = find_videos(input_base_folder,recursive=True,return_relative_paths=True,convert_slashes=True)
+image_files = find_images(input_base_folder,recursive=True,return_relative_paths=True,convert_slashes=True)
+
+n_annotated_imgs = len([fn for fn in image_files if 'annotated-imgs' in fn])
+print('Found {} images ({} in the annotated-imgs folder), {} videos'.format(
+    len(image_files),n_annotated_imgs,len(video_files)))
+
+
+#%% Read EXIF data
+
+from megadetector.data_management.read_exif import read_exif_from_folder, ReadExifOptions
+
+exif_options = ReadExifOptions()
+exif_options.n_workers = 10
+
+if os.path.isfile(exif_cache_file):
+    print('EXIF cache {} exists, skipping EXIF read'.format(exif_cache_file))
+    with open(exif_cache_file,'r') as f:
+        exif_data = json.load(f)    
+else:
+    exif_data = read_exif_from_folder(input_folder=input_base_folder,
+                                      output_file=exif_cache_file,
+                                      options=exif_options,
+                                      filenames=None,
+                                      recursive=True)
+
+assert len(exif_data) == len(image_files)
+
+
+#%% Remove EXIF data
+
+from megadetector.data_management.remove_exif import remove_exif
+remove_exif(input_base_folder,recursive=True,n_processes=1)
+
+
+#%% Read EXIF data again
+
+exif_data_post_exif_removal = read_exif_from_folder(input_folder=input_base_folder,
+                                                    output_file=exif_cache_file_post_exif_removal,
+                                                    options=exif_options,
+                                                    filenames=None,
+                                                    recursive=True)
+
+
+#%% Make sure no lat/lon data is present
+
+from tqdm import tqdm
+
+for i_image,im in enumerate(tqdm(exif_data_post_exif_removal)):
+    tags = im['exif_tags']
+    if tags is None:
+        continue
+    for k in tags:
+        assert 'gps' not in str(k).lower()
+
+
+#%% Look for images that contain humans
+
+with open(md_results_file,'r') as f:
+    md_results = json.load(f)
+
+assert len(md_results['images']) == len(image_files)
+
+human_threshold = 0.1
+human_categories = ['2','3']
+
+candidate_human_images = set()
+failed_images = set()
+
+# i_image = 0; im = md_results['images'][0]
+for i_image,im in tqdm(enumerate(md_results['images']),total=len(md_results['images'])):
+
+    if 'failure' in im:
+        failed_images.add(im['file'])
+        continue
+    
+    for det in im['detections']:
+        if det['category'] in human_categories and det['conf'] >= human_threshold:
+            candidate_human_images.add(im['file'])
+            break
+        
+    # ...for each detection
+    
+# ...for each image    
+
+print('Found {} failed images and {} candidate human images'.format(
+    len(failed_images),len(candidate_human_images)))
+
+
+#%% Copy failed images and human images to a temporary folder for review
+
+review_folder_base = r'g:/temp/review_images'
+os.makedirs(review_folder_base,exist_ok=True)
+
+images_to_review = failed_images.union(candidate_human_images)
+images_to_review = sorted(list(images_to_review))
+
+source_file_to_target_file = {}
+
+# fn_relative = images_to_review[0]
+for fn_relative in images_to_review:
+    assert '\\' not in fn_relative
+    fn_abs_source = input_base_folder + '/' + fn_relative
+    assert os.path.isfile(fn_abs_source)
+    fn_abs_dest = review_folder_base + '/' + fn_relative.replace('/','_')
+    source_file_to_target_file[fn_abs_source] = fn_abs_dest
+
+from megadetector.utils.path_utils import parallel_copy_files
+
+parallel_copy_files(input_file_to_output_file=source_file_to_target_file,
+                    max_workers=16, 
+                    use_threads=True,
+                    overwrite=False,verbose=False)
+
+
+#%% Copy videos to a temporary folder for review
+
+review_folder_base = r'g:/temp/review_videos'
+os.makedirs(review_folder_base,exist_ok=True)
+
+source_file_to_target_file = {}
+
+# fn_relative = video_files[0]
+for fn_relative in video_files:
+    assert '\\' not in fn_relative
+    fn_abs_source = input_base_folder + '/' + fn_relative
+    assert os.path.isfile(fn_abs_source)
+    fn_abs_dest = review_folder_base + '/' + fn_relative.replace('/','_')
+    source_file_to_target_file[fn_abs_source] = fn_abs_dest
+
+from megadetector.utils.path_utils import parallel_copy_files
+
+parallel_copy_files(input_file_to_output_file=source_file_to_target_file,
+                    max_workers=16, 
+                    use_threads=True,
+                    overwrite=False,verbose=False)
+
+
+#%% Track removed images
+
+removed_images = [
+    "annotated-imgs\panthera leo\Camera Trap\Events\X73Okngwe\2013\02\PvL_seq_41468415-4518-44d6-acac-2113b442f723\PICT0190.JPG",
+    "annotated-imgs\panthera leo\Camera Trap\Hoanib\FldPln_Arch\211011\PvL_seq_5a9c6379-6980-4ab8-903a-b3bcba2ad21b\PICT0039.JPG",
+    "annotated-imgs\panthera leo\Camera Trap\Hoanib\FldPln_Arch\211011\PvL_seq_5a9c6379-6980-4ab8-903a-b3bcba2ad21b\PICT0037.JPG",
+    "annotated-imgs\panthera leo\Camera Trap\Hoanib\FldPln_Arch\211011\PvL_seq_5a9c6379-6980-4ab8-903a-b3bcba2ad21b\PICT0038.JPG",
+    "annotated-imgs\panthera leo\Camera Trap\2015\09\PvL_seq_da9c9ab1-74a2-485e-b6e7-3827b0c2a2f0\20150924-RCX_0835.JPG",
+    "annotated-imgs\panthera leo\Camera Trap\2015\09\PvL_seq_b0c1c6c5-474e-4844-a66c-e2bf5513d47a\20150924-RCX_0841.JPG",
+    "annotated-imgs\oryx gazella\Camera Trap\Video_Clips\Leylands\CDY_0003.AVI"
+]
+
+removed_images = [fn.replace('\\','/') for fn in removed_images]
+
+
+#%% Map filenames to datetimes
+
+filename_to_datetime = {}
+n_valid_datetimes = 0
+
+# im = exif_data[0]
+for im in tqdm(exif_data):
+    if im['exif_tags'] is None or len(im['exif_tags']) == 0:
+        filename_to_datetime[im['file_name']] = None
+        continue
+    dt = im['exif_tags']['DateTime']
+    assert len(dt) == 19
+    filename_to_datetime[im['file_name']] = dt
+    n_valid_datetimes += 1
+
+print('\nFound datetime information for {} of {} images'.format(
+    n_valid_datetimes,len(exif_data)))    
+
+
+#%% Convert "annotated_imgs" folder to COCO Camera Traps
+
+from megadetector.utils.path_utils import recursive_file_list
+
+species_name_to_category_id = {}
+
+filenames_relative = \
+    recursive_file_list(export_base,return_relative_paths=True,recursive=True,convert_slashes=True)
+    
+short_species_names = ['aves','cn-owls','cn-francolins','cn-raptors',
+                       'columbidae','equus zebra hartmannae','numididae',
+                       'pteroclidae']
+
+images = []
+annotations = []
+n_datetimes = 0
+
+for fn in filenames_relative:
+    
+    assert fn.lower().endswith('.jpg') or fn.lower().endswith('.avi') or fn.lower().endswith('.json')
+    
+    if fn.lower().endswith('.json'):
+        continue
+
+    tokens = fn.split('/')
+    species_name = tokens[0]
+    assert species_name in short_species_names or len(species_name.split(' ')) == 2
+    
+    if species_name not in species_name_to_category_id:
+        category_id = len(species_name_to_category_id)
+        species_name_to_category_id[species_name] = category_id
+    else:
+        category_id = species_name_to_category_id[species_name]
+                
+    im = {}
+    im['id'] = fn
+    im['file_name'] = fn
+    im['location'] = 'unknown'
+    
+    fn_for_datetime_lookup = 'annotated-imgs/' + fn
+    if (fn_for_datetime_lookup in filename_to_datetime) and \
+        (filename_to_datetime[fn_for_datetime_lookup] is not None):
+        im['datetime'] = filename_to_datetime[fn_for_datetime_lookup]
+        n_datetimes += 1
+
+    ann = {}
+    ann['image_id'] = im['id']
+    ann['id'] = im['id'] + ':ann_00'
+    ann['sequence_level_annotation'] = False
+    ann['category_id'] = category_id
+    
+    images.append(im)
+    annotations.append(ann)
+    
+# ...for each filename
+
+categories = []
+for species_name in species_name_to_category_id:
+    category = {}
+    category['name'] = species_name
+    category['id'] = species_name_to_category_id[species_name]
+    categories.append(category)
+
+info = {}
+info['version'] = '2024.07.15_00'
+info['description'] = 'Desert Lion Camera Traps'
+
+d = {}
+d['info'] = info
+d['images'] = images
+d['annotations'] = annotations
+d['categories'] = categories
+
+with open(output_file,'w') as f:
+    json.dump(d,f,indent=1)
+
+
+#%% Integrity check
+
+from megadetector.data_management.databases.integrity_check_json_db import \
+    IntegrityCheckOptions, integrity_check_json_db
+    
+integrity_check_options = IntegrityCheckOptions()
+
+integrity_check_options.baseDir = export_base
+integrity_check_options.bCheckImageExistence = True
+integrity_check_options.bRequireLocation = True
+integrity_check_options.nThreads = 10
+integrity_check_options.verbose = True
+integrity_check_options.allowIntIDs = False
+
+integrity_check_results = integrity_check_json_db(output_file,integrity_check_options)
+
+
+#%% Preview    
+
+from megadetector.visualization.visualize_db \
+    import DbVizOptions, visualize_db
+    
+viz_options = DbVizOptions()
+viz_options.num_to_visualize = 2500
+
+html_output_file,_ = visualize_db(output_file, preview_dir, export_base, options=viz_options)
+
+from megadetector.utils.path_utils import open_file
+open_file(html_output_file)
+
+
+#%% Make MD results paths line up with the output
+
+md_results_remapped_file = md_results_file.replace('-all','')
+assert md_results_remapped_file != md_results_file
+
+with open(output_file,'r') as f:
+    d = json.load(f)
+    
+image_filenames = [im['file_name'] for im in d['images']]
+image_filenames_set = set(image_filenames)
+
+with open(md_results_file,'r') as f:
+    md_results = json.load(f)
+
+md_results_images_remapped = []
+
+# im = md_results['images'][0]
+for im in md_results['images']:
+    assert im['file'].startswith('annotated-imgs/') or im['file'].startswith('bboxes/')
+    if im['file'].startswith('bboxes/'):
+        continue
+    im['file'] = im['file'].replace('annotated-imgs/','')
+    md_results_images_remapped.append(im)
+    
+print('Keeping {} of {} images in MD results'.format(
+    len(md_results_images_remapped),len(md_results['images'])))
+
+d['images'] = md_results_images_remapped
+
+with open(md_results_remapped_file,'w') as f:
+    json.dump(d,f,indent=1)
+
+
+#%% Zip MD results and COCO file
+
+from megadetector.utils.path_utils import zip_file
+
+zip_file(input_fn=md_results_remapped_file, output_fn=None, overwrite=True, verbose=True, compresslevel=9)
+zip_file(input_fn=output_file, output_fn=None, overwrite=True, verbose=True, compresslevel=9)
+
+
+#%% Zip images
+
+from megadetector.utils.path_utils import zip_folder
+
+zip_folder(input_folder=export_base, output_fn=output_zipfile, overwrite=True, verbose=True, compresslevel=0)
+
+
+#%% Copy lion images to a folder for thumbnail selection
+
+review_folder_base = r'g:/temp/thumbnail-candidates'
+os.makedirs(review_folder_base,exist_ok=True)
+
+source_file_to_target_file = {}
+
+# fn_relative = image_files[0]
+for fn_relative in image_files:
+    assert '\\' not in fn_relative
+    if '/lion/' not in fn_relative and '/panthera leo/' not in fn_relative:
+        continue
+    fn_abs_source = input_base_folder + '/' + fn_relative
+    assert os.path.isfile(fn_abs_source)
+    fn_abs_dest = review_folder_base + '/' + fn_relative.replace('/','_')
+    source_file_to_target_file[fn_abs_source] = fn_abs_dest
+
+from megadetector.utils.path_utils import parallel_copy_files
+
+parallel_copy_files(input_file_to_output_file=source_file_to_target_file,
+                    max_workers=16, 
+                    use_threads=True,
+                    overwrite=False,verbose=False)

megadetector/data_management/lila/generate_lila_per_image_labels.py

@@ -354,7 +354,7 @@ print('Processed {} datasets'.format(len(metadata_table)))
 #%% Read the .csv back
 
 df = pd.read_csv(output_file)
-print('Read {} lines from {}'.format(len(df),output_file))
+print('Read {} rows from {}'.format(len(df),output_file))
 
 
 #%% Do some post-hoc integrity checking
@@ -403,9 +403,9 @@ else:
         check_row(row)
 
 
-#%% Check for datasets that have only one location string
+#%% Check for datasets that have only one location string (typically "unknown")
 
-# Expected: ENA24, Missouri Camera Traps
+# Expected: ENA24, Missouri Camera Traps, Desert Lion Conservation Camera Traps
 
 for ds_name in dataset_name_to_locations.keys():
     if len(dataset_name_to_locations[ds_name]) == 1:

megadetector/data_management/lila/test_lila_metadata_urls.py

@@ -72,8 +72,8 @@ for ds_name in metadata_table.keys():
 url_to_source = {}
 
 # The first image in a dataset is disproportionately likely to be human (and thus 404),
-# so we pick a semi-arbitrary image that isn't the first.  How about the 1000th?
-image_index = 1000
+# so we pick a semi-arbitrary image that isn't the first.  How about the 2000th?
+image_index = 2000
 
 # ds_name = list(metadata_table.keys())[0]
 for ds_name in metadata_table.keys():

megadetector/data_management/remove_exif.py

@@ -3,64 +3,89 @@
 remove_exif.py
 
 Removes all EXIF/IPTC/XMP metadata from a folder of images, without making 
-backup copies, using pyexiv2.
+backup copies, using pyexiv2.  Ignores non-jpeg images.
 
-TODO: This is a one-off script waiting to be cleaned up for more general use.
+This module is rarely used, and pyexiv2 is not thread-safe, so pyexiv2 is not
+included in package-level dependency lists.  YMMV.
 
 """
 
-input_base = r'f:\images'
-
-
 #%% Imports and constants
 
 import os
 import glob
 
-def main():
-
-    assert os.path.isdir(input_base)
+from multiprocessing.pool import Pool as Pool
+from tqdm import tqdm
 
-    ##%% List files
 
-    all_files = [f for f in glob.glob(input_base + "*/**", recursive=True)]
-    image_files = [s for s in all_files if (s.lower().endswith('.jpg'))]
-        
+#%% Support functions
 
-    ##%% Remove EXIF data (support)
+# Pyexif2 is not thread safe, do not call this function in parallel within a process
+#
+# Parallelizing across processes is fine.
+def remove_exif_from_image(fn):
 
     import pyexiv2
+    
+    try:
+        img = pyexiv2.Image(fn)
+        img.clear_exif()
+        img.clear_iptc()
+        img.clear_xmp()
+        img.close()        
+    except Exception as e:
+        print('EXIF error on {}: {}'.format(fn,str(e)))
+    
+    return True
+
+
+#%% Remove EXIF data
+
+def remove_exif(image_base_folder,recursive=True,n_processes=1):
+    """
+    Removes all EXIF/IPTC/XMP metadata from a folder of images, without making 
+    backup copies, using pyexiv2.  Ignores non-jpeg images.
+    
+    Args:
+        image_base_folder (str): the folder from which we should remove EXIF data
+        recursive (bool, optional): whether to process [image_base_folder] recursively
+        n_processes (int, optional): number of concurrent workers.  Because pyexiv2 is not
+            thread-safe, only process-based parallelism is supported.        
+    """
+    try:
+        import pyexiv2 #noqa
+    except:
+        print('pyexiv2 not available; try "pip install pyexiv2"')
+        raise
 
-    # PYEXIV2 IS NOT THREAD SAFE; DO NOT CALL THIS IN PARALLEL FROM A SINGLE PROCESS
-    def remove_exif(fn):
         
-        try:
-            img = pyexiv2.Image(fn)
-            # data = img.read_exif(); print(data)
-            img.clear_exif()
-            img.clear_iptc()
-            img.clear_xmp()
-            img.close()        
-        except Exception as e:
-            print('EXIF error on {}: {}'.format(fn,str(e)))
+    ##%% List files
+
+    assert os.path.isdir(image_base_folder), \
+        'Could not find folder {}'.format(image_base_folder)
+    all_files = [f for f in glob.glob(image_base_folder+ "*/**", recursive=recursive)]
+    image_files = [s for s in all_files if \
+                   (s.lower().endswith('.jpg') or s.lower().endswith('.jpeg'))]
         
 
     ##%% Remove EXIF data (execution)
 
-    from joblib import Parallel, delayed
-
-    n_exif_threads = 50
-        
-    if n_exif_threads == 1:
+    if n_processes == 1:
         
         # fn = image_files[0]
-        for fn in image_files:
-            remove_exif(fn)
+        for fn in tqdm(image_files):
+            remove_exif_from_image(fn)
             
     else:
-        # joblib.Parallel defaults to a process-based backend, but let's be sure
-        # results = Parallel(n_jobs=n_exif_threads,verbose=2,prefer='processes')(delayed(remove_exif)(fn) for fn in image_files[0:10])
-        _ = Parallel(n_jobs=n_exif_threads,verbose=2,prefer='processes')(delayed(remove_exif)(fn) for fn in image_files)
+        # pyexiv2 is not thread-safe, so we need to use processes
+        print('Starting parallel process pool with {} workers'.format(n_processes))
+        pool = Pool(n_processes)
+        _ = list(tqdm(pool.imap(remove_exif_from_image,image_files),total=len(image_files)))
             
-if __name__ == '__main__':
-    main()
+# ...remove_exif(...)
+
+
+#%% Command-line driver
+
+## TODO

megadetector/data_management/yolo_to_coco.py

@@ -29,9 +29,9 @@ from megadetector.data_management.yolo_output_to_md_output import read_classes_f
 
 def _filename_to_image_id(fn):
     """
-    Image IDs can't have spaces in them, replae spaces with underscores
+    Image IDs can't have spaces in them, replace spaces with underscores
     """
-    return fn.replace(' ','_')
+    return fn.replace(' ','_').replace('\\','/')
 
 
 def _process_image(fn_abs,input_folder,category_id_to_name):
@@ -40,7 +40,9 @@ def _process_image(fn_abs,input_folder,category_id_to_name):
     """
     
     # Create the image object for this image
-    fn_relative = os.path.relpath(fn_abs,input_folder)
+    #
+    # Always use forward slashes in image filenames and IDs
+    fn_relative = os.path.relpath(fn_abs,input_folder).replace('\\','/')
     image_id = _filename_to_image_id(fn_relative)
     
     # This is done in a separate loop now
@@ -51,7 +53,7 @@ def _process_image(fn_abs,input_folder,category_id_to_name):
     # image_ids.add(image_id)
     
     im = {}
-    im['file_name'] = fn_relative    
+    im['file_name'] = fn_relative
     im['id'] = image_id
     
     annotations_this_image = []
@@ -393,7 +395,8 @@ def yolo_to_coco(input_folder,
                  pool_type='thread',
                  recursive=True,
                  exclude_string=None,
-                 include_string=None):
+                 include_string=None,
+                 overwrite_handling='overwrite'):
     """
     Converts a YOLO-formatted dataset to a COCO-formatted dataset.
     
@@ -427,6 +430,8 @@ def yolo_to_coco(input_folder,
         recursive (bool, optional): whether to recurse into [input_folder]
         exclude_string (str, optional): exclude any images whose filename contains a string
         include_string (str, optional): include only images whose filename contains a string
+        overwrite_handling (bool, optional): behavior if output_file exists ('load', 'overwrite', or 
+            'error')
     
     Returns:
         dict: COCO-formatted data, the same as what's written to [output_file]
@@ -441,7 +446,21 @@ def yolo_to_coco(input_folder,
         ('no_annotations','empty_annotations','skip','error'), \
             'Unrecognized empty image handling spec: {}'.format(empty_image_handling)
      
-            
+    if (output_file is not None) and os.path.isfile(output_file):
+        
+            if overwrite_handling == 'overwrite':
+                print('Warning: output file {} exists, over-writing'.format(output_file))
+            elif overwrite_handling == 'load':
+                print('Output file {} exists, loading and returning'.format(output_file))
+                with open(output_file,'r') as f:
+                    d = json.load(f)
+                return d
+            elif overwrite_handling == 'error':
+                raise ValueError('Output file {} exists'.format(output_file))
+            else:
+                raise ValueError('Unrecognized overwrite_handling value: {}'.format(overwrite_handling))
+                
+                
     ## Read class names
     
     category_id_to_name = load_yolo_class_list(class_name_file)