megadetector 5.0.15__py3-none-any.whl → 5.0.17__py3-none-any.whl

megadetector/data_management/importers/import_desert_lion_conservation_camera_traps.py

@@ -0,0 +1,387 @@
+"""
+
+import_desert_lion_conservation_camera_traps.py
+
+Prepare the Desert Lion Conservation Camera Traps dataset for release on LILA.
+
+"""
+
+#%% Imports and constants
+
+import os
+import json
+
+input_base_folder = r'i:/data/desert-lion'
+assert os.path.isdir(input_base_folder)
+
+# md_results_file = r'i:/data/desert-lion/desert-lion-camera-traps-2024-07-14-v5a.0.0_detections-all.json'
+md_results_file = r'i:/data/desert-lion/desert-lion-camera-traps-2024-07-14-v5a.0.0_detections.json'
+assert os.path.isfile(md_results_file)
+
+export_base = os.path.join(input_base_folder,'annotated-imgs')
+assert os.path.isdir(export_base)
+
+preview_dir = r'g:\temp\desert-lion-viz'
+output_file = os.path.join(input_base_folder,'desert_lion_camera_traps.json')
+output_zipfile = os.path.join(input_base_folder,'desert-lion-camera-traps-images.zip')
+
+exif_cache_file_post_exif_removal = os.path.join(input_base_folder,'exif_data_post_exif_removal.json')
+exif_cache_file = os.path.join(input_base_folder,'exif_data.json')
+
+
+#%% Find images and videos
+
+from megadetector.detection.video_utils import find_videos
+from megadetector.utils.path_utils import find_images
+
+video_files = find_videos(input_base_folder,recursive=True,return_relative_paths=True,convert_slashes=True)
+image_files = find_images(input_base_folder,recursive=True,return_relative_paths=True,convert_slashes=True)
+
+n_annotated_imgs = len([fn for fn in image_files if 'annotated-imgs' in fn])
+print('Found {} images ({} in the annotated-imgs folder), {} videos'.format(
+    len(image_files),n_annotated_imgs,len(video_files)))
+
+
+#%% Read EXIF data
+
+from megadetector.data_management.read_exif import read_exif_from_folder, ReadExifOptions
+
+exif_options = ReadExifOptions()
+exif_options.n_workers = 10
+
+if os.path.isfile(exif_cache_file):
+    print('EXIF cache {} exists, skipping EXIF read'.format(exif_cache_file))
+    with open(exif_cache_file,'r') as f:
+        exif_data = json.load(f)    
+else:
+    exif_data = read_exif_from_folder(input_folder=input_base_folder,
+                                      output_file=exif_cache_file,
+                                      options=exif_options,
+                                      filenames=None,
+                                      recursive=True)
+
+assert len(exif_data) == len(image_files)
+
+
+#%% Remove EXIF data
+
+from megadetector.data_management.remove_exif import remove_exif
+remove_exif(input_base_folder,recursive=True,n_processes=1)
+
+
+#%% Read EXIF data again
+
+exif_data_post_exif_removal = read_exif_from_folder(input_folder=input_base_folder,
+                                                    output_file=exif_cache_file_post_exif_removal,
+                                                    options=exif_options,
+                                                    filenames=None,
+                                                    recursive=True)
+
+
+#%% Make sure no lat/lon data is present
+
+from tqdm import tqdm
+
+for i_image,im in enumerate(tqdm(exif_data_post_exif_removal)):
+    tags = im['exif_tags']
+    if tags is None:
+        continue
+    for k in tags:
+        assert 'gps' not in str(k).lower()
+
+
+#%% Look for images that contain humans
+
+with open(md_results_file,'r') as f:
+    md_results = json.load(f)
+
+assert len(md_results['images']) == len(image_files)
+
+human_threshold = 0.1
+human_categories = ['2','3']
+
+candidate_human_images = set()
+failed_images = set()
+
+# i_image = 0; im = md_results['images'][0]
+for i_image,im in tqdm(enumerate(md_results['images']),total=len(md_results['images'])):
+
+    if 'failure' in im:
+        failed_images.add(im['file'])
+        continue
+    
+    for det in im['detections']:
+        if det['category'] in human_categories and det['conf'] >= human_threshold:
+            candidate_human_images.add(im['file'])
+            break
+        
+    # ...for each detection
+    
+# ...for each image    
+
+print('Found {} failed images and {} candidate human images'.format(
+    len(failed_images),len(candidate_human_images)))
+
+
+#%% Copy failed images and human images to a temporary folder for review
+
+review_folder_base = r'g:/temp/review_images'
+os.makedirs(review_folder_base,exist_ok=True)
+
+images_to_review = failed_images.union(candidate_human_images)
+images_to_review = sorted(list(images_to_review))
+
+source_file_to_target_file = {}
+
+# fn_relative = images_to_review[0]
+for fn_relative in images_to_review:
+    assert '\\' not in fn_relative
+    fn_abs_source = input_base_folder + '/' + fn_relative
+    assert os.path.isfile(fn_abs_source)
+    fn_abs_dest = review_folder_base + '/' + fn_relative.replace('/','_')
+    source_file_to_target_file[fn_abs_source] = fn_abs_dest
+
+from megadetector.utils.path_utils import parallel_copy_files
+
+parallel_copy_files(input_file_to_output_file=source_file_to_target_file,
+                    max_workers=16, 
+                    use_threads=True,
+                    overwrite=False,verbose=False)
+
+
+#%% Copy videos to a temporary folder for review
+
+review_folder_base = r'g:/temp/review_videos'
+os.makedirs(review_folder_base,exist_ok=True)
+
+source_file_to_target_file = {}
+
+# fn_relative = video_files[0]
+for fn_relative in video_files:
+    assert '\\' not in fn_relative
+    fn_abs_source = input_base_folder + '/' + fn_relative
+    assert os.path.isfile(fn_abs_source)
+    fn_abs_dest = review_folder_base + '/' + fn_relative.replace('/','_')
+    source_file_to_target_file[fn_abs_source] = fn_abs_dest
+
+from megadetector.utils.path_utils import parallel_copy_files
+
+parallel_copy_files(input_file_to_output_file=source_file_to_target_file,
+                    max_workers=16, 
+                    use_threads=True,
+                    overwrite=False,verbose=False)
+
+
+#%% Track removed images
+
+removed_images = [
+    "annotated-imgs\panthera leo\Camera Trap\Events\X73Okngwe\2013\02\PvL_seq_41468415-4518-44d6-acac-2113b442f723\PICT0190.JPG",
+    "annotated-imgs\panthera leo\Camera Trap\Hoanib\FldPln_Arch\211011\PvL_seq_5a9c6379-6980-4ab8-903a-b3bcba2ad21b\PICT0039.JPG",
+    "annotated-imgs\panthera leo\Camera Trap\Hoanib\FldPln_Arch\211011\PvL_seq_5a9c6379-6980-4ab8-903a-b3bcba2ad21b\PICT0037.JPG",
+    "annotated-imgs\panthera leo\Camera Trap\Hoanib\FldPln_Arch\211011\PvL_seq_5a9c6379-6980-4ab8-903a-b3bcba2ad21b\PICT0038.JPG",
+    "annotated-imgs\panthera leo\Camera Trap\2015\09\PvL_seq_da9c9ab1-74a2-485e-b6e7-3827b0c2a2f0\20150924-RCX_0835.JPG",
+    "annotated-imgs\panthera leo\Camera Trap\2015\09\PvL_seq_b0c1c6c5-474e-4844-a66c-e2bf5513d47a\20150924-RCX_0841.JPG",
+    "annotated-imgs\oryx gazella\Camera Trap\Video_Clips\Leylands\CDY_0003.AVI"
+]
+
+removed_images = [fn.replace('\\','/') for fn in removed_images]
+
+
+#%% Map filenames to datetimes
+
+filename_to_datetime = {}
+n_valid_datetimes = 0
+
+# im = exif_data[0]
+for im in tqdm(exif_data):
+    if im['exif_tags'] is None or len(im['exif_tags']) == 0:
+        filename_to_datetime[im['file_name']] = None
+        continue
+    dt = im['exif_tags']['DateTime']
+    assert len(dt) == 19
+    filename_to_datetime[im['file_name']] = dt
+    n_valid_datetimes += 1
+
+print('\nFound datetime information for {} of {} images'.format(
+    n_valid_datetimes,len(exif_data)))    
+
+
+#%% Convert "annotated_imgs" folder to COCO Camera Traps
+
+from megadetector.utils.path_utils import recursive_file_list
+
+species_name_to_category_id = {}
+
+filenames_relative = \
+    recursive_file_list(export_base,return_relative_paths=True,recursive=True,convert_slashes=True)
+    
+short_species_names = ['aves','cn-owls','cn-francolins','cn-raptors',
+                       'columbidae','equus zebra hartmannae','numididae',
+                       'pteroclidae']
+
+images = []
+annotations = []
+n_datetimes = 0
+
+for fn in filenames_relative:
+    
+    assert fn.lower().endswith('.jpg') or fn.lower().endswith('.avi') or fn.lower().endswith('.json')
+    
+    if fn.lower().endswith('.json'):
+        continue
+
+    tokens = fn.split('/')
+    species_name = tokens[0]
+    assert species_name in short_species_names or len(species_name.split(' ')) == 2
+    
+    if species_name not in species_name_to_category_id:
+        category_id = len(species_name_to_category_id)
+        species_name_to_category_id[species_name] = category_id
+    else:
+        category_id = species_name_to_category_id[species_name]
+                
+    im = {}
+    im['id'] = fn
+    im['file_name'] = fn
+    im['location'] = 'unknown'
+    
+    fn_for_datetime_lookup = 'annotated-imgs/' + fn
+    if (fn_for_datetime_lookup in filename_to_datetime) and \
+        (filename_to_datetime[fn_for_datetime_lookup] is not None):
+        im['datetime'] = filename_to_datetime[fn_for_datetime_lookup]
+        n_datetimes += 1
+
+    ann = {}
+    ann['image_id'] = im['id']
+    ann['id'] = im['id'] + ':ann_00'
+    ann['sequence_level_annotation'] = False
+    ann['category_id'] = category_id
+    
+    images.append(im)
+    annotations.append(ann)
+    
+# ...for each filename
+
+categories = []
+for species_name in species_name_to_category_id:
+    category = {}
+    category['name'] = species_name
+    category['id'] = species_name_to_category_id[species_name]
+    categories.append(category)
+
+info = {}
+info['version'] = '2024.07.15_00'
+info['description'] = 'Desert Lion Camera Traps'
+
+d = {}
+d['info'] = info
+d['images'] = images
+d['annotations'] = annotations
+d['categories'] = categories
+
+with open(output_file,'w') as f:
+    json.dump(d,f,indent=1)
+
+
+#%% Integrity check
+
+from megadetector.data_management.databases.integrity_check_json_db import \
+    IntegrityCheckOptions, integrity_check_json_db
+    
+integrity_check_options = IntegrityCheckOptions()
+
+integrity_check_options.baseDir = export_base
+integrity_check_options.bCheckImageExistence = True
+integrity_check_options.bRequireLocation = True
+integrity_check_options.nThreads = 10
+integrity_check_options.verbose = True
+integrity_check_options.allowIntIDs = False
+
+integrity_check_results = integrity_check_json_db(output_file,integrity_check_options)
+
+
+#%% Preview    
+
+from megadetector.visualization.visualize_db \
+    import DbVizOptions, visualize_db
+    
+viz_options = DbVizOptions()
+viz_options.num_to_visualize = 2500
+
+html_output_file,_ = visualize_db(output_file, preview_dir, export_base, options=viz_options)
+
+from megadetector.utils.path_utils import open_file
+open_file(html_output_file)
+
+
+#%% Make MD results paths line up with the output
+
+md_results_remapped_file = md_results_file.replace('-all','')
+assert md_results_remapped_file != md_results_file
+
+with open(output_file,'r') as f:
+    d = json.load(f)
+    
+image_filenames = [im['file_name'] for im in d['images']]
+image_filenames_set = set(image_filenames)
+
+with open(md_results_file,'r') as f:
+    md_results = json.load(f)
+
+md_results_images_remapped = []
+
+# im = md_results['images'][0]
+for im in md_results['images']:
+    assert im['file'].startswith('annotated-imgs/') or im['file'].startswith('bboxes/')
+    if im['file'].startswith('bboxes/'):
+        continue
+    im['file'] = im['file'].replace('annotated-imgs/','')
+    md_results_images_remapped.append(im)
+    
+print('Keeping {} of {} images in MD results'.format(
+    len(md_results_images_remapped),len(md_results['images'])))
+
+d['images'] = md_results_images_remapped
+
+with open(md_results_remapped_file,'w') as f:
+    json.dump(d,f,indent=1)
+
+
+#%% Zip MD results and COCO file
+
+from megadetector.utils.path_utils import zip_file
+
+zip_file(input_fn=md_results_remapped_file, output_fn=None, overwrite=True, verbose=True, compresslevel=9)
+zip_file(input_fn=output_file, output_fn=None, overwrite=True, verbose=True, compresslevel=9)
+
+
+#%% Zip images
+
+from megadetector.utils.path_utils import zip_folder
+
+zip_folder(input_folder=export_base, output_fn=output_zipfile, overwrite=True, verbose=True, compresslevel=0)
+
+
+#%% Copy lion images to a folder for thumbnail selection
+
+review_folder_base = r'g:/temp/thumbnail-candidates'
+os.makedirs(review_folder_base,exist_ok=True)
+
+source_file_to_target_file = {}
+
+# fn_relative = image_files[0]
+for fn_relative in image_files:
+    assert '\\' not in fn_relative
+    if '/lion/' not in fn_relative and '/panthera leo/' not in fn_relative:
+        continue
+    fn_abs_source = input_base_folder + '/' + fn_relative
+    assert os.path.isfile(fn_abs_source)
+    fn_abs_dest = review_folder_base + '/' + fn_relative.replace('/','_')
+    source_file_to_target_file[fn_abs_source] = fn_abs_dest
+
+from megadetector.utils.path_utils import parallel_copy_files
+
+parallel_copy_files(input_file_to_output_file=source_file_to_target_file,
+                    max_workers=16, 
+                    use_threads=True,
+                    overwrite=False,verbose=False)

megadetector/data_management/importers/snapshot_safari_importer_reprise.py

@@ -24,7 +24,7 @@ from collections import defaultdict
 
 from megadetector.utils import path_utils
 
-input_base = '/media/user/Elements'
+input_base = 'e:/'
 output_base = os.path.expanduser('~/data/snapshot-safari-metadata')
 file_list_cache_file = os.path.join(output_base,'file_list.json')
 
@@ -76,23 +76,16 @@ print('Found a total of {} files, {} of which are images'.format(
     len(all_files_relative),len(all_image_files)))
 
 
-#%% Copy all csv files to the annotation cache folder
+#%% Copy all .csv files to the annotation cache folder
 
 # fn = csv_files[0]
-for fn in csv_files:
+for fn in tqdm(csv_files):
+    
     target_file = os.path.join(annotation_cache_dir,os.path.basename(fn))
     source_file = os.path.join(input_base,fn)
     shutil.copyfile(source_file,target_file)    
 
-def read_cached_csv_file(fn):
-    """
-    Later cells will ask to read a .csv file from the original hard drive;
-    read from the annotation cache instead.
-    """
-    
-    cached_csv_file = os.path.join(annotation_cache_dir,os.path.basename(fn))
-    df = pd.read_csv(cached_csv_file)
-    return df
+print('Copied {} .csv files to cache folder'.format(len(csv_files)))
 
 
 #%% List project folders
@@ -123,6 +116,21 @@ project_folder_to_project_code = {v: k for k, v in project_code_to_project_folde
 project_codes = sorted(list(project_code_to_project_folder.keys()))
 project_folders = sorted(list(project_code_to_project_folder.values()))
 
+print('Eumerated {} project folders'.format(len(project_folders)))
+
+
+#%% Support functions
+
+def read_cached_csv_file(fn):
+    """
+    Later cells will ask to read a .csv file from the original hard drive;
+    read from the annotation cache instead.
+    """
+    
+    cached_csv_file = os.path.join(annotation_cache_dir,os.path.basename(fn))
+    df = pd.read_csv(cached_csv_file)
+    return df
+
 def file_to_project_folder(fn):
     """
     For a given filename relative to the drive root, return the corresponding
@@ -138,7 +146,6 @@ def file_to_project_folder(fn):
     assert project_folder in project_folders
     return project_folder
 
-
 def file_to_project_code(fn):
     """
     For a given filename relative to the drive root, return the corresponding
@@ -147,6 +154,9 @@ def file_to_project_code(fn):
     
     return project_folder_to_project_code[file_to_project_folder(fn)]
 
+
+#%% Consistency checking
+
 assert file_to_project_folder(
     'APN/APN_S2/DW/DW_R5/APN_S2_DW_R5_IMAG0003.JPG') == 'APN'
 assert file_to_project_folder(
@@ -163,9 +173,11 @@ assert file_to_project_code(
 #
 # E.g.:
 #
-# 'DHP': ['Snapshot South Africa/DHP/LILA_Reports/DHP_S1_report_lila.csv',
-# 'Snapshot South Africa/DHP/LILA_Reports/DHP_S2_report_lila.csv',
-# 'Snapshot South Africa/DHP/LILA_Reports/DHP_S3_report_lila.csv']
+# 'DHP': [
+#        'Snapshot South Africa/DHP/LILA_Reports/DHP_S1_report_lila.csv',
+#        'Snapshot South Africa/DHP/LILA_Reports/DHP_S2_report_lila.csv',
+#        'Snapshot South Africa/DHP/LILA_Reports/DHP_S3_report_lila.csv'
+#       ]
 #
 project_code_to_report_files = defaultdict(list)
 

megadetector/data_management/lila/generate_lila_per_image_labels.py

@@ -354,7 +354,7 @@ print('Processed {} datasets'.format(len(metadata_table)))
 #%% Read the .csv back
 
 df = pd.read_csv(output_file)
-print('Read {} lines from {}'.format(len(df),output_file))
+print('Read {} rows from {}'.format(len(df),output_file))
 
 
 #%% Do some post-hoc integrity checking
@@ -403,9 +403,9 @@ else:
         check_row(row)
 
 
-#%% Check for datasets that have only one location string
+#%% Check for datasets that have only one location string (typically "unknown")
 
-# Expected: ENA24, Missouri Camera Traps
+# Expected: ENA24, Missouri Camera Traps, Desert Lion Conservation Camera Traps
 
 for ds_name in dataset_name_to_locations.keys():
     if len(dataset_name_to_locations[ds_name]) == 1:

megadetector/data_management/lila/test_lila_metadata_urls.py

@@ -72,8 +72,8 @@ for ds_name in metadata_table.keys():
 url_to_source = {}
 
 # The first image in a dataset is disproportionately likely to be human (and thus 404),
-# so we pick a semi-arbitrary image that isn't the first.  How about the 1000th?
-image_index = 1000
+# so we pick a semi-arbitrary image that isn't the first.  How about the 2000th?
+image_index = 2000
 
 # ds_name = list(metadata_table.keys())[0]
 for ds_name in metadata_table.keys():

megadetector/data_management/remove_exif.py

@@ -3,64 +3,89 @@
 remove_exif.py
 
 Removes all EXIF/IPTC/XMP metadata from a folder of images, without making 
-backup copies, using pyexiv2.
+backup copies, using pyexiv2.  Ignores non-jpeg images.
 
-TODO: This is a one-off script waiting to be cleaned up for more general use.
+This module is rarely used, and pyexiv2 is not thread-safe, so pyexiv2 is not
+included in package-level dependency lists.  YMMV.
 
 """
 
-input_base = r'f:\images'
-
-
 #%% Imports and constants
 
 import os
 import glob
 
-def main():
-
-    assert os.path.isdir(input_base)
+from multiprocessing.pool import Pool as Pool
+from tqdm import tqdm
 
-    ##%% List files
 
-    all_files = [f for f in glob.glob(input_base + "*/**", recursive=True)]
-    image_files = [s for s in all_files if (s.lower().endswith('.jpg'))]
-        
+#%% Support functions
 
-    ##%% Remove EXIF data (support)
+# Pyexif2 is not thread safe, do not call this function in parallel within a process
+#
+# Parallelizing across processes is fine.
+def remove_exif_from_image(fn):
 
     import pyexiv2
+    
+    try:
+        img = pyexiv2.Image(fn)
+        img.clear_exif()
+        img.clear_iptc()
+        img.clear_xmp()
+        img.close()        
+    except Exception as e:
+        print('EXIF error on {}: {}'.format(fn,str(e)))
+    
+    return True
+
+
+#%% Remove EXIF data
+
+def remove_exif(image_base_folder,recursive=True,n_processes=1):
+    """
+    Removes all EXIF/IPTC/XMP metadata from a folder of images, without making 
+    backup copies, using pyexiv2.  Ignores non-jpeg images.
+    
+    Args:
+        image_base_folder (str): the folder from which we should remove EXIF data
+        recursive (bool, optional): whether to process [image_base_folder] recursively
+        n_processes (int, optional): number of concurrent workers.  Because pyexiv2 is not
+            thread-safe, only process-based parallelism is supported.        
+    """
+    try:
+        import pyexiv2 #noqa
+    except:
+        print('pyexiv2 not available; try "pip install pyexiv2"')
+        raise
 
-    # PYEXIV2 IS NOT THREAD SAFE; DO NOT CALL THIS IN PARALLEL FROM A SINGLE PROCESS
-    def remove_exif(fn):
         
-        try:
-            img = pyexiv2.Image(fn)
-            # data = img.read_exif(); print(data)
-            img.clear_exif()
-            img.clear_iptc()
-            img.clear_xmp()
-            img.close()        
-        except Exception as e:
-            print('EXIF error on {}: {}'.format(fn,str(e)))
+    ##%% List files
+
+    assert os.path.isdir(image_base_folder), \
+        'Could not find folder {}'.format(image_base_folder)
+    all_files = [f for f in glob.glob(image_base_folder+ "*/**", recursive=recursive)]
+    image_files = [s for s in all_files if \
+                   (s.lower().endswith('.jpg') or s.lower().endswith('.jpeg'))]
         
 
     ##%% Remove EXIF data (execution)
 
-    from joblib import Parallel, delayed
-
-    n_exif_threads = 50
-        
-    if n_exif_threads == 1:
+    if n_processes == 1:
         
         # fn = image_files[0]
-        for fn in image_files:
-            remove_exif(fn)
+        for fn in tqdm(image_files):
+            remove_exif_from_image(fn)
             
     else:
-        # joblib.Parallel defaults to a process-based backend, but let's be sure
-        # results = Parallel(n_jobs=n_exif_threads,verbose=2,prefer='processes')(delayed(remove_exif)(fn) for fn in image_files[0:10])
-        _ = Parallel(n_jobs=n_exif_threads,verbose=2,prefer='processes')(delayed(remove_exif)(fn) for fn in image_files)
+        # pyexiv2 is not thread-safe, so we need to use processes
+        print('Starting parallel process pool with {} workers'.format(n_processes))
+        pool = Pool(n_processes)
+        _ = list(tqdm(pool.imap(remove_exif_from_image,image_files),total=len(image_files)))
             
-if __name__ == '__main__':
-    main()
+# ...remove_exif(...)
+
+
+#%% Command-line driver
+
+## TODO