megadetector 5.0.11__py3-none-any.whl → 5.0.12__py3-none-any.whl

megadetector/data_management/importers/snapshotserengeti/make_full_SS_json.py

@@ -0,0 +1,150 @@
+#
+# make_full_SS_json.py
+#
+# Create a COCO-camera-traps .json file for Snapshot Serengeti data from
+# the original .csv files provided on Dryad.
+#
+# This was used for "version 1.0" of the public Snapshot Serengeti archive; it's no 
+# longer used as of v2.0 (early 2020).  See snapshot_serengeti_lila.py for the updated
+# Snapshot Safari preparation process.
+#
+
+#%% Imports and constants
+
+import csv
+import json
+import uuid
+import datetime
+
+output_file = '/datadrive/snapshotserengeti/databases/SnapshotSerengeti_multiple_classes.json'
+csv_file_name = '/datadrive/snapshotserengeti/databases/consensus_data.csv'
+all_image_file = '/datadrive/snapshotserengeti/databases/all_images.csv'
+
+
+#%% Read annotation .csv file, format into a dictionary mapping field names to data arrays
+
+data = []
+with open(csv_file_name,'r') as f:
+    reader = csv.reader(f, dialect = 'excel')
+    for row in reader:
+        data.append(row)
+
+data_fields = data[0]
+
+data_dicts = {}
+for event in data[1:]:
+    if event[0] not in data_dicts:
+        data_dicts[event[0]] = []
+    data_dicts[event[0]].append({data_fields[i]:event[i] for i in range(len(data_fields))})
+
+# Count the number of images with multiple species
+mult_species = 0
+for event in data_dicts:
+    if len(data_dicts[event]) > 1:
+        mult_species += 1
+
+
+#%% Read image .csv file, format into a dictionary mapping images to capture events
+
+with open(all_image_file,'r') as f:
+    reader = csv.reader(f,dialect = 'excel')
+    next(reader)
+    im_name_to_cap_id = {row[1]:row[0] for row in reader}
+    
+total_ims = len(im_name_to_cap_id)
+total_seqs = len(data_dicts)
+print('Percent seqs with mult species: ',float(mult_species)/float(total_seqs))
+
+
+#%% Create CCT-style .json 
+
+images = []
+annotations = []
+categories = []
+
+capture_ims = {i:[] for i in im_name_to_cap_id.values()}
+for im_id in im_name_to_cap_id:
+    capture_ims[im_name_to_cap_id[im_id]].append(im_id)
+
+im_to_seq_num = {im:None for im in im_name_to_cap_id}
+for event in capture_ims:
+    capture_ims[event] = sorted(capture_ims[event])
+    seq_count = 0
+    for im in capture_ims[event]:
+        im_to_seq_num[im] = seq_count
+        seq_count += 1
+
+cat_to_id = {}
+cat_to_id['empty'] = 0
+cat_count = 1
+seasons = []
+
+for im_id in im_name_to_cap_id:
+    
+    im = {}
+    im['id'] = im_id.split('.')[0]
+    im['file_name'] = im_id
+    
+    im['location'] = im_id.split('/')[1]
+    im['season'] = im_id.split('/')[0]
+    if im['season'] not in seasons:
+        seasons.append(im['season'])
+    im['seq_id'] = im_name_to_cap_id[im_id]
+    im['frame_num'] = im_to_seq_num[im_id]
+    im['seq_num_frames'] = len(capture_ims[im['seq_id']])
+
+    ann = {}
+    ann['id'] = str(uuid.uuid1())
+    ann['image_id'] = im['id']
+
+    if im_name_to_cap_id[im_id] in data_dicts:
+        im_data_per_ann = data_dicts[im_name_to_cap_id[im_id]]
+        for im_data in im_data_per_ann:
+            im['datetime'] = im_data['DateTime']
+            if im_data['Species'] not in cat_to_id:
+                cat_to_id[im_data['Species']] = cat_count
+                cat_count += 1
+            ann = {}
+            ann['id'] = str(uuid.uuid1())
+            ann['image_id'] = im['id']
+            ann['category_id'] = cat_to_id[im_data['Species']]
+            annotations.append(ann)
+    else:
+        ann = {}
+        ann['id'] = str(uuid.uuid1())
+        ann['image_id'] = im['id']
+        ann['category_id'] = 0
+        annotations.append(ann)
+            
+    # still need image width and height
+    images.append(im)
+    
+# ...for each image
+    
+print(seasons)
+
+for cat in cat_to_id:
+    new_cat = {}
+    new_cat['id'] = cat_to_id[cat]
+    new_cat['name'] = cat
+    categories.append(new_cat) 
+
+
+#%% Write output files
+    
+json_data = {}
+json_data['images'] = images
+json_data['annotations'] = annotations
+json_data['categories'] = categories
+info = {}
+info['year'] = 2018
+info['version'] = 1
+info['description'] = 'COCO style Snapshot Serengeti database'
+info['contributor'] = 'SMB'
+info['date_created'] = str(datetime.date.today())
+json_data['info'] = info
+json.dump(json_data,open(output_file,'w'))
+
+print(images[0])
+print(annotations[0]) 
+

megadetector/data_management/importers/snapshotserengeti/make_per_season_SS_json.py

@@ -0,0 +1,153 @@
+#
+# make_per_season_SS_json.py
+#
+# Create a COCO-camera-traps .json file for each Snapshot Serengeti season from 
+# the original .csv files provided on Dryad.
+#
+# This was used for "version 1.0" of the public Snapshot Serengeti archive; it's no 
+# longer used as of v2.0 (early 2020).  See snapshot_serengeti_lila.py for the updated
+# Snapshot Safari preparation process.
+#
+
+#%% Imports and constants
+
+import csv
+import json
+import uuid
+import datetime
+
+output_file_folder = 'C:/Users/t-sabeer/Documents/databases/'
+csv_file_name = 'D:/consensus_data.csv'
+
+
+#%% Read annotation .csv file, format into a dictionary mapping field names to data arrays
+
+data = []
+with open(csv_file_name,'r') as f:
+    reader = csv.reader(f, dialect = 'excel')
+    for row in reader:
+        data.append(row)
+
+data_fields = data[0]
+
+data_dicts = {}
+for event in data[1:]:
+    data_dicts[event[0]] = {data_fields[i]:event[i] for i in range(len(data_fields))}
+
+
+#%% Read image .csv file, format into a dictionary mapping images to capture events
+    
+all_image_file = 'D:/all_images.csv'
+with open(all_image_file,'r') as f:
+    reader = csv.reader(f,dialect = 'excel')
+    next(reader)
+    im_name_to_cap_id = {row[1]:row[0] for row in reader}
+
+
+#%% Create CCT-style .json 
+    
+images = []
+annotations = []
+categories = []
+
+capture_ims = {i:[] for i in im_name_to_cap_id.values()}
+for im_id in im_name_to_cap_id:
+    capture_ims[im_name_to_cap_id[im_id]].append(im_id)
+
+im_to_seq_num = {im:None for im in im_name_to_cap_id}
+for event in capture_ims:
+    capture_ims[event] = sorted(capture_ims[event])
+    seq_count = 0
+    for im in capture_ims[event]:
+        im_to_seq_num[im] = seq_count
+        seq_count += 1
+
+cat_to_id = {}
+cat_to_id['empty'] = 0
+cat_count = 1
+seasons = []
+
+for im_id in im_name_to_cap_id:
+    im = {}
+    im['id'] = im_id.split('.')[0]
+    im['file_name'] = im_id
+    
+    im['location'] = im_id.split('/')[1]
+    im['season'] = im_id.split('/')[0]
+    im['seq_id'] = im_name_to_cap_id[im_id]
+    im['frame_num'] = im_to_seq_num[im_id]
+    im['seq_num_frames'] = len(capture_ims[im['seq_id']])
+    if im['season'] not in seasons:
+        seasons.append(im['season'])
+
+    ann = {}
+    ann['id'] = str(uuid.uuid1())
+    ann['image_id'] = im['id']
+
+    if im_name_to_cap_id[im_id] in data_dicts:
+        im_data = data_dicts[im_name_to_cap_id[im_id]]
+        im['datetime'] = im_data['DateTime']
+        if im_data['Species'] not in cat_to_id:
+            cat_to_id[im_data['Species']] = cat_count
+            cat_count += 1
+        ann['category_id'] = cat_to_id[im_data['Species']]
+    else:
+        ann['category_id'] = 0
+            
+    #still need image width and height
+    images.append(im)
+    annotations.append(ann)
+
+# ...for each image ID
+    
+for cat in cat_to_id:
+    new_cat = {}
+    new_cat['id'] = cat_to_id[cat]
+    new_cat['name'] = cat
+    categories.append(new_cat) 
+
+
+#%% Write output files
+    
+output_file = output_file_folder + 'SnapshotSerengeti.json'
+json_data = {}
+json_data['images'] = images
+json_data['annotations'] = annotations
+json_data['categories'] = categories
+info = {}
+info['year'] = 2018
+info['version'] = 1
+info['description'] = 'COCO style Snapshot Serengeti database'
+info['contributor'] = 'SMB'
+info['date_created'] = str(datetime.date.today())
+json_data['info'] = info
+json.dump(json_data,open(output_file,'w'))
+
+for season in seasons:
+    
+    output_file = output_file_folder + season + '.json'
+    inSeason = {im['id']:False for im in images}
+    for im in images:
+        if im['season'] == season:
+            inSeason[im['id']] = True
+    new_ims = [im for im in images if inSeason[im['id']]]
+    new_anns = [ann for ann in annotations if inSeason[ann['image_id']]]
+
+    json_data = {}
+    json_data['images'] = new_ims
+    json_data['annotations'] = new_anns
+    json_data['categories'] = categories
+    info = {}
+    info['year'] = 2018
+    info['version'] = 1
+    info['description'] = 'COCO style Snapshot Serengeti database. season ' + season
+    info['contributor'] = 'SMB'
+    info['date_created'] = str(datetime.date.today())
+    json_data['info'] = info
+    json.dump(json_data,open(output_file,'w'))
+    
+    print('Season ' + season)
+    print(str(len(new_ims)) + ' images')
+    print(str(len(new_anns)) + ' annotations') 
+    
+# ...for each season    

megadetector/data_management/importers/sulross_get_exif.py

@@ -0,0 +1,65 @@
+"""
+
+ sulross_get_exif.py
+
+ For the Sul Ross dataset, species informationw was stored in XMP metadata; pull 
+ all that metadata out to .json.
+
+"""
+
+import os
+import json
+from tqdm import tqdm
+
+import exiftool
+
+image_ids_path = '/home/beaver/cameratraps/data/sulross/20190522_image_ids.json'
+data_dir = '/home/beaver/cameratraps/mnt/sulross'
+
+
+def get_metadata():
+    
+    image_ids = json.load(open(image_ids_path))
+
+    image_id_to_metadata = {}
+
+    # exiftool can process a batch of images at a time, but bottleneck is blobfuse reading the images
+    batch_size = 20  
+
+    num_images_processed = 0
+
+    with exiftool.ExifTool() as et:
+        for i in tqdm(range(0, len(image_ids), batch_size)):
+            batch_ids = image_ids[i: i + batch_size]
+
+            batch_paths = [os.path.join(data_dir, i) for i in batch_ids]
+
+            try:
+                metadatas = et.get_metadata_batch(batch_paths)
+
+                for id, metadata in zip(batch_ids, metadatas):
+                    image_id_to_metadata[id] = metadata['XMP:HierarchicalSubject']
+            except Exception as e:
+                print('Exception! {}'.format(e))
+                continue
+
+            num_images_processed += batch_size
+            if num_images_processed % 1000 == 0:
+                print('Finished processing {} images; image ID {}'.format(
+                    num_images_processed, image_ids[num_images_processed - 1]))
+                print(image_id_to_metadata[id])
+                print()
+
+            # checkpoint
+            if num_images_processed % 10000 == 0:
+                print('Saving results so far...')
+                with open('/home/beaver/cameratraps/data/sulross/20190522_metadata.json', 'w') as f:
+                    json.dump(image_id_to_metadata, f, indent=1)
+
+    print('Length of meta data read: ', len(image_id_to_metadata))
+    with open('/home/beaver/cameratraps/data/sulross/20190522_metadata.json', 'w') as f:
+        json.dump(image_id_to_metadata, f, indent=1)
+    print('Results saved. Done!')
+
+if __name__ == '__main__':
+    get_metadata()