megadetector 5.0.11__py3-none-any.whl → 5.0.12__py3-none-any.whl

megadetector/data_management/importers/wellington_to_json.py

@@ -0,0 +1,263 @@
+"""
+
+ wellington_to_json.py
+
+ Convert the .csv file provided for the Wellington data set to a 
+ COCO-camera-traps .json file
+
+"""
+
+#%% Constants and environment
+
+import pandas as pd
+import os
+import glob
+import json
+import re
+import uuid
+import time
+import ntpath
+import humanfriendly
+import PIL
+
+from tqdm import tqdm
+
+input_metadata_file = os.path.expanduser('~/data/wct/wellington_camera_traps.csv')
+output_file = os.path.expanduser('~/data/wct/wellington_camera_traps.json')
+image_directory = os.path.expanduser('~/data/wct/images')
+preview_dir = os.path.expanduser('~/data/wct/preview')
+
+assert(os.path.isdir(image_directory))
+
+
+#%% Read source data
+
+input_metadata = pd.read_csv(input_metadata_file)
+
+print('Read {} columns and {} rows from metadata file'.format(len(input_metadata.columns),
+      len(input_metadata)))
+
+# Filenames were provided as *.jpg, but images were *.JPG, converting here
+input_metadata['file'] = input_metadata['file'].apply(lambda x: x.replace('.jpg','.JPG'))
+
+print('Converted extensions to uppercase')
+
+
+#%% Map filenames to rows, verify image existence
+
+# Takes ~30 seconds, since it's checking the existence of ~270k images
+
+start_time = time.time()
+filenames_to_rows = {}
+image_filenames = input_metadata.file
+
+duplicate_rows = []
+
+# Build up a map from filenames to a list of rows, checking image existence as we go
+for i_file,fn in enumerate(image_filenames):
+    
+    if (fn in filenames_to_rows):
+        duplicate_rows.append(i_file)
+        filenames_to_rows[fn].append(i_file)
+    else:
+        filenames_to_rows[fn] = [i_file]
+        image_path = os.path.join(image_directory,fn)
+        assert(os.path.isfile(image_path))
+
+elapsed = time.time() - start_time
+print('Finished verifying image existence in {}, found {} filenames with multiple labels'.format(
+      humanfriendly.format_timespan(elapsed),len(duplicate_rows)))
+
+# I didn't expect this to be true a priori, but it appears to be true, and
+# it saves us the trouble of checking consistency across multiple occurrences
+# of an image.
+assert(len(duplicate_rows) == 0)    
+    
+    
+#%% Check for images that aren't included in the metadata file
+
+# Enumerate all images
+image_full_paths = glob.glob(os.path.join(image_directory,'*.JPG'))
+
+for i_image,image_path in enumerate(image_full_paths):
+    
+    fn = ntpath.basename(image_path)
+    assert(fn in filenames_to_rows)
+
+print('Finished checking {} images to make sure they\'re in the metadata'.format(
+        len(image_full_paths)))
+
+
+#%% Create CCT dictionaries
+
+# Also gets image sizes, so this takes ~6 minutes
+#
+# Implicitly checks images for overt corruptness, i.e. by not crashing.
+
+images = []
+annotations = []
+
+# Map categories to integer IDs (that's what COCO likes)
+next_category_id = 0
+categories_to_category_id = {}
+categories_to_counts = {}
+
+# For each image
+#
+# Because in practice images are 1:1 with annotations in this data set,
+# this is also a loop over annotations.
+
+start_time = time.time()
+
+sequence_frame_ids = set()
+
+# image_name = image_filenames[0]
+for image_name in tqdm(image_filenames):
+    
+    rows = filenames_to_rows[image_name]
+    
+    # As per above, this is convenient and appears to be true; asserting to be safe
+    assert(len(rows) == 1)    
+    i_row = rows[0]
+    
+    row = input_metadata.iloc[i_row]
+    
+    im = {}
+    # Filenames look like "290716114012001a1116.jpg"
+    im['id'] = image_name.split('.')[0]    
+    im['file_name'] = image_name
+    
+    # This gets imported as an int64
+    im['seq_id'] = str(row['sequence'])
+        
+    # These appear as "image1", "image2", etc.
+    frame_id = row['image_sequence']
+    m = re.match('^image(\d+)$',frame_id)
+    assert (m is not None)
+    im['frame_num'] = int(m.group(1))-1
+    
+    # Make sure we haven't seen this sequence before
+    sequence_frame_id = im['seq_id'] + '_' + str(im['frame_num'])
+    assert sequence_frame_id not in sequence_frame_ids
+    sequence_frame_ids.add(sequence_frame_id)
+    
+    # In the form "001a"
+    im['location'] = row['site']
+    
+    # Can be in the form '111' or 's46'
+    im['camera'] = row['camera']
+    
+    # In the form "7/29/2016 11:40"
+    im['datetime'] = row['date']
+    
+    # Check image height and width
+    image_path = os.path.join(image_directory,fn)
+    assert(os.path.isfile(image_path))
+    pil_image = PIL.Image.open(image_path)
+    width, height = pil_image.size
+    im['width'] = width
+    im['height'] = height
+
+    images.append(im)
+    
+    category = row['label'].lower()
+    
+    # Use 'empty', to be consistent with other data on lila    
+    if (category == 'nothinghere'):
+        category = 'empty'
+        
+    # Have we seen this category before?
+    if category in categories_to_category_id:
+        category_id = categories_to_category_id[category]
+        categories_to_counts[category] += 1
+    else:
+        category_id = next_category_id
+        categories_to_category_id[category] = category_id
+        categories_to_counts[category] = 0
+        next_category_id += 1
+    
+    # Create an annotation
+    ann = {}
+    
+    # The Internet tells me this guarantees uniqueness to a reasonable extent, even
+    # beyond the sheer improbability of collisions.
+    ann['id'] = str(uuid.uuid1())
+    ann['image_id'] = im['id']    
+    ann['category_id'] = category_id
+    
+    annotations.append(ann)
+    
+# ...for each image
+    
+# Convert categories to a CCT-style dictionary
+
+categories = []
+
+for category in categories_to_counts:
+    print('Category {}, count {}'.format(category,categories_to_counts[category]))
+    category_id = categories_to_category_id[category]
+    cat = {}
+    cat['name'] = category
+    cat['id'] = category_id
+    categories.append(cat)
+    
+elapsed = time.time() - start_time
+print('Finished creating CCT dictionaries in {}'.format(
+      humanfriendly.format_timespan(elapsed)))
+    
+
+#%% Create info struct
+
+info = {}
+info['year'] = 2018
+info['version'] = '1.01'
+info['description'] = 'Wellington Camera Traps'
+info['secondary_contributor'] = 'Converted to COCO .json by Dan Morris'
+info['contributor'] = 'Victor Anton'
+
+
+#%% Write output
+
+json_data = {}
+json_data['images'] = images
+json_data['annotations'] = annotations
+json_data['categories'] = categories
+json_data['info'] = info
+json.dump(json_data,open(output_file,'w'),indent=1)
+
+print('Finished writing .json file with {} images, {} annotations, and {} categories'.format(
+        len(images),len(annotations),len(categories)))
+
+
+#%% Validate .json files
+
+from megadetector.data_management.databases import integrity_check_json_db
+
+options = integrity_check_json_db.IntegrityCheckOptions()
+options.baseDir = image_directory
+options.bCheckImageSizes = False
+options.bCheckImageExistence = True
+options.bFindUnusedImages = True
+
+sorted_categories, data, error_info = integrity_check_json_db.integrity_check_json_db(output_file, options)
+
+
+#%% Preview labels
+
+from megadetector.visualization import visualize_db
+
+viz_options = visualize_db.DbVizOptions()
+viz_options.num_to_visualize = 2000
+viz_options.trim_to_images_with_bboxes = False
+viz_options.add_search_links = False
+viz_options.sort_by_filename = False
+viz_options.parallelize_rendering = True
+viz_options.classes_to_exclude = ['test']
+html_output_file, image_db = visualize_db.visualize_db(db_path=output_file,
+                                                         output_dir=os.path.join(
+                                                         preview_dir),
+                                                         image_base_dir=image_directory,
+                                                         options=viz_options)
+
+from megadetector.utils import path_utils
+path_utils.open_file(html_output_file)

megadetector/data_management/importers/wi_to_json.py

@@ -0,0 +1,442 @@
+"""
+
+ wi_to_json
+
+ Prepares CCT-formatted metadata based on a Wildlife Insights data export.
+ 
+ Mostly assumes you have the images also, for validation/QA.
+
+"""
+
+#%% Imports and constants
+
+import os
+import json
+import pandas as pd
+import shutil
+import uuid
+import datetime
+import dateutil.parser
+import sys
+import subprocess
+import copy
+
+from collections import defaultdict
+from tqdm import tqdm
+
+from megadetector.visualization import visualize_db
+from megadetector.data_management.databases import integrity_check_json_db
+
+organization_name = 'organization'
+input_base = os.path.expanduser('~/data/' + organization_name)
+image_base = os.path.join(input_base,'deployment')
+image_csv = os.path.join(input_base,'images.csv')
+output_json_filename = os.path.join(input_base, organization_name + '_camera_traps.json')
+preview_base = os.path.expanduser('~/data/' + organization_name + '/preview')
+
+assert os.path.isfile(image_csv)
+assert os.path.isdir(image_base)
+
+MISSING_COMMON_NAME_TOKEN = 'MISSING'
+
+output_encoding = 'utf-8'
+
+# Because WI filenames are GUIDs, it's not practical to page through sequences in an
+# image viewer.  So we're going to (optionally) create a copy of the data set where
+# images are ordered.
+create_ordered_dataset = False
+
+ordered_image_base = os.path.join(input_base,'deployment-ordered')
+ordered_json_filename = os.path.join(input_base, organization_name + '_camera_traps_ordered.json')
+ordered_preview_base = os.path.expanduser('~/data/' + organization_name + '/preview-ordered')
+
+info = {}
+info['year'] = 2020
+info['version'] = '1.0'
+info['description'] = organization_name + ' camera traps)'
+info['contributor'] = organization_name
+info['date_created'] = str(datetime.date.today())
+
+def open_file(filename):
+    if sys.platform == "win32":
+        os.startfile(filename)
+    else:
+        opener = "open" if sys.platform == "darwin" else "xdg-open"
+        subprocess.call([opener, filename])
+
+
+#%% Load ground truth
+
+images_df = pd.read_csv(image_csv)
+
+print('Loaded {} ground truth annotations'.format(
+    len(images_df)))
+
+
+#%% Take everything out of Pandas
+
+images = images_df.to_dict('records')
+
+
+#%% Synthesize common names when they're not available
+
+for im in images:
+        
+    if not isinstance(im['common_name'],str):
+        
+        # Blank rows should always have "Blank" as the common name
+        assert im['is_blank'] == 0
+        assert isinstance(im['genus'],str) and isinstance(im['species'],str)
+        # print('Warning: missing common name for row {} ({})'.format(i_row,row['filename']))
+        im['common_name'] = im['genus'].strip() + ' ' + im['species'].strip()
+    
+    
+#%% Convert string timestamps to Python datetimes
+
+all_locations = set()
+
+# im = ground_truth_dicts[0]
+for im in tqdm(images):
+    dt = dateutil.parser.isoparse(im['timestamp'])
+    assert dt.year >= 2019 and dt.year <= 2021
+    im['datetime'] = dt
+    
+    # The field called "location" in the WI .csv file is a URL, we want to reclaim
+    # the "location" keyword for CCT output
+    im['url'] = im['location']
+    
+    # Filenames look like, e.g., N36/100EK113/06040726.JPG    
+    im['location'] = im['deployment_id']
+    all_locations.add(im['location'])
+    
+
+#%% Synthesize sequence information
+
+locations = all_locations
+print('Found {} locations'.format(len(locations)))
+
+locations = list(locations)
+
+sequences = set()
+sequence_to_images = defaultdict(list) 
+max_seconds_within_sequence = 10
+
+# Sort images by time within each location
+# i_location=0; location = locations[i_location]
+for i_location,location in tqdm(enumerate(locations),total=len(locations)):
+    
+    images_this_location = [im for im in images if im['location'] == location]
+    sorted_images_this_location = sorted(images_this_location, key = lambda im: im['datetime'])
+    
+    current_sequence_id = None
+    next_frame_number = 0
+    previous_datetime = None
+        
+    # previous_datetime = sorted_images_this_location[0]['datetime']
+    # im = sorted_images_this_camera[1]
+    for i_image,im in enumerate(sorted_images_this_location):
+
+        # Timestamp for this image, may be None
+        dt = im['datetime']
+        
+        # Start a new sequence if:
+        #
+        # * This image has no timestamp
+        # * This image has a frame number of zero
+        # * We have no previous image timestamp
+        #
+        if dt is None:
+            delta = None
+        elif previous_datetime is None:
+            delta = None
+        else:
+            assert isinstance(dt,datetime.datetime)
+            delta = (dt - previous_datetime).total_seconds()
+        
+        # Start a new sequence if necessary
+        if delta is None or delta > max_seconds_within_sequence:
+            next_frame_number = 0
+            current_sequence_id = str(uuid.uuid1())
+            sequences.add(current_sequence_id)
+        assert current_sequence_id is not None
+        
+        im['seq_id'] = current_sequence_id
+        im['synthetic_frame_number'] = next_frame_number
+        next_frame_number = next_frame_number + 1
+        previous_datetime = dt
+        sequence_to_images[im['seq_id']].append(im)
+    
+    # ...for each image in this location
+
+# ...for each location
+
+
+#%% Create category dict and category IDs
+
+categories_to_counts = defaultdict(int)
+category_mappings = {'blank':'empty',
+                     'mammal':'unknown_mammal',
+                     'bird':'unknown_bird',
+                     'unknown_species':'unknown'
+                     }
+
+for c in category_mappings.values():
+    assert ' ' not in c
+    
+# im = images[0]
+for im in tqdm(images):
+    
+    category_name = im['common_name'].lower().replace("'",'').replace(' ','_')
+    if category_name in category_mappings:
+        category_name = category_mappings[category_name]
+    categories_to_counts[category_name] += 1
+    im['category_name'] = category_name
+
+categories_to_counts_sorted = {k: v for k, v in sorted(categories_to_counts.items(),
+                                                       key=lambda item: item[1],reverse=True)}
+
+print('\n')
+for s in categories_to_counts_sorted.keys():
+    print('{}: {}'.format(s,categories_to_counts_sorted[s]))
+
+
+#%% Count frames in each sequence
+
+sequence_id_to_n_frames = defaultdict(int)
+
+for im in tqdm(images):
+    seq_id = im['seq_id']
+    sequence_id_to_n_frames[seq_id] = sequence_id_to_n_frames[seq_id] + 1
+
+for im in tqdm(images):
+    seq_id = im['seq_id']
+    im['seq_num_frames'] = sequence_id_to_n_frames[seq_id]
+
+
+#%% Build relative paths
+
+missing_images = []
+
+# im = images[0]
+for i_image,im in enumerate(tqdm(images)):
+    # Sample URL:
+    #
+    # gs://project-asfasdfd/deployment/21444549/asdfasdfd-616a-4d10-a921-45ac456c568a.jpg'
+    relative_path = im['url'].split('/deployment/')[1]
+    assert relative_path is not None and len(relative_path) > 0
+    im['relative_path'] = relative_path
+    
+    if not os.path.isfile(os.path.join(image_base,relative_path)):
+        missing_images.append(im)
+    
+print('{} images are missing'.format(len(missing_images)))
+
+
+#%% Double check images with multiple annotations
+
+filename_to_images = defaultdict(list)
+
+# im = images[0]
+for im in tqdm(images):
+    filename_to_images[im['relative_path']].append(im)
+    
+filenames_with_multiple_annotations = [fn for fn in filename_to_images.keys() if len(filename_to_images[fn]) > 1]
+
+print('\nFound {} filenames with multiple annotations'.format(len(filenames_with_multiple_annotations)))
+
+
+#%% Assemble dictionaries
+
+images_out = []
+image_id_to_image = {}
+annotations = []
+categories = []
+
+category_name_to_category = {}
+category_id_to_category = {}
+
+# Force the empty category to be ID 0
+empty_category = {}
+empty_category['name'] = 'empty'
+empty_category['id'] = 0
+empty_category['count'] = 0
+
+category_id_to_category[0] = empty_category
+category_name_to_category['empty'] = empty_category
+categories.append(empty_category)
+next_id = 1
+
+# input_im = images[0]
+for input_im in tqdm(images):
+
+    category_name = input_im['category_name'].lower().strip()
+    
+    if category_name not in category_name_to_category:        
+
+        category_id = next_id
+        next_id += 1
+        category = {}
+        category['id'] = category_id
+        category['name'] = category_name
+        category['count'] = 0
+        categories.append(category)
+        category_name_to_category[category_name] = category
+        category_id_to_category[category_id] = category
+
+    else:
+        
+        category = category_name_to_category[category_name]
+        
+    category_id = category['id']    
+    category['count'] += 1
+    
+    im = {}
+    im['id'] = input_im['relative_path'].replace('/','_')
+    im['datetime'] = str(input_im['datetime'])
+    im['file_name'] = input_im['relative_path']
+    im['seq_id'] = input_im['seq_id']
+    im['frame_num'] = input_im['synthetic_frame_number']
+    im['seq_num_frames'] = input_im['seq_num_frames']
+    im['location'] = input_im['location']
+    
+    if im['id'] in image_id_to_image:
+        # print('Warning: image ID {} ({}) has multiple annotations'.format(im['id'],im['id'].replace('_','/')))
+        pass
+    else:
+        image_id_to_image[im['id']] = im    
+        images_out.append(im)
+    
+    ann = {}
+    
+    ann['id'] = str(uuid.uuid1())
+    ann['image_id'] = im['id']
+    ann['category_id'] = category_id
+    ann['sequence_level_annotation'] = False
+    annotations.append(ann)
+
+# ...for each image
+
+
+#%% Write output .json
+
+data = {}
+data['info'] = info
+data['images'] = images_out
+data['annotations'] = annotations
+data['categories'] = categories
+
+with open(output_json_filename, 'w') as f:
+    json.dump(data, f, indent=1)
+    
+print('Finished writing json to {}'.format(output_json_filename))
+
+    
+#%% Validate .json file
+
+options = integrity_check_json_db.IntegrityCheckOptions()
+options.baseDir = image_base
+options.bCheckImageSizes = False
+options.bCheckImageExistence = True
+options.bFindUnusedImages = True
+
+_, _, _ = integrity_check_json_db.integrity_check_json_db(output_json_filename, options)
+
+
+#%% Preview labels
+
+viz_options = visualize_db.DbVizOptions()
+viz_options.num_to_visualize = 300
+viz_options.trim_to_images_with_bboxes = False
+viz_options.add_search_links = True
+viz_options.sort_by_filename = False
+viz_options.parallelize_rendering = True
+viz_options.include_filename_links = True
+
+html_output_file, _ = visualize_db.visualize_db(db_path=output_json_filename,
+                                                         output_dir=preview_base,
+                                                         image_base_dir=image_base,
+                                                         options=viz_options)
+open_file(html_output_file)
+# open_file(os.path.join(image_base,'2100703/1141a545-88d2-498b-a684-7431f7aeb324.jpg'))
+
+
+#%%
+
+if create_ordered_dataset:
+    
+    pass
+
+    #%% Create ordered dataset
+    
+    # Because WI filenames are GUIDs, it's not practical to page through sequences in an
+    # image viewer.  So we're going to create a copy of the data set where images are 
+    # ordered.
+    
+    os.makedirs(ordered_image_base,exist_ok=True)
+    
+    ordered_images = {}
+    
+    # im = images_out[0]; im
+    for im in tqdm(images_out):
+        im_out = copy.deepcopy(im)
+        ordered_filename = im['location'] + '_' + im['seq_id'] + '_' +\
+            str(im['frame_num']) + '_' + os.path.basename(im['file_name'])
+        assert ordered_filename not in ordered_images
+        im_out['original_file'] = im_out['file_name']
+        im_out['file_name'] = ordered_filename
+        ordered_images[ordered_filename] = im_out
+        
+    ordered_images = list(ordered_images.values())
+    
+    
+    #%% Create ordered .json 
+    
+    data_ordered = copy.copy(data)
+    data_ordered['images'] = ordered_images
+    
+    with open(ordered_json_filename, 'w') as f:
+        json.dump(data_ordered, f, indent=1)
+        
+    print('Finished writing json to {}'.format(ordered_json_filename))
+    
+    
+    #%% Copy files to their new locations
+        
+    # im = ordered_images[0]
+    for im in tqdm(ordered_images):
+        output_file = os.path.join(ordered_image_base,im['file_name'])
+        input_file = os.path.join(image_base,im['original_file'])
+        if not os.path.isfile(input_file):
+            print('Warning: file {} is missing'.format(input_file))
+            continue
+        shutil.copyfile(input_file,output_file)
+    
+    original_fn_to_ordered_fn = {}
+    # im = data_ordered['images'][0]
+    for im in data_ordered['images']:
+        original_fn_to_ordered_fn[im['original_file']] = im['file_name']    
+    
+    
+    #%% Preview labels in the ordered dataset
+    
+    viz_options = visualize_db.DbVizOptions()
+    viz_options.num_to_visualize = 300
+    viz_options.trim_to_images_with_bboxes = False
+    viz_options.add_search_links = True
+    viz_options.sort_by_filename = False
+    viz_options.parallelize_rendering = True
+    viz_options.include_filename_links = True
+    
+    html_output_file, _ = visualize_db.visualize_db(db_path=ordered_json_filename,
+                                                             output_dir=ordered_preview_base,
+                                                             image_base_dir=ordered_image_base,
+                                                             options=viz_options)
+    open_file(html_output_file)
+    # open_file(os.path.join(image_base,'2100703/1141a545-88d2-498b-a684-7431f7aeb324.jpg'))
+        
+    
+    #%% Open an ordered filename from the unordered filename    
+    
+    unordered_filename = '2100557/54e5c751-28b4-42e3-b6d4-e8ee290228ae.jpg'
+    fn = os.path.join(ordered_image_base,original_fn_to_ordered_fn[unordered_filename])
+    open_file(fn)