megadetector 5.0.11__py3-none-any.whl → 5.0.12__py3-none-any.whl

megadetector/data_management/importers/ena24_to_json.py

@@ -0,0 +1,276 @@
+"""
+
+ ena24_to_json_2017.py
+
+ Convert the ENA24 data set to a COCO-camera-traps .json file
+
+"""
+
+#%% Constants and environment
+
+import os
+import json
+import uuid
+import time
+import humanfriendly
+import numpy as np
+import shutil
+import zipfile
+
+from PIL import Image
+from tqdm import tqdm
+
+base_directory = r'e:\wildlife_data\ena24'
+output_file = os.path.join(base_directory,'ena24.json')
+image_directory = os.path.join(base_directory,'images')
+label_directory = os.path.join(base_directory,'labels')
+
+assert(os.path.isdir(label_directory))
+assert(os.path.isdir(image_directory))
+
+# Temporary folders for human and non-human images
+human_dir = os.path.join(base_directory, 'human')
+non_human_dir = os.path.join(base_directory, 'non-human')
+
+human_zipfile = os.path.join(base_directory, 'ena24_humans.zip')
+non_human_zipfile = os.path.join(base_directory, 'ena24.zip')
+
+# Clean existing output folders/zipfiles
+if os.path.isdir(human_dir):
+    shutil.rmtree(human_dir)
+if os.path.isdir(non_human_dir):
+    shutil.rmtree(non_human_dir)    
+
+if os.path.isfile(human_zipfile):
+    os.remove(human_zipfile)
+if os.path.isfile(human_zipfile):
+    os.remove(non_human_zipfile)
+    
+os.makedirs(human_dir,exist_ok=True)
+os.makedirs(non_human_dir,exist_ok=True)
+
+labels = ['White_Tailed_Deer', 'Dog', 'Bobcat', 'Red Fox', 'Horse', 
+          'Domestic Cat', 'American Black Bear', 'Eastern Cottontail', 'Grey Fox', 'Coyote', 
+          'Eastern Fox Squirrel', 'Eastern Gray Squirrel', 'Vehicle', 'Eastern Chipmunk', 'Wild Turkey',
+          'Northern Raccoon', 'Striped Skunk', 'Woodchuck', 'Virginia Opossum', 'Human', 
+          'Bird', 'American Crow', 'Chicken']
+
+
+#%% Support functions
+
+def zipdir(path, zipfilename, basepath=None):
+    """
+    Zip everything in [path] into [zipfilename], with paths in the zipfile relative to [basepath]
+    """
+    ziph = zipfile.ZipFile(zipfilename, 'w', zipfile.ZIP_STORED)
+    
+    for root, dirs, files in os.walk(path):
+        for file in files:
+            src = os.path.join(root, file)
+            if basepath is None:
+                dst = file
+            else:
+                dst = os.path.relpath(src,basepath)
+            ziph.write(src, dst, zipfile.ZIP_STORED)
+
+    ziph.close()
+
+
+#%% Read source data
+
+image_list = os.listdir(label_directory)
+print('Enumerated {} label files'.format(len(image_list)))
+
+
+#%% Map filenames to rows, verify image existence
+
+startTime = time.time()
+
+# Build up a map from filenames to a list of rows, checking image existence as we go
+for filename in image_list:
+    imagePath = os.path.join(image_directory, "{}.jpg".format(filename.split(".")[0]))
+    assert(os.path.isfile(imagePath))
+
+elapsed = time.time() - startTime
+print('Finished verifying image existence for {} files in {}'.format(
+      len(image_list),humanfriendly.format_timespan(elapsed)))
+
+
+#%% Create CCT dictionaries
+
+# Also gets image sizes, so this takes ~6 minutes
+#
+# Implicitly checks images for overt corruptness, i.e. by not crashing.
+
+images = []
+annotations = []
+
+# Map categories to integer IDs (that's what COCO likes)
+nextCategoryID = 0
+categoriesToCategoryId = {}
+categoriesToCounts = {}
+
+# For each image
+#
+# Because in practice images are 1:1 with annotations in this data set,
+# this is also a loop over annotations.
+
+startTime = time.time()
+for filename in tqdm(image_list):
+    
+    contains_human = False    
+    im = {}
+    im['id'] = filename.split('.')[0]
+    fn = "{}.jpg".format(filename.split('.')[0])
+    im['file_name'] = fn
+    
+    # Check image height and width
+    imagePath = os.path.join(image_directory, fn)
+    assert(os.path.isfile(imagePath))
+    pilImage = Image.open(imagePath)
+    width, height = pilImage.size
+    im['width'] = width
+    im['height'] = height
+
+    images.append(im)
+    
+    label_path = os.path.join(label_directory, filename)
+    file_data = open(label_path, 'r').read()
+    row = file_data.split()
+    category = labels[int(row[0])-1]
+
+    rows = np.loadtxt(label_path)
+    
+    # Each row is category, [box coordinates]    
+    
+    # If there's just one row, loadtxt reads it as a 1d array; make it a 2d array 
+    # with one row 
+    if len(rows.shape)==1:
+        rows = rows.reshape(1,-5)
+    
+    assert (len(rows.shape)==2 and rows.shape[1] == 5)
+    
+    categories_this_image = set()
+
+    # Each row is a bounding box
+    for row in rows:
+        
+        i_category = int(row[0])-1        
+        category = labels[i_category]
+        if category == 'Human':
+            contains_human = True
+        categories_this_image.add(category)
+        
+        # Have we seen this category before?
+        if category in categoriesToCategoryId:
+            categoryID = categoriesToCategoryId[category]
+            categoriesToCounts[category] += 1
+        else:
+            categoryID = nextCategoryID
+            categoriesToCategoryId[category] = categoryID
+            categoriesToCounts[category] = 0
+            nextCategoryID += 1
+        
+        # Create an annotation
+        ann = {}
+        
+        ann['id'] = str(uuid.uuid1())
+        ann['image_id'] = im['id']    
+        ann['category_id'] = categoryID
+        ann['bbox'] = [row[1]*width, row[2]*height, row[3]*width, row[4]*height]
+        annotations.append(ann)
+        
+    # ...for each bounding box
+    
+    # This was here for debugging; nearly every instance is Human+Horse, Human+Vehicle,
+    # or Human+Dog, but there is one Rabbit+Opossium, and a few Deer+Chicken!
+    if False:
+        if len(categories_this_image) > 1:
+            print('Image {} has multiple categories: '.format(filename),end='')
+            for c in categories_this_image:
+                print(c, end=',')
+            print('')
+    
+    if contains_human:
+          shutil.copy(imagePath, os.path.join(base_directory, human_dir))
+    else:
+          shutil.copy(imagePath, os.path.join(base_directory, non_human_dir))
+
+# ...for each image
+    
+# Convert categories to a CCT-style dictionary
+
+categories = []
+
+for category in categoriesToCounts:
+    print('Category {}, count {}'.format(category, categoriesToCounts[category]))
+    categoryID = categoriesToCategoryId[category]
+    cat = {}
+    cat['name'] = category
+    cat['id'] = categoryID
+    categories.append(cat)    
+    
+elapsed = time.time() - startTime
+print('Finished creating CCT dictionaries in {}'.format(
+      humanfriendly.format_timespan(elapsed)))
+    
+
+#%% Create info struct
+
+info = {}
+info['year'] = 2016
+info['version'] = 1
+info['description'] = ''
+info['secondary_contributor'] = 'Converted to COCO .json by Vardhan Duvvuri'
+info['contributor'] = 'University of Missouri'
+
+
+#%% Write output
+
+json_data = {}
+json_data['images'] = images
+json_data['annotations'] = annotations
+json_data['categories'] = categories
+json_data['info'] = info
+json.dump(json_data, open(output_file, 'w'), indent=2)
+
+print('Finished writing .json file with {} images, {} annotations, and {} categories'.format(
+        len(images),len(annotations),len(categories)))
+
+
+#%% Create ZIP files for human and non human
+
+zipdir(human_dir,human_zipfile)
+zipdir(non_human_dir,non_human_zipfile)
+
+
+#%% Validate output
+
+from megadetector.data_management.databases import integrity_check_json_db
+
+fn = output_file
+options = integrity_check_json_db.IntegrityCheckOptions()
+options.baseDir = image_directory
+options.bCheckImageSizes = False
+options.bCheckImageExistence = True
+options.bFindUnusedImages = True
+    
+sortedCategories, data = integrity_check_json_db.integrity_check_json_db(fn,options)
+
+
+#%% Preview labels
+
+from megadetector.visualization import visualize_db
+from megadetector.data_management.databases import integrity_check_json_db
+
+viz_options = visualize_db.DbVizOptions()
+viz_options.num_to_visualize = None
+viz_options.trim_to_images_with_bboxes = False
+viz_options.add_search_links = True
+viz_options.sort_by_filename = False
+viz_options.parallelize_rendering = True
+html_output_file,image_db = visualize_db.visualize_db(db_path=output_file,
+                                                        output_dir=os.path.join(base_directory,'preview'),
+                                                        image_base_dir=image_directory,
+                                                        options=viz_options)
+os.startfile(html_output_file)

megadetector/data_management/importers/filenames_to_json.py

@@ -0,0 +1,386 @@
+"""
+
+ filenames_to_json.py
+
+ Take a directory of images in which species labels are encoded by folder
+ names, and produces a COCO-style .json file 
+ 
+"""
+
+#%% Constants and imports
+
+import json
+import io
+import os
+import uuid
+import csv
+import warnings
+import datetime
+
+from PIL import Image
+
+from megadetector.utils.path_utils import find_images
+
+# ignoring all "PIL cannot read EXIF metainfo for the images" warnings
+warnings.filterwarnings("ignore", "(Possibly )?corrupt EXIF data", UserWarning)
+# Metadata Warning, tag 256 had too many entries: 42, expected 1
+warnings.filterwarnings("ignore", "Metadata warning", UserWarning)
+
+# Filenames will be stored in the output .json relative to this base dir
+baseDir = r'D:\wildlife_data\bellevue_camera_traps\bellevue_camera_traps.19.06.02.1320'
+outputJsonFilename = os.path.join(baseDir,'bellevue_camera_traps.19.06.02.1320.json')
+outputCsvFilename = os.path.join(baseDir,'bellevue_camera_traps.19.06.02.1320.csv')
+
+# rawClassListFilename = os.path.join(baseDir,'bellevue_camera_traps.19.06.02.1320_classes.csv')
+# classMappingsFilename = os.path.join(baseDir,'bellevue_camera_traps.19.06.02.1320_class_mapping.csv')
+outputEncoding = 'utf-8'
+
+classMappings = {'transitional':'unlabeled','moving':'unlabeled','setup':'unlabeled','blurry':'unlabeled','transitional':'unlabeled','junk':'unlabeled','unknown':'unlabeled'}
+
+bLoadFileListIfAvailable = True
+
+info = {}
+info['year'] = 2019
+info['version'] = '1.0'
+info['description'] = 'Bellevue Camera Traps'
+info['contributor'] = 'Dan Morris'
+info['date_created'] = str(datetime.date.today())
+
+maxFiles = -1
+bReadImageSizes = False
+bUseExternalRemappingTable = False
+
+
+#%% Enumerate files, read image sizes
+
+# Each element will be a list of relative path/full path/width/height
+fileInfo = []
+nonImages = []
+nFiles = 0
+
+if bLoadFileListIfAvailable and os.path.isfile(outputCsvFilename):
+    
+    print('Loading file list from {}'.format(outputCsvFilename))
+    
+    with open(outputCsvFilename,'r') as f:
+        reader = csv.reader(f)
+        csvInfo = list(list(item) for item in csv.reader(f, delimiter=','))
+    
+    for iRow in range(len(csvInfo)):
+        csvInfo[iRow][2] = int(csvInfo[iRow][2])
+        csvInfo[iRow][3] = int(csvInfo[iRow][3])
+    
+    fileInfo = csvInfo
+    
+    print('Finished reading list of {} files'.format(len(fileInfo)))
+    
+else:
+        
+    print('Enumerating files from {} to {}'.format(baseDir,outputCsvFilename))
+    
+    image_files = find_images(baseDir,bRecursive=True)
+    print('Enumerated {} images'.format(len(image_files)))
+    
+    with io.open(outputCsvFilename, "w", encoding=outputEncoding) as outputFileHandle:    
+    
+        for fname in image_files:
+      
+            nFiles = nFiles + 1
+            if maxFiles >= 0 and nFiles > maxFiles:            
+                print('Warning: early break at {} files'.format(maxFiles))
+                break
+            
+            fullPath = fname
+            relativePath = os.path.relpath(fullPath,baseDir)
+             
+            if maxFiles >= 0:
+                print(relativePath)
+        
+            h = -1
+            w = -1
+
+            if bReadImageSizes:
+            
+                # Read the image
+                try:
+                
+                    im = Image.open(fullPath)
+                    h = im.height
+                    w = im.width
+                    
+                except:
+                    # Corrupt or not an image
+                    nonImages.append(fullPath)
+                    continue
+            
+            # Store file info
+            imageInfo = [relativePath, fullPath, w, h]
+            fileInfo.append(imageInfo)
+            
+            # Write to output file
+            outputFileHandle.write('"' + relativePath + '"' + ',' + 
+                                   '"' + fullPath + '"' + ',' + 
+                                   str(w) + ',' + str(h) + '\n')
+                                   
+        # ...for each image file
+                
+    # ...csv file output
+        
+    print("Finished writing {} file names to {}".format(nFiles,outputCsvFilename))
+
+# ...if the file list is/isn't available
+    
+    
+#%% Enumerate classes
+
+# Maps classes to counts
+classList = {}
+
+for iRow,row in enumerate(fileInfo):
+        
+    fullPath = row[0]
+    className = os.path.split(os.path.dirname(fullPath))[1]
+    className = className.lower().strip()
+    if className in classList:
+        classList[className] += 1
+    else:
+        classList[className] = 1
+    row.append(className)
+
+classNames = list(classList.keys())
+
+# We like 'empty' to be class 0
+if 'empty' in classNames:
+    classNames.remove('empty')    
+classNames.insert(0,'empty')
+
+print('Finished enumerating {} classes'.format(len(classList)))
+
+
+#%% Assemble dictionaries
+
+images = []
+annotations = []
+categories = []
+
+categoryNameToId = {}
+idToCategory = {}
+imageIdToImage = {}
+
+nextId = 0
+    
+for categoryName in classNames:
+    
+    catId = nextId
+    nextId += 1
+    categoryNameToId[categoryName] = catId
+    newCat = {}
+    newCat['id'] = categoryNameToId[categoryName]
+    newCat['name'] = categoryName
+    newCat['count'] = 0
+    categories.append(newCat) 
+    idToCategory[catId] = newCat
+            
+# ...for each category
+    
+    
+# Each element is a list of relative path/full path/width/height/className
+    
+for iRow,row in enumerate(fileInfo):
+    
+    relativePath = row[0]
+    w = row[2]
+    h = row[3]    
+    className = row[4]  
+    
+    assert className in categoryNameToId
+    categoryId = categoryNameToId[className]
+    
+    im = {}
+    im['id'] = str(uuid.uuid1())
+    im['file_name'] = relativePath
+    im['height'] = h
+    im['width'] = w
+    images.append(im)
+    imageIdToImage[im['id']] = im
+    
+    ann = {}
+    ann['id'] = str(uuid.uuid1())
+    ann['image_id'] = im['id']
+    ann['category_id'] = categoryId
+    annotations.append(ann)
+    
+    cat = idToCategory[categoryId]
+    cat['count'] += 1
+    
+# ...for each image
+
+oldNameToOldId = categoryNameToId
+originalCategories = categories
+
+print('Finished assembling dictionaries')
+
+
+#%% External class mapping
+
+if bUseExternalRemappingTable:
+    
+    assert classMappings is None
+    
+    
+    #%% Write raw class table
+    
+    # cat = categories[0]
+    if os.path.isfile(rawClassListFilename):
+        
+        print('Not over-writing raw class table')
+    
+    else:
+        
+        with io.open(rawClassListFilename, "w", encoding=outputEncoding) as classListFileHandle:
+            for cat in categories:
+                catId = cat['id']
+                categoryName = cat['name']
+                categoryCount = cat['count']
+                classListFileHandle.write(str(catId) + ',"' + categoryName + '",' + str(categoryCount) + '\n')
+        
+        print('Finished writing raw class table')
+    
+    
+    #%% Read the mapped class table
+    
+    classMappings = {}
+    
+    if os.path.isfile(classMappingsFilename):
+    
+        print('Loading file list from {}'.format(classMappingsFilename))
+            
+        with open(classMappingsFilename,'r') as f:
+            reader = csv.reader(f)
+            mappingInfo = list(list(item) for item in csv.reader(f, delimiter=','))
+        
+        for mapping in mappingInfo:
+            assert len(mapping) == 4
+            
+            # id, source, count, target
+            sourceClass = mapping[1]
+            targetClass = mapping[3]
+            assert sourceClass not in classMappings
+            classMappings[sourceClass] = targetClass
+        
+        print('Finished reading list of {} class mappings'.format(len(mappingInfo)))
+        
+else:
+
+#%% Make classMappings contain *all* classes, not just remapped classes
+    
+    # cat = categories[0]
+    for cat in categories:
+        if cat['name'] not in classMappings:
+            classMappings[cat['name']] = cat['name']
+
+      
+#%% Create new class list
+    
+categories = []
+categoryNameToId = {}
+oldIdToNewId = {}
+
+# Start at 1, explicitly assign 0 to "empty"
+nextCategoryId = 1
+for sourceClass in classMappings:
+    targetClass = classMappings[sourceClass]
+    
+    if targetClass not in categoryNameToId:
+        
+        if targetClass == 'empty':
+            categoryId = 0
+        else:
+            categoryId = nextCategoryId
+            nextCategoryId = nextCategoryId + 1
+            
+        categoryNameToId[targetClass] = categoryId
+        newCat = {}
+        newCat['id'] = categoryId
+        newCat['name'] = targetClass
+        newCat['count'] = 0
+        
+        if targetClass == 'empty':
+            categories.insert(0,newCat)
+        else:
+            categories.append(newCat)
+
+    else:
+        
+        categoryId = categoryNameToId[targetClass]
+    
+    # One-off issue with character encoding
+    if sourceClass == 'human':
+        sourceClass = 'human'
+        
+    assert sourceClass in oldNameToOldId
+    oldId = oldNameToOldId[sourceClass]
+    oldIdToNewId[oldId] = categoryId
+
+categoryIdToCat = {}
+for cat in categories:
+    categoryIdToCat[cat['id']] = cat
+    
+print('Mapped {} original classes to {} new classes'.format(len(originalCategories),len(categories)))
+
+
+#%% Re-map annotations
+            
+# ann = annotations[0]            
+for ann in annotations:
+
+    ann['category_id'] = oldIdToNewId[ann['category_id']]
+    
+    
+#%% Write output .json
+
+data = {}
+data['info'] = info
+data['images'] = images
+data['annotations'] = annotations
+data['categories'] = categories
+
+json.dump(data, open(outputJsonFilename,'w'), indent=4)
+
+print('Finished writing json to {}'.format(outputJsonFilename))
+
+
+#%% Utilities
+
+if False:
+    
+    #%% 
+    # Find images with a particular tag
+    className = 'hum'
+    matches = []
+    assert className in categoryNameToId
+    catId = categoryNameToId[className]
+    for ann in annotations:
+        if ann['category_id'] == catId:
+            imageId = ann['image_id']
+            im = imageIdToImage[imageId]
+            matches.append(im['file_name'])
+    print('Found {} matches'.format(len(matches)))
+    
+    os.startfile(os.path.join(baseDir,matches[0]))
+    
+    
+    #%% Randomly sample annotations
+    
+    import random
+    nAnnotations = len(annotations)
+    iAnn = random.randint(0,nAnnotations)
+    ann = annotations[iAnn]
+    catId = ann['category_id']
+    imageId = ann['image_id']
+    im = imageIdToImage[imageId]
+    fn = os.path.join(baseDir,im['file_name'])
+    cat = categoryIdToCat[catId]
+    className = cat['name']
+    print('This should be a {}'.format(className))
+    os.startfile(fn)