megadetector 5.0.9__py3-none-any.whl → 5.0.11__py3-none-any.whl

data_management/importers/ena24_to_json.py

@@ -1,275 +0,0 @@
-"""
-
- ena24_to_json_2017.py
-
- Convert the ENA24 data set to a COCO-camera-traps .json file
-
-"""
-
-#%% Constants and environment
-
-import os
-import json
-import uuid
-import time
-import humanfriendly
-import numpy as np
-from PIL import Image
-from tqdm import tqdm
-import shutil
-import zipfile
-
-base_directory = r'e:\wildlife_data\ena24'
-output_file = os.path.join(base_directory,'ena24.json')
-image_directory = os.path.join(base_directory,'images')
-label_directory = os.path.join(base_directory,'labels')
-
-assert(os.path.isdir(label_directory))
-assert(os.path.isdir(image_directory))
-
-# Temporary folders for human and non-human images
-human_dir = os.path.join(base_directory, 'human')
-non_human_dir = os.path.join(base_directory, 'non-human')
-
-human_zipfile = os.path.join(base_directory, 'ena24_humans.zip')
-non_human_zipfile = os.path.join(base_directory, 'ena24.zip')
-
-# Clean existing output folders/zipfiles
-if os.path.isdir(human_dir):
-    shutil.rmtree(human_dir)
-if os.path.isdir(non_human_dir):
-    shutil.rmtree(non_human_dir)    
-
-if os.path.isfile(human_zipfile):
-    os.remove(human_zipfile)
-if os.path.isfile(human_zipfile):
-    os.remove(non_human_zipfile)
-    
-os.makedirs(human_dir,exist_ok=True)
-os.makedirs(non_human_dir,exist_ok=True)
-
-labels = ['White_Tailed_Deer', 'Dog', 'Bobcat', 'Red Fox', 'Horse', 
-          'Domestic Cat', 'American Black Bear', 'Eastern Cottontail', 'Grey Fox', 'Coyote', 
-          'Eastern Fox Squirrel', 'Eastern Gray Squirrel', 'Vehicle', 'Eastern Chipmunk', 'Wild Turkey',
-          'Northern Raccoon', 'Striped Skunk', 'Woodchuck', 'Virginia Opossum', 'Human', 
-          'Bird', 'American Crow', 'Chicken']
-
-
-#%% Support functions
-
-def zipdir(path, zipfilename, basepath=None):
-    """
-    Zip everything in [path] into [zipfilename], with paths in the zipfile relative to [basepath]
-    """
-    ziph = zipfile.ZipFile(zipfilename, 'w', zipfile.ZIP_STORED)
-    
-    for root, dirs, files in os.walk(path):
-        for file in files:
-            src = os.path.join(root, file)
-            if basepath is None:
-                dst = file
-            else:
-                dst = os.path.relpath(src,basepath)
-            ziph.write(src, dst, zipfile.ZIP_STORED)
-
-    ziph.close()
-
-
-#%% Read source data
-
-image_list = os.listdir(label_directory)
-print('Enumerated {} label files'.format(len(image_list)))
-
-
-#%% Map filenames to rows, verify image existence
-
-startTime = time.time()
-
-# Build up a map from filenames to a list of rows, checking image existence as we go
-for filename in image_list:
-    imagePath = os.path.join(image_directory, "{}.jpg".format(filename.split(".")[0]))
-    assert(os.path.isfile(imagePath))
-
-elapsed = time.time() - startTime
-print('Finished verifying image existence for {} files in {}'.format(
-      len(image_list),humanfriendly.format_timespan(elapsed)))
-
-
-#%% Create CCT dictionaries
-
-# Also gets image sizes, so this takes ~6 minutes
-#
-# Implicitly checks images for overt corruptness, i.e. by not crashing.
-
-images = []
-annotations = []
-
-# Map categories to integer IDs (that's what COCO likes)
-nextCategoryID = 0
-categoriesToCategoryId = {}
-categoriesToCounts = {}
-
-# For each image
-#
-# Because in practice images are 1:1 with annotations in this data set,
-# this is also a loop over annotations.
-
-startTime = time.time()
-for filename in tqdm(image_list):
-    
-    contains_human = False    
-    im = {}
-    im['id'] = filename.split('.')[0]
-    fn = "{}.jpg".format(filename.split('.')[0])
-    im['file_name'] = fn
-    
-    # Check image height and width
-    imagePath = os.path.join(image_directory, fn)
-    assert(os.path.isfile(imagePath))
-    pilImage = Image.open(imagePath)
-    width, height = pilImage.size
-    im['width'] = width
-    im['height'] = height
-
-    images.append(im)
-    
-    label_path = os.path.join(label_directory, filename)
-    file_data = open(label_path, 'r').read()
-    row = file_data.split()
-    category = labels[int(row[0])-1]
-
-    rows = np.loadtxt(label_path)
-    
-    # Each row is category, [box coordinates]    
-    
-    # If there's just one row, loadtxt reads it as a 1d array; make it a 2d array 
-    # with one row 
-    if len(rows.shape)==1:
-        rows = rows.reshape(1,-5)
-    
-    assert (len(rows.shape)==2 and rows.shape[1] == 5)
-    
-    categories_this_image = set()
-
-    # Each row is a bounding box
-    for row in rows:
-        
-        i_category = int(row[0])-1        
-        category = labels[i_category]
-        if category == 'Human':
-            contains_human = True
-        categories_this_image.add(category)
-        
-        # Have we seen this category before?
-        if category in categoriesToCategoryId:
-            categoryID = categoriesToCategoryId[category]
-            categoriesToCounts[category] += 1
-        else:
-            categoryID = nextCategoryID
-            categoriesToCategoryId[category] = categoryID
-            categoriesToCounts[category] = 0
-            nextCategoryID += 1
-        
-        # Create an annotation
-        ann = {}
-        
-        ann['id'] = str(uuid.uuid1())
-        ann['image_id'] = im['id']    
-        ann['category_id'] = categoryID
-        ann['bbox'] = [row[1]*width, row[2]*height, row[3]*width, row[4]*height]
-        annotations.append(ann)
-        
-    # ...for each bounding box
-    
-    # This was here for debugging; nearly every instance is Human+Horse, Human+Vehicle,
-    # or Human+Dog, but there is one Rabbit+Opossium, and a few Deer+Chicken!
-    if False:
-        if len(categories_this_image) > 1:
-            print('Image {} has multiple categories: '.format(filename),end='')
-            for c in categories_this_image:
-                print(c, end=',')
-            print('')
-    
-    if contains_human:
-          shutil.copy(imagePath, os.path.join(base_directory, human_dir))
-    else:
-          shutil.copy(imagePath, os.path.join(base_directory, non_human_dir))
-
-# ...for each image
-    
-# Convert categories to a CCT-style dictionary
-
-categories = []
-
-for category in categoriesToCounts:
-    print('Category {}, count {}'.format(category, categoriesToCounts[category]))
-    categoryID = categoriesToCategoryId[category]
-    cat = {}
-    cat['name'] = category
-    cat['id'] = categoryID
-    categories.append(cat)    
-    
-elapsed = time.time() - startTime
-print('Finished creating CCT dictionaries in {}'.format(
-      humanfriendly.format_timespan(elapsed)))
-    
-
-#%% Create info struct
-
-info = {}
-info['year'] = 2016
-info['version'] = 1
-info['description'] = ''
-info['secondary_contributor'] = 'Converted to COCO .json by Vardhan Duvvuri'
-info['contributor'] = 'University of Missouri'
-
-
-#%% Write output
-
-json_data = {}
-json_data['images'] = images
-json_data['annotations'] = annotations
-json_data['categories'] = categories
-json_data['info'] = info
-json.dump(json_data, open(output_file, 'w'), indent=2)
-
-print('Finished writing .json file with {} images, {} annotations, and {} categories'.format(
-        len(images),len(annotations),len(categories)))
-
-
-#%% Create ZIP files for human and non human
-
-zipdir(human_dir,human_zipfile)
-zipdir(non_human_dir,non_human_zipfile)
-
-
-#%% Validate output
-
-from data_management.databases import integrity_check_json_db
-
-fn = output_file
-options = integrity_check_json_db.IntegrityCheckOptions()
-options.baseDir = image_directory
-options.bCheckImageSizes = False
-options.bCheckImageExistence = True
-options.bFindUnusedImages = True
-    
-sortedCategories, data = integrity_check_json_db.integrity_check_json_db(fn,options)
-
-
-#%% Preview labels
-
-from md_visualization import visualize_db
-from data_management.databases import integrity_check_json_db
-
-viz_options = visualize_db.DbVizOptions()
-viz_options.num_to_visualize = None
-viz_options.trim_to_images_with_bboxes = False
-viz_options.add_search_links = True
-viz_options.sort_by_filename = False
-viz_options.parallelize_rendering = True
-html_output_file,image_db = visualize_db.visualize_db(db_path=output_file,
-                                                        output_dir=os.path.join(base_directory,'preview'),
-                                                        image_base_dir=image_directory,
-                                                        options=viz_options)
-os.startfile(html_output_file)

data_management/importers/filenames_to_json.py

@@ -1,385 +0,0 @@
-"""
-
- filenames_to_json.py
-
- Take a directory of images in which species labels are encoded by folder
- names, and produces a COCO-style .json file 
- 
-"""
-
-#%% Constants and imports
-
-import json
-import io
-import os
-import uuid
-import csv
-import warnings
-import datetime
-from PIL import Image
-
-from md_utils.path_utils import find_images
-
-# ignoring all "PIL cannot read EXIF metainfo for the images" warnings
-warnings.filterwarnings("ignore", "(Possibly )?corrupt EXIF data", UserWarning)
-# Metadata Warning, tag 256 had too many entries: 42, expected 1
-warnings.filterwarnings("ignore", "Metadata warning", UserWarning)
-
-# Filenames will be stored in the output .json relative to this base dir
-baseDir = r'D:\wildlife_data\bellevue_camera_traps\bellevue_camera_traps.19.06.02.1320'
-outputJsonFilename = os.path.join(baseDir,'bellevue_camera_traps.19.06.02.1320.json')
-outputCsvFilename = os.path.join(baseDir,'bellevue_camera_traps.19.06.02.1320.csv')
-
-# rawClassListFilename = os.path.join(baseDir,'bellevue_camera_traps.19.06.02.1320_classes.csv')
-# classMappingsFilename = os.path.join(baseDir,'bellevue_camera_traps.19.06.02.1320_class_mapping.csv')
-outputEncoding = 'utf-8'
-
-classMappings = {'transitional':'unlabeled','moving':'unlabeled','setup':'unlabeled','blurry':'unlabeled','transitional':'unlabeled','junk':'unlabeled','unknown':'unlabeled'}
-
-bLoadFileListIfAvailable = True
-
-info = {}
-info['year'] = 2019
-info['version'] = '1.0'
-info['description'] = 'Bellevue Camera Traps'
-info['contributor'] = 'Dan Morris'
-info['date_created'] = str(datetime.date.today())
-
-maxFiles = -1
-bReadImageSizes = False
-bUseExternalRemappingTable = False
-
-
-#%% Enumerate files, read image sizes
-
-# Each element will be a list of relative path/full path/width/height
-fileInfo = []
-nonImages = []
-nFiles = 0
-
-if bLoadFileListIfAvailable and os.path.isfile(outputCsvFilename):
-    
-    print('Loading file list from {}'.format(outputCsvFilename))
-    
-    with open(outputCsvFilename,'r') as f:
-        reader = csv.reader(f)
-        csvInfo = list(list(item) for item in csv.reader(f, delimiter=','))
-    
-    for iRow in range(len(csvInfo)):
-        csvInfo[iRow][2] = int(csvInfo[iRow][2])
-        csvInfo[iRow][3] = int(csvInfo[iRow][3])
-    
-    fileInfo = csvInfo
-    
-    print('Finished reading list of {} files'.format(len(fileInfo)))
-    
-else:
-        
-    print('Enumerating files from {} to {}'.format(baseDir,outputCsvFilename))
-    
-    image_files = find_images(baseDir,bRecursive=True)
-    print('Enumerated {} images'.format(len(image_files)))
-    
-    with io.open(outputCsvFilename, "w", encoding=outputEncoding) as outputFileHandle:    
-    
-        for fname in image_files:
-      
-            nFiles = nFiles + 1
-            if maxFiles >= 0 and nFiles > maxFiles:            
-                print('Warning: early break at {} files'.format(maxFiles))
-                break
-            
-            fullPath = fname
-            relativePath = os.path.relpath(fullPath,baseDir)
-             
-            if maxFiles >= 0:
-                print(relativePath)
-        
-            h = -1
-            w = -1
-
-            if bReadImageSizes:
-            
-                # Read the image
-                try:
-                
-                    im = Image.open(fullPath)
-                    h = im.height
-                    w = im.width
-                    
-                except:
-                    # Corrupt or not an image
-                    nonImages.append(fullPath)
-                    continue
-            
-            # Store file info
-            imageInfo = [relativePath, fullPath, w, h]
-            fileInfo.append(imageInfo)
-            
-            # Write to output file
-            outputFileHandle.write('"' + relativePath + '"' + ',' + 
-                                   '"' + fullPath + '"' + ',' + 
-                                   str(w) + ',' + str(h) + '\n')
-                                   
-        # ...for each image file
-                
-    # ...csv file output
-        
-    print("Finished writing {} file names to {}".format(nFiles,outputCsvFilename))
-
-# ...if the file list is/isn't available
-    
-    
-#%% Enumerate classes
-
-# Maps classes to counts
-classList = {}
-
-for iRow,row in enumerate(fileInfo):
-        
-    fullPath = row[0]
-    className = os.path.split(os.path.dirname(fullPath))[1]
-    className = className.lower().strip()
-    if className in classList:
-        classList[className] += 1
-    else:
-        classList[className] = 1
-    row.append(className)
-
-classNames = list(classList.keys())
-
-# We like 'empty' to be class 0
-if 'empty' in classNames:
-    classNames.remove('empty')    
-classNames.insert(0,'empty')
-
-print('Finished enumerating {} classes'.format(len(classList)))
-
-
-#%% Assemble dictionaries
-
-images = []
-annotations = []
-categories = []
-
-categoryNameToId = {}
-idToCategory = {}
-imageIdToImage = {}
-
-nextId = 0
-    
-for categoryName in classNames:
-    
-    catId = nextId
-    nextId += 1
-    categoryNameToId[categoryName] = catId
-    newCat = {}
-    newCat['id'] = categoryNameToId[categoryName]
-    newCat['name'] = categoryName
-    newCat['count'] = 0
-    categories.append(newCat) 
-    idToCategory[catId] = newCat
-            
-# ...for each category
-    
-    
-# Each element is a list of relative path/full path/width/height/className
-    
-for iRow,row in enumerate(fileInfo):
-    
-    relativePath = row[0]
-    w = row[2]
-    h = row[3]    
-    className = row[4]  
-    
-    assert className in categoryNameToId
-    categoryId = categoryNameToId[className]
-    
-    im = {}
-    im['id'] = str(uuid.uuid1())
-    im['file_name'] = relativePath
-    im['height'] = h
-    im['width'] = w
-    images.append(im)
-    imageIdToImage[im['id']] = im
-    
-    ann = {}
-    ann['id'] = str(uuid.uuid1())
-    ann['image_id'] = im['id']
-    ann['category_id'] = categoryId
-    annotations.append(ann)
-    
-    cat = idToCategory[categoryId]
-    cat['count'] += 1
-    
-# ...for each image
-
-oldNameToOldId = categoryNameToId
-originalCategories = categories
-
-print('Finished assembling dictionaries')
-
-
-#%% External class mapping
-
-if bUseExternalRemappingTable:
-    
-    assert classMappings is None
-    
-    
-    #%% Write raw class table
-    
-    # cat = categories[0]
-    if os.path.isfile(rawClassListFilename):
-        
-        print('Not over-writing raw class table')
-    
-    else:
-        
-        with io.open(rawClassListFilename, "w", encoding=outputEncoding) as classListFileHandle:
-            for cat in categories:
-                catId = cat['id']
-                categoryName = cat['name']
-                categoryCount = cat['count']
-                classListFileHandle.write(str(catId) + ',"' + categoryName + '",' + str(categoryCount) + '\n')
-        
-        print('Finished writing raw class table')
-    
-    
-    #%% Read the mapped class table
-    
-    classMappings = {}
-    
-    if os.path.isfile(classMappingsFilename):
-    
-        print('Loading file list from {}'.format(classMappingsFilename))
-            
-        with open(classMappingsFilename,'r') as f:
-            reader = csv.reader(f)
-            mappingInfo = list(list(item) for item in csv.reader(f, delimiter=','))
-        
-        for mapping in mappingInfo:
-            assert len(mapping) == 4
-            
-            # id, source, count, target
-            sourceClass = mapping[1]
-            targetClass = mapping[3]
-            assert sourceClass not in classMappings
-            classMappings[sourceClass] = targetClass
-        
-        print('Finished reading list of {} class mappings'.format(len(mappingInfo)))
-        
-else:
-
-#%% Make classMappings contain *all* classes, not just remapped classes
-    
-    # cat = categories[0]
-    for cat in categories:
-        if cat['name'] not in classMappings:
-            classMappings[cat['name']] = cat['name']
-
-      
-#%% Create new class list
-    
-categories = []
-categoryNameToId = {}
-oldIdToNewId = {}
-
-# Start at 1, explicitly assign 0 to "empty"
-nextCategoryId = 1
-for sourceClass in classMappings:
-    targetClass = classMappings[sourceClass]
-    
-    if targetClass not in categoryNameToId:
-        
-        if targetClass == 'empty':
-            categoryId = 0
-        else:
-            categoryId = nextCategoryId
-            nextCategoryId = nextCategoryId + 1
-            
-        categoryNameToId[targetClass] = categoryId
-        newCat = {}
-        newCat['id'] = categoryId
-        newCat['name'] = targetClass
-        newCat['count'] = 0
-        
-        if targetClass == 'empty':
-            categories.insert(0,newCat)
-        else:
-            categories.append(newCat)
-
-    else:
-        
-        categoryId = categoryNameToId[targetClass]
-    
-    # One-off issue with character encoding
-    if sourceClass == 'human':
-        sourceClass = 'human'
-        
-    assert sourceClass in oldNameToOldId
-    oldId = oldNameToOldId[sourceClass]
-    oldIdToNewId[oldId] = categoryId
-
-categoryIdToCat = {}
-for cat in categories:
-    categoryIdToCat[cat['id']] = cat
-    
-print('Mapped {} original classes to {} new classes'.format(len(originalCategories),len(categories)))
-
-
-#%% Re-map annotations
-            
-# ann = annotations[0]            
-for ann in annotations:
-
-    ann['category_id'] = oldIdToNewId[ann['category_id']]
-    
-    
-#%% Write output .json
-
-data = {}
-data['info'] = info
-data['images'] = images
-data['annotations'] = annotations
-data['categories'] = categories
-
-json.dump(data, open(outputJsonFilename,'w'), indent=4)
-
-print('Finished writing json to {}'.format(outputJsonFilename))
-
-
-#%% Utilities
-
-if False:
-    
-    #%% 
-    # Find images with a particular tag
-    className = 'hum'
-    matches = []
-    assert className in categoryNameToId
-    catId = categoryNameToId[className]
-    for ann in annotations:
-        if ann['category_id'] == catId:
-            imageId = ann['image_id']
-            im = imageIdToImage[imageId]
-            matches.append(im['file_name'])
-    print('Found {} matches'.format(len(matches)))
-    
-    os.startfile(os.path.join(baseDir,matches[0]))
-    
-    
-    #%% Randomly sample annotations
-    
-    import random
-    nAnnotations = len(annotations)
-    iAnn = random.randint(0,nAnnotations)
-    ann = annotations[iAnn]
-    catId = ann['category_id']
-    imageId = ann['image_id']
-    im = imageIdToImage[imageId]
-    fn = os.path.join(baseDir,im['file_name'])
-    cat = categoryIdToCat[catId]
-    className = cat['name']
-    print('This should be a {}'.format(className))
-    os.startfile(fn)