megadetector 5.0.28__py3-none-any.whl → 10.0.0__py3-none-any.whl

megadetector/data_management/importers/jb_csv_to_json.py

@@ -1,150 +0,0 @@
-"""
-
- jb_csv_to_json.py
-
- Convert a particular .csv file to CCT format.  Images were not available at
- the time I wrote this script, so this is much shorter than other scripts 
- in this folder.
-
-"""
-
-#%% Constants and environment
-
-import pandas as pd
-import uuid
-import json
-
-input_metadata_file = r'd:\temp\pre_bounding_box.csv'
-output_file = r'd:\temp\pre_bounding_box.json'
-filename_col = 'filename'
-label_col = 'category'
-
-
-#%% Read source data
-
-input_metadata = pd.read_csv(input_metadata_file)
-
-print('Read {} columns and {} rows from metadata file'.format(len(input_metadata.columns),
-      len(input_metadata)))
-
-
-#%% Confirm filename uniqueness (this data set has one label per image)
-
-imageFilenames = input_metadata[filename_col]
-
-duplicateRows = []
-filenamesToRows = {}
-
-# Build up a map from filenames to a list of rows, checking image existence as we go
-for iFile,fn in enumerate(imageFilenames):
-    
-    if (fn in filenamesToRows):
-        duplicateRows.append(iFile)
-        filenamesToRows[fn].append(iFile)
-    else:
-        filenamesToRows[fn] = [iFile]
-
-assert(len(duplicateRows) == 0)    
-    
-
-#%% Create CCT dictionaries
-
-images = []
-annotations = []
-
-# Map categories to integer IDs (that's what COCO likes)
-nextCategoryID = 1
-categories = []
-categoryNamesToCategories = {}
-
-cat = {}
-cat['name'] = 'empty'
-cat['id'] = 0
-categories.append(cat)    
-categoryNamesToCategories['empty'] = cat
-
-# For each image
-#
-# Because in practice images are 1:1 with annotations in this data set,
-# this is also a loop over annotations.
-
-# imageName = imageFilenames[0]
-for imageName in imageFilenames:
-    
-    rows = filenamesToRows[imageName]
-    
-    # As per above, this is convenient and appears to be true; asserting to be safe
-    assert(len(rows) == 1)    
-    iRow = rows[0]
-    
-    row = input_metadata.iloc[iRow]
-    
-    im = {}
-    # Filenames look like "290716114012001a1116.jpg"
-    im['id'] = imageName.split('.')[0]
-    im['file_name'] = imageName    
-    im['seq_id'] = '-1'
-    
-    images.append(im)
-    
-    categoryName = row[label_col].lower()
-    
-    # Have we seen this category before?
-    if categoryName in categoryNamesToCategories:
-        categoryID = categoryNamesToCategories[categoryName]['id']
-    else:
-        cat = {}
-        categoryID = nextCategoryID 
-        cat['name'] = categoryName
-        cat['id'] = nextCategoryID 
-        categories.append(cat)    
-        categoryNamesToCategories[categoryName] = cat
-        nextCategoryID += 1
-        
-    # Create an annotation
-    ann = {}
-    
-    # The Internet tells me this guarantees uniqueness to a reasonable extent, even
-    # beyond the sheer improbability of collisions.
-    ann['id'] = str(uuid.uuid1())
-    ann['image_id'] = im['id']    
-    ann['category_id'] = categoryID
-    
-    annotations.append(ann)
-    
-# ...for each image
-        
-print('Finished creating dictionaries')
-
-
-#%% Create info struct
-
-info = {}
-info['year'] = 2019
-info['version'] = 1
-info['description'] = 'COCO style database'
-info['secondary_contributor'] = 'Converted to COCO .json by Dan Morris'
-info['contributor'] = ''
-
-
-#%% Write output
-
-json_data = {}
-json_data['images'] = images
-json_data['annotations'] = annotations
-json_data['categories'] = categories
-json_data['info'] = info
-json.dump(json_data, open(output_file,'w'), indent=4)
-
-print('Finished writing .json file with {} images, {} annotations, and {} categories'.format(
-        len(images),len(annotations),len(categories)))
-
-
-#%% Validate
-
-from megadetector.data_management.databases import integrity_check_json_db
-
-options = integrity_check_json_db.IntegrityCheckOptions()
-sortedCategories,data = integrity_check_json_db.integrity_check_json_db(output_file, options)
-
-

megadetector/data_management/importers/mcgill_to_json.py

@@ -1,250 +0,0 @@
-"""
-
- mcgill_to_json.py
-
- Convert the .csv file provided for the McGill test data set to a 
- COCO-camera-traps .json file
-
-"""
-
-#%% Constants and environment
-
-import pandas as pd
-import os
-import glob
-import json
-import uuid
-import time
-import ntpath
-import humanfriendly
-import PIL
-import math
-
-baseDir = r'D:\wildlife_data\mcgill_test'
-input_metadata_file = os.path.join(baseDir, 'dan_500_photos_metadata.csv')
-output_file = os.path.join(baseDir, 'mcgill_test.json')
-image_directory = baseDir
-
-assert(os.path.isdir(image_directory))
-assert(os.path.isfile(input_metadata_file))
-
-
-#%% Read source data
-
-input_metadata = pd.read_csv(input_metadata_file)
-
-print('Read {} columns and {} rows from metadata file'.format(len(input_metadata.columns),
-      len(input_metadata)))
-
-
-#%% Map filenames to rows, verify image existence
-
-# Create an additional column for concatenated filenames
-input_metadata['relative_path'] = ''
-input_metadata['full_path'] = ''
-
-startTime = time.time()
-
-# Maps relative filenames to rows
-filenamesToRows = {}
-
-duplicateRows = []
-
-# Build up a map from filenames to a list of rows, checking image existence as we go
-# row = input_metadata.iloc[0]
-for iFile,row in input_metadata.iterrows():
-
-    relativePath = os.path.join(row['site'],row['date_range'],str(row['camera']),
-                                str(row['folder']),row['filename'])
-    fullPath = os.path.join(baseDir,relativePath)
-    
-    if (relativePath in filenamesToRows):
-        duplicateRows.append(iFile)
-        filenamesToRows[relativePath].append(iFile)
-    else:
-        filenamesToRows[relativePath] = [iFile]
-        assert(os.path.isfile(fullPath))
-
-    row['relative_path'] = relativePath
-    row['full_path'] = fullPath
-    
-    input_metadata.iloc[iFile] = row
-    
-elapsed = time.time() - startTime
-print('Finished verifying image existence in {}, found {} filenames with multiple labels'.format(
-      humanfriendly.format_timespan(elapsed),len(duplicateRows)))
-
-# I didn't expect this to be true a priori, but it appears to be true, and
-# it saves us the trouble of checking consistency across multiple occurrences
-# of an image.
-assert(len(duplicateRows) == 0)    
-    
-    
-#%% Check for images that aren't included in the metadata file
-
-# Enumerate all images
-imageFullPaths = glob.glob(os.path.join(image_directory,'**/*.JPG'), recursive=True)
-
-for iImage,imagePath in enumerate(imageFullPaths):
-    
-    imageRelPath = ntpath.relpath(imagePath, image_directory)
-    assert(imageRelPath in filenamesToRows)
-
-print('Finished checking {} images to make sure they\'re in the metadata'.format(
-        len(imageFullPaths)))
-
-
-#%% Create CCT dictionaries
-
-# Also gets image sizes, so this takes ~6 minutes
-#
-# Implicitly checks images for overt corruptness, i.e. by not crashing.
-
-images = []
-annotations = []
-categories = []
-
-emptyCategory = {}
-emptyCategory['id'] = 0
-emptyCategory['name'] = 'empty'
-emptyCategory['latin'] = 'empty'
-emptyCategory['count'] = 0
-categories.append(emptyCategory)
-
-# Map categories to integer IDs (that's what COCO likes)
-nextCategoryID = 1
-labelToCategory = {'empty':emptyCategory}
-
-# For each image
-#
-# Because in practice images are 1:1 with annotations in this data set,
-# this is also a loop over annotations.
-
-startTime = time.time()
-
-# row = input_metadata.iloc[0]
-for iFile,row in input_metadata.iterrows():
-
-    relPath = row['relative_path'].replace('\\','/')
-    im = {}
-    # Filenames look like "290716114012001a1116.jpg"
-    im['id'] = relPath.replace('/','_').replace(' ','_')
-    
-    im['file_name'] = relPath
-    
-    im['seq_id'] = -1
-    im['frame_num'] = -1
-    
-    # In the form "001a"
-    im['site']= row['site']
-    
-    # Can be in the form '111' or 's46'
-    im['camera'] = row['camera']
-    
-    # In the form "7/29/2016 11:40"
-    im['datetime'] = row['timestamp']
-    
-    otherFields = ['motion','temp_F','n_present','n_waterhole','n_contact','notes']
-    
-    for s in otherFields:
-        im[s] = row[s]
-        
-    # Check image height and width
-    fullPath = row['full_path']
-    assert(os.path.isfile(fullPath))
-    pilImage = PIL.Image.open(fullPath)
-    width, height = pilImage.size
-    im['width'] = width
-    im['height'] = height
-
-    images.append(im)
-    
-    label = row['species']
-    if not isinstance(label,str):
-        # NaN is the only thing we should see that's not a string        
-        assert math.isnan(label)
-        label = 'empty'
-    else:
-        label = label.lower()
-    
-    latin = row['binomial']
-    if not isinstance(latin,str):
-        # NaN is the only thing we should see that's not a string
-        assert math.isnan(latin)
-        latin = 'empty'
-    else:
-        latin = latin.lower()
-
-    if label == 'empty':
-        if latin != 'empty':
-            latin = 'empty'
-
-    if label == 'unknown':
-        if latin != 'unknown':
-            latin = 'unknown'
-            
-    if label not in labelToCategory:
-        print('Adding category {} ({})'.format(label,latin))
-        category = {}
-        categoryID = nextCategoryID
-        category['id'] = categoryID        
-        nextCategoryID += 1
-        category['name'] = label
-        category['latin'] = latin
-        category['count'] = 1
-        labelToCategory[label] = category
-        categories.append(category)
-    else:
-        category = labelToCategory[label]
-        category['count'] = category['count'] + 1
-        categoryID = category['id']        
-        
-    # Create an annotation
-    ann = {}
-    
-    # The Internet tells me this guarantees uniqueness to a reasonable extent, even
-    # beyond the sheer improbability of collisions.
-    ann['id'] = str(uuid.uuid1())
-    ann['image_id'] = im['id']    
-    ann['category_id'] = categoryID
-    
-    annotations.append(ann)
-    
-# ...for each image
-    
-# Convert categories to a CCT-style dictionary
-
-
-for category in categories:
-    print('Category {}, count {}'.format(category['name'],category['count']))
-    
-elapsed = time.time() - startTime
-print('Finished creating CCT dictionaries in {}'.format(
-      humanfriendly.format_timespan(elapsed)))
-    
-
-#%% Create info struct
-
-info = {}
-info['year'] = 2019
-info['version'] = 1
-info['description'] = 'COCO style database'
-info['secondary_contributor'] = 'Converted to COCO .json by Dan Morris'
-info['contributor'] = 'McGill University'
-
-
-#%% Write output
-
-json_data = {}
-json_data['images'] = images
-json_data['annotations'] = annotations
-json_data['categories'] = categories
-json_data['info'] = info
-json.dump(json_data, open(output_file,'w'), indent=4)
-
-print('Finished writing .json file with {} images, {} annotations, and {} categories'.format(
-        len(images),len(annotations),len(categories)))
-
-
-
-