megadetector 5.0.9__py3-none-any.whl → 5.0.11__py3-none-any.whl

data_management/importers/auckland_doc_to_json.py

@@ -1,200 +0,0 @@
-"""
-
- auckland_doc_to_json.py
-
- Convert Auckland DOC data set to COCO camera traps format.  This was
- for a training data set where class names were encoded in path names.
-
-"""
-
-#%% Constants and imports
-
-import json
-import os
-import uuid
-import datetime
-from tqdm import tqdm
-
-from md_visualization import visualize_db
-from data_management.databases import integrity_check_json_db
-from md_utils.path_utils import find_images, split_path, insert_before_extension
-
-# Filenames will be stored in the output .json relative to this base dir
-input_base_dir = 'y:\\'
-output_base_dir = r'f:\auckland-doc'
-output_json_filename = os.path.join(output_base_dir, 'auckland-doc-Maukahuka_Auckland_Island.json')
-
-assert os.path.isdir(input_base_dir)
-os.makedirs(output_base_dir,exist_ok=True)
-
-output_encoding = 'utf-8'
-read_image_sizes = True
-
-info = {}
-info['year'] = 2019
-info['version'] = '1.0'
-info['description'] = 'Auckaland DOC Camera Traps'
-info['contributor'] = 'Auckland DOC'
-info['date_created'] = str(datetime.date.today())
-
-
-#%% Enumerate files
-
-print('Enumerating files from {}'.format(input_base_dir))
-image_files = find_images(input_base_dir, bRecursive=True)
-print('Enumerated {} images'.format(len(image_files)))
-
-
-#%% Assemble dictionaries
-
-images = []
-image_id_to_image = {}
-annotations = []
-categories = []
-
-category_name_to_category = {}
-category_id_to_category = {}
-
-# Force the empty category to be ID 0
-empty_category = {}
-empty_category['name'] = 'empty'
-empty_category['id'] = 0
-category_id_to_category[0] = empty_category
-categories.append(empty_category)
-next_id = 1
-
-behaviors = set()
-
-# fn = image_files[0]; print(fn)
-for fn in tqdm(image_files):
-
-    # Typically y:\Maukahuka_Auckland_Island\1_Training\Winter_Trial_2019\cat\cat\eat\20190903_IDdY_34_E3_tmp_201908240051.JPG
-    relative_path = os.path.relpath(fn,input_base_dir)
-    tokens = split_path(fn)
-    assert tokens[1] == 'Maukahuka_Auckland_Island'
-    
-    trainval_split = tokens[2]
-    assert trainval_split in ['1_Training','2_Testing']
-    
-    # This data set has two top-level folders, "1_Training" (which has class names encoded
-    # in paths) and "2_Testing" (which has no class information).
-    if trainval_split == '2_Testing':
-        category_name = 'test'
-    else:
-        category_name = tokens[-3]
-        if category_name.startswith('2_'):
-            category_name = category_name.replace('2_', '')
-        category_name = category_name.lower().strip()
-
-    if category_name not in category_name_to_category:        
-
-        category_id = next_id
-        next_id += 1
-        category = {}
-        category['id'] = category_id
-        category['name'] = category_name
-        category['count'] = 0
-        categories.append(category)
-        category_name_to_category[category_name] = category
-        category_id_to_category[category_id] = category
-
-    else:
-        
-        category = category_name_to_category[category_name]
-        
-    category_id = category['id']
-    
-    category['count'] += 1
-    behavior = None
-    if (category_name) != 'test':
-        behavior = fn.split('\\')[-2]
-        behaviors.add(behavior)            
-
-    im = {}
-    im['id'] = str(uuid.uuid1())
-    im['file_name'] = relative_path
-    image_id_to_image[im['id']] = im
-    
-    images.append(im)
-    
-    ann = {}
-    
-    ann['id'] = str(uuid.uuid1())
-    ann['image_id'] = im['id']
-    ann['category_id'] = category_id
-    if behavior is not None:
-        ann['behavior'] = behavior
-    annotations.append(ann)
-
-# ...for each image
-    
-
-#%% Write output .json
-
-data = {}
-data['info'] = info
-data['images'] = images
-data['annotations'] = annotations
-data['categories'] = categories
-
-json.dump(data, open(output_json_filename, 'w'), indent=2)
-print('Finished writing json to {}'.format(output_json_filename))
-
-
-#%% Write train/test .jsons
-
-train_images = []; test_images = []
-train_annotations = []; test_annotations = []
-
-for ann in tqdm(annotations):
-    category_id = ann['category_id']
-    image_id = ann['image_id']
-    category_name = category_id_to_category[category_id]['name']
-    im = image_id_to_image[image_id]
-    if category_name == 'test':
-        test_images.append(im)
-        test_annotations.append(ann)
-    else:
-        train_images.append(im)
-        train_annotations.append(ann)
-
-train_fn = insert_before_extension(output_json_filename,'train')
-test_fn = insert_before_extension(output_json_filename,'test')
-
-data['images'] = train_images
-data['annotations'] = train_annotations
-json.dump(data, open(train_fn, 'w'), indent=2)
-
-data['images'] = test_images
-data['annotations'] = test_annotations
-json.dump(data, open(test_fn, 'w'), indent=2)
-
-    
-#%% Validate .json files
-
-options = integrity_check_json_db.IntegrityCheckOptions()
-options.baseDir = input_base_dir
-options.bCheckImageSizes = False
-options.bCheckImageExistence = True
-options.bFindUnusedImages = True
-
-sorted_categories, data, _ = integrity_check_json_db.integrity_check_json_db(output_json_filename, options)
-sorted_categories, data, _ = integrity_check_json_db.integrity_check_json_db(train_fn, options)
-sorted_categories, data, _ = integrity_check_json_db.integrity_check_json_db(test_fn, options)
-
-
-#%% Preview labels
-
-viz_options = visualize_db.DbVizOptions()
-viz_options.num_to_visualize = 2000
-viz_options.trim_to_images_with_bboxes = False
-viz_options.add_search_links = False
-viz_options.sort_by_filename = False
-viz_options.parallelize_rendering = True
-viz_options.classes_to_exclude = ['test']
-html_output_file, image_db = visualize_db.visualize_db(db_path=output_json_filename,
-                                                         output_dir=os.path.join(
-                                                         output_base_dir, 'preview'),
-                                                         image_base_dir=input_base_dir,
-                                                         options=viz_options)
-os.startfile(html_output_file)

data_management/importers/awc_to_json.py

@@ -1,189 +0,0 @@
-"""
-
- awc_to_json.py
-
- Convert a particular .csv file from Australian Wildlife Conservancy to CCT format.
-
-"""
-
-#%% Constants and environment
-
-import pandas as pd
-import uuid
-import json
-import time
-from tqdm import tqdm
-import humanfriendly
-import os
-import PIL
-from md_visualization import visualize_db
-from md_utils import path_utils
-
-input_metadata_file = r"D:\wildlife_data\awc\awc_imageinfo.csv"
-output_file = r"D:\wildlife_data\awc\awc_imageinfo.json"
-image_base = r"D:\wildlife_data\awc"
-preview_base = r"D:\wildlife_data\awc\label_preview"
-
-filename_replacements = {'D:\\Wet Tropics':'WetTropics'}
-category_mappings = {'none':'empty'}
-
-
-#%% Read source data
-
-input_metadata = pd.read_csv(input_metadata_file)
-
-print('Read {} columns and {} rows from metadata file'.format(len(input_metadata.columns),
-      len(input_metadata)))
-
-
-#%% Main loop over labels
-
-startTime = time.time()
-
-relativePathToImage = {}
-
-images = []
-annotations = []
-categoryIDToCategories = {}
-missingFiles = []
-
-duplicateImageIDs = set()
-
-# Force the empty category to be ID 0
-emptyCat = {}
-emptyCat['name'] = 'empty'
-emptyCat['id'] = 0
-categoryIDToCategories[0] = emptyCat
-
-# iRow = 0; row = input_metadata.iloc[iRow]
-for iRow,row in tqdm(input_metadata.iterrows(),total=len(input_metadata)):
-    
-    # ImageID,FileName,FilePath,SpeciesID,CommonName
-    imageID = str(row['ImageID'])
-    fn = row['FileName']
-    for k in filename_replacements:
-        dirName = row['FilePath'].replace(k,filename_replacements[k])
-    relativePath = os.path.join(dirName,fn)
-    
-    # This makes an assumption of one annotation per image, which happens to be
-    # true in this data set.
-    if relativePath in relativePathToImage:
-
-        im = relativePathToImage[relativePath]
-        assert im['id'] == imageID
-        duplicateImageIDs.add(imageID)
-            
-    else:
-        im = {}
-        im['id'] = imageID
-        im['file_name'] = relativePath
-        im['seq_id'] = '-1'
-        images.append(im)
-        relativePathToImage[relativePath] = im
-        
-        fullPath = os.path.join(image_base,relativePath)
-        
-        if not os.path.isfile(fullPath):
-            
-            missingFiles.append(fullPath)
-        
-        else:
-            # Retrieve image width and height
-            pilImage = PIL.Image.open(fullPath)
-            width, height = pilImage.size
-            im['width'] = width
-            im['height'] = height
-
-    categoryName = row['CommonName'].lower()
-    if categoryName in category_mappings:
-        categoryName = category_mappings[categoryName]
-        
-    categoryID = row['SpeciesID']
-    assert isinstance(categoryID,int)
-    
-    if categoryID not in categoryIDToCategories:
-        category = {}
-        category['name'] = categoryName
-        category['id'] = categoryID
-        categoryIDToCategories[categoryID] = category
-    else:
-        assert categoryIDToCategories[categoryID]['name'] == categoryName
-    
-    # Create an annotation
-    ann = {}
-    
-    # The Internet tells me this guarantees uniqueness to a reasonable extent, even
-    # beyond the sheer improbability of collisions.
-    ann['id'] = str(uuid.uuid1())
-    ann['image_id'] = im['id']    
-    ann['category_id'] = categoryID
-    
-    annotations.append(ann)
-    
-categories = list(categoryIDToCategories.values())
-
-elapsed = time.time() - startTime
-print('Finished verifying file loop in {}, {} images, {} missing images, {} repeat labels'.format(
-        humanfriendly.format_timespan(elapsed), len(images), len(missingFiles), len(duplicateImageIDs)))    
-
-
-#%% Check for images that aren't included in the metadata file
-
-# Enumerate all images
-# list(relativePathToImage.keys())[0]
-
-imageFullPaths = path_utils.find_images(image_base,bRecursive=True)
-unmatchedFiles = []
-
-for iImage,imagePath in enumerate(imageFullPaths):
-    
-    fn = os.path.relpath(imagePath,image_base)    
-    if fn not in relativePathToImage:
-        unmatchedFiles.append(fn)
-
-print('Finished checking {} images to make sure they\'re in the metadata, found {} mismatches'.format(
-        len(imageFullPaths),len(unmatchedFiles)))
-
-
-#%% Create info struct
-
-info = {}
-info['year'] = 2019
-info['version'] = 1
-info['description'] = 'COCO style database'
-info['secondary_contributor'] = 'Converted to COCO .json by Dan Morris'
-info['contributor'] = ''
-
-
-#%% Write output
-
-json_data = {}
-json_data['images'] = images
-json_data['annotations'] = annotations
-json_data['categories'] = categories
-json_data['info'] = info
-json.dump(json_data, open(output_file,'w'), indent=4)
-
-print('Finished writing .json file with {} images, {} annotations, and {} categories'.format(
-        len(images),len(annotations),len(categories)))
-
-
-#%% Validate the database's integrity
-
-from data_management.databases import integrity_check_json_db
-
-options = integrity_check_json_db.IntegrityCheckOptions()
-sortedCategories,data = integrity_check_json_db.integrity_check_json_db(output_file, options)
-
-    
-#%% Render a bunch of images to make sure the labels got carried along correctly
-
-bbox_db_path = output_file
-output_dir = preview_base
-
-options = visualize_db.BboxDbVizOptions()
-options.num_to_visualize = 1000
-options.sort_by_filename = False
-
-htmlOutputFile = visualize_db.visualize_db(bbox_db_path,output_dir,image_base,options)
-

data_management/importers/bellevue_to_json.py

@@ -1,273 +0,0 @@
-"""
-
- bellevue_to_json.py
-
- "Bellevue Camera Traps" is the rather unremarkable camera trap data set
- used by one of the repo's maintainers for testing.  It's organized as:
-
- approximate_date/[loose_camera_specifier/]/species    
-
- E.g.:
-    
- "2018.03.30\coyote\DSCF0091.JPG"
- "2018.07.18\oldcam\empty\DSCF0001.JPG"
-
-"""
-
-#%% Constants and imports
-
-import json
-import os
-import uuid
-import datetime
-
-from PIL import Image
-from PIL.ExifTags import TAGS
-from tqdm import tqdm
-
-from md_utils.path_utils import find_images
-
-# Filenames will be stored in the output .json relative to this base dir
-base_dir = r'C:\temp\camera_trap_images_no_people'
-output_base = r'c:\temp\previews'
-output_filename = os.path.join(base_dir,'bellevue_camera_traps.{}.json'.format(str(datetime.date.today())))
-
-class_mappings = {'transitional':'unlabeled','moving':'unlabeled','setup':'unlabeled','blurry':'unlabeled','transitional':'unlabeled','junk':'unlabeled','unknown':'unlabeled','blurry':'unlabeled'}
-class_mappings['dan'] = 'human'
-class_mappings['dan_and_dog'] = 'human,dog'
-class_mappings['dan and dog'] = 'human,dog'
-class_mappings['unknown'] = 'unknown animal'
-class_mappings['racoon'] = 'raccoon'
-
-
-info = {}
-info['year'] = 2020
-info['version'] = '2.0'
-info['description'] = 'Bellevue Camera Traps'
-info['contributor'] = 'Dan Morris'
-info['date_created'] = str(datetime.date.today())
-
-max_files = -1
-
-max_seconds_within_sequence = 10.0
-
-assert os.path.isdir(base_dir)
-
-#%% Exif functions
-
-def get_exif_tags(fn=None,im=None):
-    
-    assert (fn is not None) or (im is not None)
-    ret = {}
-    if im is None:
-        im = Image.open(fn)
-    info = im._getexif()
-    for tag, value in info.items():
-        decoded = TAGS.get(tag, tag)
-        ret[decoded] = value
-        
-    return ret
-
-
-#%% Enumerate files, create image/annotation/category info
-
-annotations = []
-
-category_name_to_category = {}
-
-# Force the empty category to be ID 0
-empty_category = {}
-empty_category['id'] = 0
-empty_category['name'] = 'empty'
-category_name_to_category['empty'] = empty_category
-next_category_id = 1
-
-# Keep track of unique camera folders
-camera_folders = set()
-
-# Each element will be a dictionary with fields:
-#
-# relative_path, width, height, datetime    
-images = []
-non_image_files = []
-
-print('Enumerating files from {}'.format(base_dir))
-
-image_files = find_images(base_dir,recursive=True)
-print('Enumerated {} images'.format(len(image_files)))
-
-# fname = image_files[0]
-for fname in tqdm(image_files):
-  
-    if max_files >= 0 and len(images) > max_files:            
-        print('Warning: early break at {} files'.format(max_files))
-        break
-    
-    full_path = fname
-    relative_path = os.path.relpath(full_path,base_dir)
-     
-    try:
-        im = Image.open(full_path)
-        h = im.height
-        w = im.width
-        tags = get_exif_tags(None,im)
-        s = tags['DateTimeOriginal']
-        dt = datetime.datetime.strptime(s,'%Y:%m:%d %H:%M:%S')
-    except:
-        # Corrupt or not an image
-        print('Warning: could not read {}'.format(fname))
-        non_image_files.append(full_path)
-        continue
-    
-    # Store file info
-    image_info = {}
-    image_info['file_name'] = relative_path
-    image_info['width'] = w
-    image_info['height'] = h
-    image_info['datetime'] = dt
-    image_info['location'] = 'unknown'
-    image_info['id'] = str(uuid.uuid4())
-
-    images.append(image_info)
-    
-    # E.g. 2018.03.30/coyote/DSCF0091.JPG
-    relative_path = image_info['file_name'].replace('\\','/')
-    tokens = relative_path.split('/')
-    camera_path_tokens = tokens[0:-2]
-    camera_path = '/'.join(camera_path_tokens)
-    camera_folders.add(camera_path)
-    image_info['camera_path'] = camera_path
-    
-    category_name = tokens[-2].lower()
-    if category_name in class_mappings:
-        category_name = class_mappings[category_name]
-        
-    if category_name not in category_name_to_category:
-        category = {}
-        category['id'] = next_category_id
-        category['name'] = category_name
-        next_category_id = next_category_id + 1
-        category_name_to_category[category_name] = category
-    else:
-        category = category_name_to_category[category_name]
-    
-    annotation = {}
-    annotation['sequence_level_annotation'] = False
-    annotation['id'] = str(uuid.uuid4())
-    annotation['category_id'] = category['id']
-    annotation['image_id'] = image_info['id']
-    annotations.append(annotation)
-        
-# ...for each image file
-
-assert len(annotations) == len(images)
-
-categories = list(category_name_to_category.values())
-
-
-#%% Synthesize sequence information        
-
-print('Found {} camera folders'.format(len(camera_folders)))
-
-camera_folders = list(camera_folders)
-
-all_sequences = set()
-
-# Sort images by time within each folder
-# camera_path = camera_folders[0]
-for i_camera,camera_path in enumerate(camera_folders):
-    
-    images_this_camera = [im for im in images if im['camera_path'] == camera_path]
-    sorted_images_this_camera = sorted(images_this_camera, key = lambda im: im['datetime'])
-    
-    current_sequence_id = None
-    next_sequence_index = 0
-    previous_datetime = None
-        
-    # previous_datetime = sorted_images_this_camera[0]['datetime']
-    # im = sorted_images_this_camera[1]
-    for im in sorted_images_this_camera:
-        
-        if previous_datetime is None:
-            delta = None
-        else:
-            delta = (im['datetime'] - previous_datetime).total_seconds()
-        
-        # Start a new sequence if necessary
-        if delta is None or delta > max_seconds_within_sequence:
-            next_sequence_index = 0
-            current_sequence_id = str(uuid.uuid4())
-            all_sequences.add(current_sequence_id)
-            
-        im['seq_id'] = current_sequence_id
-        im['seq_num_frames'] = None
-        im['frame_num'] = next_sequence_index
-        next_sequence_index = next_sequence_index + 1
-        previous_datetime = im['datetime']
-    
-    # ...for each image in this camera
-
-# ...for each camera
-
-print('Created {} sequences from {} images'.format(len(all_sequences),len(images)))
-
-# Fill in seq_num_frames
-num_frames_per_sequence = {}
-for seq_id in all_sequences:
-    images_this_sequence = [im for im in images if im['seq_id'] == seq_id]
-    num_frames_per_sequence[seq_id] = len(images_this_sequence)
-    for im in images_this_sequence:
-        im['seq_num_frames'] = len(images_this_sequence)
-
-
-#%% A little cleanup 
-
-for im in tqdm(images):
-    if 'camera_path' in im:
-        del im['camera_path']
-    if not isinstance(im['datetime'],str):
-        im['datetime'] = str(im['datetime'])
-    
-    
-#%% Write output .json
-
-data = {}
-data['info'] = info
-data['images'] = images
-data['annotations'] = annotations
-data['categories'] = categories
-
-json.dump(data, open(output_filename,'w'), indent=1)
-
-print('Finished writing json to {}'.format(output_filename))
-
-
-#%% Validate data
-
-from data_management.databases import integrity_check_json_db
-
-options = integrity_check_json_db.IntegrityCheckOptions()
-options.baseDir = base_dir
-options.bCheckImageSizes = False
-options.bCheckImageExistence = True
-options.bFindUnusedImages = False
-    
-sorted_categories  = integrity_check_json_db.integrity_check_json_db(output_filename,options)
-
-
-#%% Label previews
-
-from md_visualization import visualize_db
-
-viz_options = visualize_db.DbVizOptions()
-viz_options.num_to_visualize = None
-viz_options.parallelize_rendering_n_cores = 8
-viz_options.parallelize_rendering = True
-viz_options.trim_to_images_with_bboxes = False
-viz_options.add_search_links = True
-viz_options.sort_by_filename = False
-html_output_file,image_db = visualize_db.visualize_db(output_filename,
-                                                        os.path.join(output_base,'preview'),
-                                                        base_dir,viz_options)
-os.startfile(html_output_file)
-