megadetector 5.0.27__py3-none-any.whl → 5.0.29__py3-none-any.whl

megadetector/data_management/importers/pc_to_json.py

@@ -1,365 +0,0 @@
-"""
-
- pc_to_json.py
-
- Convert a particular collection of .csv files from Parks Canada to CCT format.
-
-"""
-
-#%% Constants and environment
-
-import pandas as pd
-import uuid
-import json
-import time
-
-import numpy as np
-from tqdm import tqdm
-
-import humanfriendly
-import os
-import PIL
-
-from megadetector.data_management.databases import integrity_check_json_db
-from megadetector.data_management.cct_json_utils import IndexedJsonDb
-from megadetector.data_management import cct_json_to_filename_json
-from megadetector.visualization import visualize_db
-from megadetector.utils import path_utils
-
-input_base = r"g:\20190715"
-output_file = r"D:\wildlife_data\parks_canada\pc_20190715.json"
-preview_base = r"D:\wildlife_data\parks_canada\preview"
-
-filename_replacements = {}
-category_mappings = {'':'unlabeled'}
-
-csv_prefix = 'ImageData_Microsoft___'
-
-expected_columns = 'Location,DateImage,TimeImage,Species,Total,Horses,DogsOnLeash,DogsOffLeash,AdultFemale,AdultMale,AdultUnknown,Subadult,YLY,YOY,ImageName'.split(',')
-columns_to_copy = {'Total':'count','Horses':'horses','DogsOnLeash':'dogsonleash','DogsOffLeash':'dogsoffleash',
-                   'AdultFemale':'adultfemale','AdultMale':'adultmale','AdultUnknown':'adultunknown',
-                   'Subadult':'subadult','YLY':'yearling','YOY':'youngofyear'}
-
-retrieve_image_sizes = False
-
-max_num_csvs = -1
-
-db_sampling_scheme = 'preview' # 'labeled','all'
-n_unlabeled_to_sample = -1
-cap_unlabeled_to_labeled = True
-
-
-#%% Read and concatenate source data
-
-# List files
-input_files = os.listdir(input_base)
-
-# List of dataframes, one per .csv file; we'll concatenate later
-all_input_metadata = []
-
-# i_file = 87; fn = input_files[i_file]
-for i_file,fn in enumerate(input_files):
-
-    if max_num_csvs > 0 and len(all_input_metadata) >= max_num_csvs:
-        break
-    
-    if not fn.endswith('.csv'):
-        continue
-    if not fn.startswith(csv_prefix):
-        continue
-    dirname = fn.replace(csv_prefix,'').replace('.csv','')
-    dirfullpath = os.path.join(input_base,dirname)
-    if not os.path.isdir(dirfullpath):
-        dirname = fn.replace(csv_prefix,'').replace('.csv','').replace('  ',' ')
-        dirfullpath = os.path.join(input_base,dirname)    
-    assert(os.path.isdir(dirfullpath))
-    
-    metadata_fullpath = os.path.join(input_base,fn)
-    print('Reading {}'.format(metadata_fullpath))
-    df = pd.read_csv(metadata_fullpath)
-    assert list(df.columns) == expected_columns
-    df['DirName'] = dirname
-    all_input_metadata.append(df)
-
-# Concatenate into a giant data frame
-input_metadata = pd.concat(all_input_metadata)
-
-print('Read {} rows total'.format(len(input_metadata)))
-
-
-#%% List files
-
-print('Listing images...')
-image_full_paths = path_utils.find_images(input_base,bRecursive=True)
-print('Finished listing {} images'.format(len(image_full_paths)))
-
-image_relative_paths = []
-for s in image_full_paths:
-    image_relative_paths.append(os.path.relpath(s,input_base))
-image_relative_paths = set(image_relative_paths)
-
-image_relative_paths_lower = set()
-for s in image_relative_paths:
-    image_relative_paths_lower.add(s.lower())
-    
-    
-#%% Main loop over labels (prep)
-
-start_time = time.time()
-
-relative_path_to_image = {}
-
-images = []
-annotations = []
-category_name_to_category = {}
-missing_files = []
-
-# Force the empty category to be ID 0
-empty_category = {}
-empty_category['name'] = 'empty'
-empty_category['id'] = 0
-category_name_to_category['empty'] = empty_category
-next_category_id = 1
-
-labeled_images = []
-unlabeled_images = []
-
-
-#%% Main loop over labels (loop)
-
-# iRow = 0; row = input_metadata.iloc[iRow]
-for iRow,row in tqdm(input_metadata.iterrows(),total=len(input_metadata)):
-    
-    # ImageID,FileName,FilePath,SpeciesID,CommonName
-    image_id = str(uuid.uuid1())
-    relative_path = os.path.normpath(row['ImageName'])
-    
-    if relative_path not in image_relative_paths:
-        if relative_path.lower() in image_relative_paths_lower:
-            print('Warning: lower-case version of {} in path list'.format(relative_path))
-        else:
-            missing_files.append(relative_path)
-            continue
-        
-    full_path = os.path.join(input_base,relative_path)
-
-    # assert os.path.isfile(full_path)    
- 
-    if relative_path in relative_path_to_image:
-        
-        im = relative_path_to_image[relative_path]
-
-    else:
-        
-        im = {}
-        im['id'] = image_id
-        im['file_name'] = relative_path
-        im['seq_id'] = '-1'
-        im['location'] = row['Location']
-        im['datetime'] = row['DateImage'] + ' ' + row['TimeImage']        
-        
-        images.append(im)
-        relative_path_to_image[relative_path] = im
-        
-        if retrieve_image_sizes:
-        
-            # Retrieve image width and height
-            pil_im = PIL.Image.open(full_path)
-            width, height = pil_im.size
-            im['width'] = width
-            im['height'] = height
-
-    species = row['Species']
-    if isinstance(species,float):
-        assert np.isnan(species)
-        species = 'unlabeled'
-        
-    category_name = species.lower().strip()
-    if category_name in category_mappings:
-        category_name = category_mappings[category_name]
-        
-    if category_name not in category_name_to_category:
-        category = {}
-        category['name'] = category_name
-        category['id'] = next_category_id
-        next_category_id += 1
-        category_name_to_category[category_name] = category
-    else:
-        category = category_name_to_category[category_name]
-        assert category['name'] == category_name
-
-    category_id = category['id']
-
-    if category_name == 'unlabeled':
-        unlabeled_images.append(im)
-    else:
-        labeled_images.append(im)
-        
-    # Create an annotation
-    ann = {}
-    
-    # The Internet tells me this guarantees uniqueness to a reasonable extent, even
-    # beyond the sheer improbability of collisions.
-    ann['id'] = str(uuid.uuid1())
-    ann['image_id'] = im['id']    
-    ann['category_id'] = category_id
-
-    for col in columns_to_copy:
-        ann[columns_to_copy[col]] = row[col]
-        
-    annotations.append(ann)
-    
-# ...for each image
-
-categories = list(category_name_to_category.values())
-
-elapsed = time.time() - start_time
-print('Finished verifying file loop in {}, {} matched images, {} missing images, {} unlabeled images'.format(
-        humanfriendly.format_timespan(elapsed), len(images), len(missing_files), len(unlabeled_images))) 
-
-
-#%% See what's up with missing files
-
-dirnames = set()
-# s = list(image_relative_paths)[0]
-for s in image_relative_paths:
-    image_dir = os.path.dirname(s)
-    dirnames.add(image_dir)
-        
-missing_images_with_missing_dirs = []
-missing_images_with_non_missing_dirs = []
-
-missing_dirs = set()
-
-# s = missing_files[0]
-for s in missing_files:
-    assert s not in image_relative_paths
-    dirname = os.path.dirname(s)
-    if dirname not in dirnames:
-        missing_images_with_missing_dirs.append(s)
-        missing_dirs.add(dirname)
-    else:
-        missing_images_with_non_missing_dirs.append(s)
-
-print('Of {} missing files, {} are due to {} missing folders'.format(
-        len(missing_files),len(missing_images_with_missing_dirs),len(missing_dirs)))
-    
-
-#%% Check for images that aren't included in the metadata file
-
-unmatched_files = []
-
-for i_image,relative_path in tqdm(enumerate(image_relative_paths),total=len(image_relative_paths)):
-    
-    if relative_path not in relative_path_to_image:
-        unmatched_files.append(relative_path)
-
-print('Finished checking {} images to make sure they\'re in the metadata, found {} mismatches'.format(
-        len(image_relative_paths),len(unmatched_files)))
-
-
-#%% Sample the database
-
-images_all = images
-annotations_all = annotations
-
-#%%
-
-if db_sampling_scheme == 'all':
-    
-    pass
-
-elif db_sampling_scheme == 'labeled' or db_sampling_scheme == 'preview':
-    
-    json_data = {}
-    json_data['images'] = images
-    json_data['annotations'] = annotations
-    json_data['categories'] = categories
-
-    indexed_db = IndexedJsonDb(json_data)
-    
-    # Collect the images we want
-    sampled_images = []
-    for im in images:
-        classes = indexed_db.get_classes_for_image(im)
-        if 'unlabeled' in classes and len(classes) == 1:
-            pass
-        else:
-            sampled_images.append(im)
-    
-    if db_sampling_scheme == 'preview':
-        n_sample = n_unlabeled_to_sample
-        if n_sample == -1:
-            n_sample = len(labeled_images)
-        if n_sample > len(labeled_images) and cap_unlabeled_to_labeled:
-            n_sample = len(labeled_images)
-        if n_sample > len(unlabeled_images):
-            n_sample = len(unlabeled_images)
-        print('Sampling {} of {} unlabeled images'.format(n_sample,len(unlabeled_images)))
-        from random import sample
-        sampled_images.extend(sample(unlabeled_images,n_sample))
-
-    sampled_annotations = []
-    for im in sampled_images:
-        sampled_annotations.extend(indexed_db.get_annotations_for_image(im))
-    
-    print('Sampling {} of {} images, {} of {} annotations'.format(
-            len(sampled_images),len(images),len(sampled_annotations),len(annotations)))
-  
-    images = sampled_images
-    annotations = sampled_annotations      
-    
-else:
-    
-    raise ValueError('Unrecognized DB sampling scheme {}'.format(db_sampling_scheme))
-
-
-#%% Create info struct
-
-info = {}
-info['year'] = 2019
-info['version'] = 1
-info['description'] = 'COCO style database'
-info['secondary_contributor'] = 'Converted to COCO .json by Dan Morris'
-info['contributor'] = 'Parks Canada'
-
-
-#%% Write output
-
-json_data = {}
-json_data['images'] = images
-json_data['annotations'] = annotations
-json_data['categories'] = categories
-json_data['info'] = info
-json.dump(json_data, open(output_file,'w'), indent=4)
-
-print('Finished writing .json file with {} images, {} annotations, and {} categories'.format(
-        len(images),len(annotations),len(categories)))
-
-
-#%% Validate the database's integrity
-
-json_data = json.load(open(output_file))
-options = integrity_check_json_db.IntegrityCheckOptions()
-sortedCategories,data = integrity_check_json_db.integrity_check_json_db(json_data, options)
-
-    
-#%% Render a bunch of images to make sure the labels got carried along correctly
-
-output_dir = preview_base
-
-options = visualize_db.DbVizOptions()
-options.num_to_visualize = 100
-options.sort_by_filename = False
-# options.classes_to_exclude = ['unlabeled']
-options.classes_to_exclude = None
-
-htmlOutputFile,_ = visualize_db.visualize_db(json_data,output_dir,input_base,options)
-os.startfile(htmlOutputFile)
-
-
-#%% Write out a list of files to annotate
-
-_,file_list = cct_json_to_filename_json.convertJsonToStringList(output_file,prepend="20190715/")
-os.startfile(file_list)
-

megadetector/data_management/importers/plot_wni_giraffes.py

@@ -1,123 +0,0 @@
-"""
-
- plot_wni_giraffes.py
-
- Plot keypoints on a random sample of images from the wni-giraffes data set.
-
-"""
-
-#%% Constants and imports
-
-import os
-import json
-import random
-
-from PIL import Image
-from PIL import ImageDraw
-from tqdm import tqdm
-
-input_file = r"G:\data_staging\wni-out\wni_giraffes_train.json"
-image_base = r"G:\data_staging\wni-out\images"
-output_base = r"G:\data_staging\wni-out\test-plots"
-os.makedirs(output_base,exist_ok=True)
-
-tool_colors = ['red','green','blue','magenta']
-use_fancy_ellipses = True
-draw_individual_samples = False
-
-median_radius = 20
-median_linewidth = 8
-
-sample_radius = 10
-
-n_images_to_plot = 100
-
-
-#%% Load and select data
-
-with open(input_file,'r') as f:
-    d = json.load(f)
-annotations = d['annotations']
-print(d['info'])
-
-short_tool_names = list(d['info']['tool_names'].keys())
-annotations_to_plot = random.sample(annotations,n_images_to_plot)
-
-
-#%% Support functions
-
-# https://stackoverflow.com/questions/32504246/draw-ellipse-in-python-pil-with-line-thickness
-def draw_fancy_ellipse(image, x, y, radius, width=1, outline='white', antialias=4):
-
-    bounds = (x-radius,y-radius,x+radius,y+radius)
-    
-    # Use a single channel image (mode='L') as mask.
-    # The size of the mask can be increased relative to the input image
-    # to get smoother looking results. 
-    mask = Image.new(
-        size=[int(dim * antialias) for dim in image.size],
-        mode='L', color='black')
-    draw = ImageDraw.Draw(mask)
-
-    # draw outer shape in white (color) and inner shape in black (transparent)
-    for offset, fill in (width/-2.0, 'white'), (width/2.0, 'black'):
-        left, top = [(value + offset) * antialias for value in bounds[:2]]
-        right, bottom = [(value - offset) * antialias for value in bounds[2:]]
-        draw.ellipse([left, top, right, bottom], fill=fill)
-
-    # downsample the mask using PIL.Image.LANCZOS 
-    # (a high-quality downsampling filter).
-    mask = mask.resize(image.size, Image.LANCZOS)
-    
-    # paste outline color to input image through the mask
-    image.paste(outline, mask=mask)
-
-    
-def draw_ellipse(image, x, y, radius, linewidth, color_index, use_imagedraw=False):
-    
-    if use_imagedraw:
-        draw_fancy_ellipse(image, x, y, radius=radius, width=linewidth, outline=tool_colors[color_index])
-    else:
-        draw = ImageDraw.Draw(image)
-        bounds = (x-radius,y-radius,x+radius,y+radius)
-        draw.ellipse(bounds, fill=tool_colors[color_index])
-            
-        
-#%% Plot some images
-
-# ann = annotations_to_plot[0]
-for ann in tqdm(annotations_to_plot):
-    
-    input_path = os.path.join(image_base,ann['filename'])    
-    output_path = os.path.join(output_base,ann['filename'].replace('/','_'))
-
-    im = None
-    im = Image.open(input_path)
-    
-    # i_tool = 0; tool_name = short_tool_names[i_tool]
-    for i_tool,tool_name in enumerate(short_tool_names):
-        
-        tool_keypoints = ann['keypoints'][tool_name]
-        
-        # Don't plot tools that don't have a consensus annotation
-        if tool_keypoints['median_x'] is None:
-            continue
-        
-        median_x = tool_keypoints['median_x']
-        median_y = tool_keypoints['median_y']
-        
-        draw_ellipse(im, median_x, median_y, median_radius, median_linewidth, color_index=i_tool, 
-                     use_imagedraw=use_fancy_ellipses)
-    
-        if draw_individual_samples:
-            for i_sample in range(0,len(tool_keypoints['x'])):
-                x = tool_keypoints['x'][i_sample]
-                y = tool_keypoints['y'][i_sample]
-                draw_ellipse(im, x, y, sample_radius, None, color_index=i_tool, 
-                         use_imagedraw=False)
-            
-    # ...for each tool
-        
-    im.save(output_path)
-
-# ...for each annotation

megadetector/data_management/importers/prepare_zsl_imerit.py

@@ -1,131 +0,0 @@
-"""
- 
- prepare_zsl_imerit.py
-
- Prepare ZSL Borneo data for annotation (convert input data to iMerit-friendly format).
-
-"""
-
-#%% Imports and constants
-
-import json
-import os
-
-from tqdm import tqdm
-from operator import itemgetter
-from shutil import copyfile
-
-from megadetector.visualization import visualize_db
-from megadetector.data_management.databases import integrity_check_json_db
-from megadetector.data_management.cct_json_utils import IndexedJsonDb
-
-annotation_list_filename = r'd:\wildlife_data\zsl_borneo\all_img_ids_to_bbox.json'
-image_json = r'd:\wildlife_data\zsl_borneo\201906cameratraps\0.5\zsl_camera_traps_201906.json'
-image_base = r'd:\wildlife_data\zsl_borneo\201906cameratraps\0.5'
-output_base = r'd:\wildlife_data\zsl_borneo'
-
-human_classes = ['human','hunter']
-
-
-#%% Load data
-
-with open(annotation_list_filename,'r') as f:
-    annotation_list = json.load(f)
-    
-# with open(image_json,'r') as f:
-#    data = json.load(f)
-indexedData = IndexedJsonDb(image_json)
-
-print('Done loading data')    
-    
-
-#%% Validate data
-
-options = integrity_check_json_db.IntegrityCheckOptions()
-options.baseDir = image_base
-options.bCheckImageSizes = False
-options.bCheckImageExistence = True
-options.bFindUnusedImages = False
-    
-sortedCategories = integrity_check_json_db.integrity_check_json_db(indexedData.db,options)
-
-
-#%% Label previews
-
-viz_options = visualize_db.DbVizOptions()
-viz_options.num_to_visualize = 500
-viz_options.trim_to_images_with_bboxes = False
-viz_options.add_search_links = True
-viz_options.sort_by_filename = False
-html_output_file,image_db = visualize_db.visualize_db(indexedData.db,
-                                                        os.path.join(output_base,'preview'),
-                                                        image_base,viz_options)
-os.startfile(html_output_file)
-
-
-#%% Collect images to annotate
-
-images_to_annotate = []
-
-annotation_list = set(annotation_list)
-n_humans = 0
-
-for im in tqdm(indexedData.db['images']):
-    
-    class_names = indexedData.get_classes_for_image(im)
-    b_human = False
-    for cn in class_names:
-        if cn.lower() in human_classes:
-            b_human = True
-            n_humans += 1
-            break
-        
-    if b_human or im['id'] in annotation_list:
-        images_to_annotate.append(im)
-    
-
-print('Found {} of {} images ({} humans)'.format(len(images_to_annotate),len(annotation_list),n_humans))
-assert len(images_to_annotate) >= len(annotation_list)
-
-
-
-#%% Sort by sequence and frame
-
-images_to_annotate = sorted(images_to_annotate, key=itemgetter('seq_id', 'frame_num')) 
-
-
-#%% Copy to a folder by GUID
-
-# dataset[dataset_id].seq[sequence_id].frame[frame_number].img[img_id].extension
-
-imerit_output_base = os.path.join(output_base,'imerit_batch_9')
-os.makedirs(imerit_output_base,exist_ok=True)
-
-# im = images_to_annotate[0]
-for im in tqdm(images_to_annotate):
-    
-    relative_path = im['file_name']
-    extension = os.path.splitext(relative_path)[1]
-    frame_num = im['frame_num']
-    seq_id = im['seq_id']
-    id = im['id']
-    assert '.' not in id
-    input_full_path = os.path.join(image_base,relative_path)
-    assert os.path.isfile(input_full_path)
-    output_filename = 'datasetzslborneo.seq' + '{0:0>8d}'.format(seq_id) + '.frame' + \
-        '{0:0>4d}'.format(frame_num) + '.img' + id + extension
-    im['imerit_filename'] = output_filename
-    output_full_path = os.path.join(imerit_output_base,output_filename)
-    assert not os.path.isfile(output_full_path)
-    copyfile(input_full_path,output_full_path)
-
-# ...for each image
-    
-
-#%% Write out the annotation list
-
-imerit_batch9_json_filename = os.path.join(imerit_output_base,'imerit_batch_9.json')
-with open(imerit_batch9_json_filename,'w') as f:
-    json.dump(images_to_annotate, f, indent=2)
-    
-