megadetector 5.0.9__py3-none-any.whl → 5.0.11__py3-none-any.whl

data_management/importers/prepare-noaa-fish-data-for-lila.py

@@ -1,359 +0,0 @@
-"""
-
- Prepare a LILA-ready .json file for the NOAA Puget Sound Nearshore Fish dataset.
-
-"""
-
-#%% Constants and imports
-
-import os
-import json
-import uuid
-import pandas as pd
-
-from md_utils.path_utils import open_file
-
-base_folder = r'G:\temp\noaa'
-output_json_fn = os.path.join(base_folder,'noaa_estuary_fish.json')
-edited_image_folders = ['edited_clip_2017','edited_clip_2018']
-jpeg_image_folder = 'JPEGImages'
-metadata_file = 'MasterDataForMicrosoft.xlsx'
-
-
-#%% Enumerate files
-
-edited_image_files = []
-
-# edited_image_folder = edited_image_folders[0]
-for edited_image_folder in edited_image_folders:
-    folder_path = os.path.join(base_folder,edited_image_folder)
-    image_files = os.listdir(folder_path)
-    assert all([fn.endswith('.jpg') for fn in image_files])
-    edited_image_files.extend([os.path.join(folder_path,fn) for fn in image_files])
-    
-jpeg_image_folder_files = os.listdir(os.path.join(base_folder,jpeg_image_folder))
-assert all([fn.endswith('.jpg') for fn in jpeg_image_folder_files])
-
-relative_edited_image_files_set = set()
-
-# fn = edited_image_files[0]
-for fn in edited_image_files:
-    bn = os.path.basename(fn)
-    assert bn not in relative_edited_image_files_set
-    relative_edited_image_files_set.add(bn)
-    
-jpeg_image_folder_files_set = set(jpeg_image_folder_files)
-
-assert len(jpeg_image_folder_files_set) == len(relative_edited_image_files_set)
-
-assert jpeg_image_folder_files_set == relative_edited_image_files_set
-
-
-#%% Read metadata and capture location information
-
-df = pd.read_excel(os.path.join(base_folder,metadata_file))
-
-print('Read {} rows from metadata file'.format(len(df)))
-
-id_string_to_site = {}
-
-# i_row = 0; row = df.iloc[i_row]
-for i_row,row in df.iterrows():
-    
-    assert row['sd'].lower().startswith('sd')
-    assert isinstance(row['id'],int) and row['id'] > 0 and row['id'] < 10000
-    date_string = row['date']
-    date_tokens = date_string.split('_')
-    
-    # Sometimes '2017' was just '17' in the date column
-    if len(date_tokens[2]) != 4:
-        assert len(date_tokens[2]) == 2
-        date_tokens[2] = '20' + date_tokens[2]
-        date_string = '_'.join(date_tokens)
-    else:
-        assert date_tokens[2].startswith('201')
-        
-    id_string = row['sd'].upper() + '_' + str(row['id']) + '_' + date_string
-    id_string_to_site[id_string] = row['site']
-    
-print('Found {} unique locations'.format(len(pd.unique(df['site']))))
-
-
-#%% Read the .json files and build output dictionaries
-
-json_files = [fn for fn in os.listdir(base_folder) if (fn.endswith('.json') and (fn != os.path.basename(output_json_fn)))]
-json_files = [os.path.join(base_folder,fn) for fn in json_files]
-
-fn_to_image = {}
-annotations = []
-
-CATEGORY_ID_EMPTY = 0
-CATEGORY_ID_FISH = 1
-
-categories = [{'id':CATEGORY_ID_EMPTY,'name':'empty'},{'id':CATEGORY_ID_FISH,'name':'animal'}]
-
-empty_images = set()
-non_empty_images = set()
-
-n_matched_locations = 0
-images_with_unmatched_locations = []
-
-import random
-random.seed(1)
-
-site_to_location_id = {}
-
-# json_fn = json_files[0]
-for json_fn in json_files:
-    
-    # if 'partial' in json_fn:
-    #    continue
-    
-    with open(json_fn,'r') as f:
-        
-        lines = f.readlines()
-        
-        # line = lines[0]
-        for line in lines:
-            
-            d = json.loads(line)
-            image_fn = d['image']
-        
-            # if image_fn == 'SD1_238_6_26_17_16_76.73.jpg':
-            #    asdfad
-                
-            # SD29_079_5_14_2018_17_52.85.jpg
-            
-            tokens = image_fn.split('_')
-            assert len(tokens) == 7
-            assert tokens[0].startswith('SD')
-            
-            # Re-write two-digit years as four-digit years
-            if len(tokens[4]) != 4:
-                assert len(tokens[4]) == 2
-                tokens[4] = '20' + tokens[4]
-            else:
-                assert tokens[4].startswith('201')                
-            
-            # Sometimes the year was written with two digits instead of 4
-            # assert len(tokens[4]) == 4 and tokens[4].startswith('20')
-            
-            while tokens[1].startswith('0'):
-                tokens[1] = tokens[1][1:]
-            assert not tokens[1].startswith('0')
-            assert len(tokens[1]) > 0
-
-            id_string = '_'.join(tokens[0:5])
-            
-            location_id = 'unknown'
-                        
-            if id_string in id_string_to_site:
-                
-                site_id = id_string_to_site[id_string]
-                
-                # Have we seen this location already?
-                if site_id in site_to_location_id:
-                    location_id = site_to_location_id[site_id]
-                else:
-                    location_id = 'loc_' + str(uuid.uuid1())
-                    site_to_location_id[site_id] = location_id
-                    print('Adding new location ID {} for site {}'.format(
-                        location_id,site_id))                    
-                n_matched_locations += 1
-                
-            else:
-                raise ValueError('Could not match location ID')
-                images_with_unmatched_locations.append(image_fn)
-            
-            assert image_fn in jpeg_image_folder_files_set
-            assert d['type'] == 'image/jpg'
-            input_ann = d['annotations']
-            assert len(input_ann) == 1 and len(input_ann.keys()) == 1 and 'object' in input_ann
-            input_ann = input_ann['object']
-            assert input_ann['metainfo']['image']['height'] == 1080
-            assert input_ann['metainfo']['image']['width'] == 1920
-        
-            im = {}
-                        
-            img_h = input_ann['metainfo']['image']['height']
-            img_w = input_ann['metainfo']['image']['width']
-            
-            im['width'] = img_w
-            im['height'] = img_h
-            im['file_name'] = image_fn
-            
-            if image_fn in fn_to_image:
-                assert fn_to_image[image_fn]['file_name'] == image_fn
-                assert fn_to_image[image_fn]['width'] == img_w
-                assert fn_to_image[image_fn]['height'] == img_h                
-                im = fn_to_image[image_fn]
-            else:
-                fn_to_image[image_fn] = im
-                im['location'] = location_id
-                im['id'] = image_fn # str(uuid.uuid1())                
-            
-            # Not a typo, it's actually "formateddata"
-            formatted_data = input_ann['formateddata']
-            if len(formatted_data) == 0:
-                
-                # An image shouldn't be annotated as both empty and non-empty
-                assert image_fn not in non_empty_images
-                empty_images.add(image_fn)                
-                ann = {}
-                ann['id'] = str(uuid.uuid1())
-                ann['image_id'] = im['id']
-                ann['category_id'] = CATEGORY_ID_EMPTY
-                ann['sequence_level_annotation'] = False                
-                annotations.append(ann)
-                
-            else:
-                
-                # An image shouldn't be annotated as both empty and non-empty
-                assert image_fn not in empty_images
-                non_empty_images.add(image_fn)                
-                
-                n_boxes = len(formatted_data)
-                
-                # box = formatteddata[0]
-                for box in formatted_data:
-                                        
-                    attributes = box['attribute']
-                    assert len(attributes) == 2 and 'occluded' in attributes and 'truncated' in attributes                    
-                    coordinates = box['coordinates']
-                    assert box['object_type'] == 'bbox'
-                    assert box['class']['type'] == 'Fish'
-                    assert len(coordinates) == 4
-                    for coord in coordinates:
-                        assert len(coord) == 2 and 'x' in coord and 'y' in coord
-                    assert coordinates[0]['y'] == coordinates[1]['y']
-                    assert coordinates[2]['y'] == coordinates[3]['y']
-                    assert coordinates[0]['x'] == coordinates[3]['x']
-                    assert coordinates[1]['x'] == coordinates[2]['x']
-                          
-                    assert coordinates[0]['x'] < coordinates[1]['x']
-                    assert coordinates[0]['y'] < coordinates[3]['y']
-                    
-                    if False:
-                        x = coordinates[0]['x'] / img_w
-                        y = coordinates[0]['y'] / img_h
-                        box_w = (coordinates[1]['x'] - coordinates[0]['x']) / img_w
-                        box_h = (coordinates[3]['y'] - coordinates[0]['y']) / img_h
-                    else:
-                        x = coordinates[0]['x']
-                        y = coordinates[0]['y']
-                        box_w = (coordinates[1]['x'] - coordinates[0]['x'])
-                        box_h = (coordinates[3]['y'] - coordinates[0]['y'])
-                        
-                    bbox = [x,y,box_w,box_h]
-                    
-                    ann = {}
-                    ann['id'] = str(uuid.uuid1())
-                    ann['image_id'] = im['id']
-                    ann['category_id'] = CATEGORY_ID_FISH
-                    ann['sequence_level_annotation'] = False
-                    ann['bbox'] = bbox
-                    
-                    annotations.append(ann)
-                                        
-                    # open_file(os.path.join(base_folder,jpeg_image_folder,image_fn))
-                    
-                # ...for each box
-                
-            # ...if there are boxes on this image
-        
-        # ...for each line
-        
-    # ...with open()
-    
-# ...for each json file
-        
-print('Found annotations for {} images (of {})'.format(len(fn_to_image),
-                                                       len(jpeg_image_folder_files_set)))
-                
-
-print('Matched locations for {} images (failed to match {})'.format(
-    n_matched_locations,len(images_with_unmatched_locations)))
-
-images = list(fn_to_image.values())
-
-
-#%% Prepare the output .json
-
-info = {}
-info['version'] = '2022.07.31.00'
-info['description'] = 'NOAA Estuary Fish 2022'
-info['year'] = 2022
-info['contributor'] = 'NOAA Fisheries'
-    
-d = {}
-d['info'] = info
-d['annotations'] = annotations
-d['images'] = images
-d['categories'] = categories
-
-with open(output_json_fn,'w') as f:
-    json.dump(d,f,indent=1)
-
-
-#%% Check DB integrity
-
-from data_management.databases import integrity_check_json_db
-
-options = integrity_check_json_db.IntegrityCheckOptions()
-options.baseDir = os.path.join(base_folder,jpeg_image_folder)
-options.bCheckImageSizes = False
-options.bCheckImageExistence = True
-options.bFindUnusedImages = True
-
-_, _, _ = integrity_check_json_db.integrity_check_json_db(output_json_fn, options)
-
-
-#%% Print unique locations
-
-from collections import defaultdict
-location_to_count = defaultdict(int)
-for im in d['images']:
-    location_to_count[im['location']] += 1
-for loc in location_to_count.keys():
-    print(loc + ': ' + str(location_to_count[loc]))
-
-print('{} unique locations'.format(len(location_to_count)))
-assert 'unknown' not in location_to_count.keys()
-
-# SD12_202_6_23_2017_1_31.85.jpg
-
-
-#%% Preview some images
-
-from md_visualization import visualize_db
-
-viz_options = visualize_db.DbVizOptions()
-viz_options.num_to_visualize = 10000
-viz_options.trim_to_images_with_bboxes = False
-viz_options.add_search_links = False
-viz_options.sort_by_filename = False
-viz_options.parallelize_rendering = True
-viz_options.include_filename_links = True
-
-html_output_file, _ = visualize_db.visualize_db(db_path=output_json_fn,
-                                                output_dir=os.path.join(base_folder,'preview'),
-                                                image_base_dir=os.path.join(base_folder,jpeg_image_folder),
-                                                options=viz_options)
-open_file(html_output_file)
-
-
-#%% Statistics
-
-print('Empty: {}'.format(len(empty_images)))
-print('Non-empty: {}'.format(len(non_empty_images)))
-
-images_with_no_boxes = 0
-n_boxes = 0
-for ann in annotations:
-    if 'bbox' not in ann:
-        images_with_no_boxes += 1
-    else:
-        assert len(bbox) == 4
-        n_boxes += 1
-
-print('N boxes: {}'.format(n_boxes))

data_management/importers/prepare_zsl_imerit.py

@@ -1,131 +0,0 @@
-"""
- 
- prepare_zsl_imerit.py
-
- Prepare ZSL Borneo data for annotation (convert input data to iMerit-friendly format).
-
-"""
-
-#%% Imports and constants
-
-import json
-import os
-
-from tqdm import tqdm
-from operator import itemgetter
-from shutil import copyfile
-
-from md_visualization import visualize_db
-from data_management.databases import integrity_check_json_db
-from data_management.cct_json_utils import IndexedJsonDb
-
-annotation_list_filename = r'd:\wildlife_data\zsl_borneo\all_img_ids_to_bbox.json'
-image_json = r'd:\wildlife_data\zsl_borneo\201906cameratraps\0.5\zsl_camera_traps_201906.json'
-image_base = r'd:\wildlife_data\zsl_borneo\201906cameratraps\0.5'
-output_base = r'd:\wildlife_data\zsl_borneo'
-
-human_classes = ['human','hunter']
-
-
-#%% Load data
-
-with open(annotation_list_filename,'r') as f:
-    annotation_list = json.load(f)
-    
-# with open(image_json,'r') as f:
-#    data = json.load(f)
-indexedData = IndexedJsonDb(image_json)
-
-print('Done loading data')    
-    
-
-#%% Validate data
-
-options = integrity_check_json_db.IntegrityCheckOptions()
-options.baseDir = image_base
-options.bCheckImageSizes = False
-options.bCheckImageExistence = True
-options.bFindUnusedImages = False
-    
-sortedCategories = integrity_check_json_db.integrity_check_json_db(indexedData.db,options)
-
-
-#%% Label previews
-
-viz_options = visualize_db.DbVizOptions()
-viz_options.num_to_visualize = 500
-viz_options.trim_to_images_with_bboxes = False
-viz_options.add_search_links = True
-viz_options.sort_by_filename = False
-html_output_file,image_db = visualize_db.visualize_db(indexedData.db,
-                                                        os.path.join(output_base,'preview'),
-                                                        image_base,viz_options)
-os.startfile(html_output_file)
-
-
-#%% Collect images to annotate
-
-images_to_annotate = []
-
-annotation_list = set(annotation_list)
-n_humans = 0
-
-for im in tqdm(indexedData.db['images']):
-    
-    class_names = indexedData.get_classes_for_image(im)
-    b_human = False
-    for cn in class_names:
-        if cn.lower() in human_classes:
-            b_human = True
-            n_humans += 1
-            break
-        
-    if b_human or im['id'] in annotation_list:
-        images_to_annotate.append(im)
-    
-
-print('Found {} of {} images ({} humans)'.format(len(images_to_annotate),len(annotation_list),n_humans))
-assert len(images_to_annotate) >= len(annotation_list)
-
-
-
-#%% Sort by sequence and frame
-
-images_to_annotate = sorted(images_to_annotate, key=itemgetter('seq_id', 'frame_num')) 
-
-
-#%% Copy to a folder by GUID
-
-# dataset[dataset_id].seq[sequence_id].frame[frame_number].img[img_id].extension
-
-imerit_output_base = os.path.join(output_base,'imerit_batch_9')
-os.makedirs(imerit_output_base,exist_ok=True)
-
-# im = images_to_annotate[0]
-for im in tqdm(images_to_annotate):
-    
-    relative_path = im['file_name']
-    extension = os.path.splitext(relative_path)[1]
-    frame_num = im['frame_num']
-    seq_id = im['seq_id']
-    id = im['id']
-    assert '.' not in id
-    input_full_path = os.path.join(image_base,relative_path)
-    assert os.path.isfile(input_full_path)
-    output_filename = 'datasetzslborneo.seq' + '{0:0>8d}'.format(seq_id) + '.frame' + \
-        '{0:0>4d}'.format(frame_num) + '.img' + id + extension
-    im['imerit_filename'] = output_filename
-    output_full_path = os.path.join(imerit_output_base,output_filename)
-    assert not os.path.isfile(output_full_path)
-    copyfile(input_full_path,output_full_path)
-
-# ...for each image
-    
-
-#%% Write out the annotation list
-
-imerit_batch9_json_filename = os.path.join(imerit_output_base,'imerit_batch_9.json')
-with open(imerit_batch9_json_filename,'w') as f:
-    json.dump(images_to_annotate, f, indent=2)
-    
-