megadetector 5.0.5__py3-none-any.whl → 5.0.7__py3-none-any.whl

data_management/importers/snapshot_serengeti_lila.py

@@ -228,7 +228,7 @@ if False:
              seq_id_to_annotations,annotations,categories,species_to_category = cct_cache
          
 
-#%% Take a look at categories (just sanity-checking)
+#%% Take a look at categories
 
 if False:
     
@@ -365,7 +365,7 @@ print('\nOf {} images: {} missing, {} corrupt, {} no annotation'.format(len(imag
       n_missing, n_corrupt, n_no_annotation))
     
 
-#%% Print distribution of sequence lengths (sanity-check)
+#%% Print distribution of sequence lengths
 
 seq_id_to_sequence_length = {}
 
@@ -508,7 +508,7 @@ for iSeason,season_mismatches in enumerate(mismatches_by_season):
     print('Size mismatches in season {}: {}'.format(iSeason+1,len(mismatches_by_season[iSeason])))
 
 
-#%% Sanity-check image and annotation uniqueness
+#%% Validate image and annotation uniqueness
              
 tmp_img_ids = set()
 tmp_ann_ids = set()
@@ -852,22 +852,22 @@ for i_season in seasons_to_zip:
 # ...for each season
 
  
-#%% Sanity-check .json files
+#%% Validate .json files
 
 # %logstart -o r'E:\snapshot_temp\python.txt'
     
-from data_management.databases import sanity_check_json_db
+from data_management.databases import integrity_check_json_db
 
 files_to_check = glob.glob(os.path.join(output_base,'*.json'))
 
-options = sanity_check_json_db.SanityCheckOptions()
+options = integrity_check_json_db.IntegrityCheckOptions()
 options.baseDir = image_base
 options.bCheckImageSizes = False
 options.bCheckImageExistence = True
 options.bFindUnusedImages = False
     
 for fn in files_to_check:    
-    sortedCategories, data = sanity_check_json_db.sanity_check_json_db(fn,options)
+    sortedCategories, data = integrity_check_json_db.integrity_check_json_db(fn,options)
 
 
 #%% Zip up .json and .csv files
@@ -894,7 +894,7 @@ for fn in tqdm(files_to_zip):
     zip_single_file(fn)
   
  
-#%% Super-sanity-check that S11 info isn't leaking
+#%% Validate that S11 info isn't leaking
 
 files_to_check = glob.glob(os.path.join(output_base,'*.json'))
 
@@ -992,7 +992,7 @@ viz_options.num_to_visualize = 500
 viz_options.trim_to_images_with_bboxes = True
 viz_options.add_search_links = True
 viz_options.sort_by_filename = False
-html_output_file,bbox_db = visualize_db.process_images(bbox_json_fn,os.path.join(output_base,'preview2'),image_base,viz_options)
+html_output_file,bbox_db = visualize_db.visualize_db(bbox_json_fn,os.path.join(output_base,'preview2'),image_base,viz_options)
 os.startfile(html_output_file)
 
 

data_management/importers/timelapse_csv_set_to_json.py

@@ -24,7 +24,7 @@ import numpy as np
 from tqdm import tqdm
 
 from md_visualization import visualize_db
-from data_management.databases import sanity_check_json_db
+from data_management.databases import integrity_check_json_db
 from md_utils import path_utils
 
 # Text file with relative paths to all files (images and .csv files)
@@ -84,7 +84,6 @@ for fn in image_files:
 print('Found {} image files and {} .csv files ({} non-matching files)'.format(
         len(image_files),len(csv_files),len(non_matching_files)))
 
-
     
 #%% Verify column consistency, create a giant array with all rows from all .csv files
 
@@ -473,10 +472,10 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
         len(images),len(annotations),len(categories)))
 
 
-#%% Sanity-check the database's integrity
+#%% Validate the database's integrity
 
-options = sanity_check_json_db.SanityCheckOptions()
-sortedCategories,data = sanity_check_json_db.sanity_check_json_db(output_file, options)
+options = integrity_check_json_db.IntegrityCheckOptions()
+sortedCategories,data = integrity_check_json_db.integrity_check_json_db(output_file, options)
 
 
 #%% Render a bunch of images to make sure the labels got carried along correctly
@@ -487,6 +486,5 @@ options.parallelize_rendering = True
 options.sort_by_filename = False
 options.classes_to_exclude = ['unlabeled','empty','ambiguous']
 
-html_output_file,data = visualize_db.process_images(output_file,preview_base,file_base,options)
+html_output_file,data = visualize_db.visualize_db(output_file,preview_base,file_base,options)
 os.startfile(html_output_file)
-

data_management/importers/ubc_to_json.py

@@ -25,7 +25,7 @@ from tqdm import tqdm
 from PIL import Image
 
 from md_visualization import visualize_db
-from data_management.databases import sanity_check_json_db
+from data_management.databases import integrity_check_json_db
 from md_utils.path_utils import find_images
 
 input_base = r'e:\ubc'
@@ -373,13 +373,13 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
 
 #%% Validate output
 
-options = sanity_check_json_db.SanityCheckOptions()
+options = integrity_check_json_db.IntegrityCheckOptions()
 options.baseDir = output_base
 options.bCheckImageSizes = False
 options.bCheckImageExistence = False
 options.bFindUnusedImages = True
 
-sortedCategories, data, errors = sanity_check_json_db.sanity_check_json_db(
+sortedCategories, data, errors = integrity_check_json_db.integrity_check_json_db(
     output_json_file, options)
 
 
@@ -391,7 +391,7 @@ viz_options.trim_to_images_with_bboxes = False
 viz_options.add_search_links = True
 viz_options.sort_by_filename = False
 viz_options.parallelize_rendering = True
-html_output_file, image_db = visualize_db.process_images(db_path=output_json_file,
+html_output_file, image_db = visualize_db.visualize_db(db_path=output_json_file,
                                                          output_dir=os.path.join(
                                                              output_base, 'preview'),
                                                          image_base_dir=output_base,

data_management/importers/umn_to_json.py

@@ -292,7 +292,7 @@ import uuid
 import datetime
 from tqdm import tqdm
 
-from data_management.databases import sanity_check_json_db
+from data_management.databases import integrity_check_json_db
 
 output_base = 'f:\orinoquia_camera_traps'
 output_image_base = os.path.join(output_base,'images')
@@ -431,13 +431,13 @@ print('Finished writing json to {}'.format(output_json_filename))
     
 #%% Validate .json file
 
-options = sanity_check_json_db.SanityCheckOptions()
+options = integrity_check_json_db.IntegrityCheckOptions()
 options.baseDir = output_base
 options.bCheckImageSizes = False
 options.bCheckImageExistence = False
 options.bFindUnusedImages = False
 
-_, _, _ = sanity_check_json_db.sanity_check_json_db(output_json_filename, options)
+_, _, _ = integrity_check_json_db.integrity_check_json_db(output_json_filename, options)
 
 
 #%% Map relative paths to annotation categories
@@ -497,7 +497,7 @@ viz_options.parallelize_rendering = True
 viz_options.include_filename_links = True
 
 # viz_options.classes_to_exclude = ['test']
-html_output_file, _ = visualize_db.process_images(db_path=output_json_filename,
+html_output_file, _ = visualize_db.visualize_db(db_path=output_json_filename,
                                                          output_dir=os.path.join(
                                                          output_base,'preview'),
                                                          image_base_dir=os.path.join(output_image_base,'public'),

data_management/importers/wellington_to_json.py

@@ -253,7 +253,7 @@ viz_options.add_search_links = False
 viz_options.sort_by_filename = False
 viz_options.parallelize_rendering = True
 viz_options.classes_to_exclude = ['test']
-html_output_file, image_db = visualize_db.process_images(db_path=output_file,
+html_output_file, image_db = visualize_db.visualize_db(db_path=output_file,
                                                          output_dir=os.path.join(
                                                          preview_dir),
                                                          image_base_dir=image_directory,

data_management/importers/wi_to_json.py

@@ -351,7 +351,7 @@ viz_options.sort_by_filename = False
 viz_options.parallelize_rendering = True
 viz_options.include_filename_links = True
 
-html_output_file, _ = visualize_db.process_images(db_path=output_json_filename,
+html_output_file, _ = visualize_db.visualize_db(db_path=output_json_filename,
                                                          output_dir=preview_base,
                                                          image_base_dir=image_base,
                                                          options=viz_options)
@@ -426,7 +426,7 @@ if create_ordered_dataset:
     viz_options.parallelize_rendering = True
     viz_options.include_filename_links = True
     
-    html_output_file, _ = visualize_db.process_images(db_path=ordered_json_filename,
+    html_output_file, _ = visualize_db.visualize_db(db_path=ordered_json_filename,
                                                              output_dir=ordered_preview_base,
                                                              image_base_dir=ordered_image_base,
                                                              options=viz_options)

data_management/importers/zamba_results_to_md_results.py

@@ -0,0 +1,181 @@
+########
+#
+# zamba_results_to_md_results.py
+#
+# Convert a labels.csv file produced by Zamba Cloud to a MD results file suitable
+# for import into Timelapse.
+#
+# Columns are expected to be:
+#
+# video_uuid (not used)
+# original_filename (assumed to be a relative path name)
+# top_k_label,top_k_probability, for k = 1..N
+# [category name 1],[category name 2],...
+# corrected_label
+#
+# Because the MD results file fundamentally stores detections, what we'll
+# actually do is created bogus detections that fill the entire image.  Detection
+# coordinates are not currently used in Timelapse video video anyway.
+#
+# There is no special handling of empty/blank categories; because these results are
+# based on a classifier, rather than a detector (where "blank" would be the absence of
+# all other categories), "blank" can be queried in Timelapse just like any other class.
+#
+########
+
+#%% Imports and constants
+
+import pandas as pd
+import json
+
+
+#%% Main function
+
+def zamba_results_to_md_results(input_file,output_file=None):
+    """
+    Converts the .csv file [input_file] to the MD-formatted .json file [output_file].
+    
+    If [output_file] is None, '.json' will be appended to the input file.
+    """
+    
+    if output_file is None:
+        output_file = input_file + '.json'
+
+    df = pd.read_csv(input_file)
+    
+    expected_columns = ('video_uuid','corrected_label','original_filename')
+    for s in expected_columns:
+        assert s in df.columns,\
+            'Expected column {} not found, are you sure this is a Zamba results .csv file?'.format(
+                s)
+            
+    # How many results are included per file?
+    assert 'top_1_probability' in df.columns and 'top_1_label' in df.columns
+    top_k = 2
+    while(True):
+        p_string = 'top_' + str(top_k) + '_probability'
+        label_string = 'top_' + str(top_k) + '_label'
+        
+        if p_string in df.columns:
+            assert label_string in df.columns,\
+                'Oops, {} is a column but {} is not'.format(
+                    p_string,label_string)
+            top_k += 1
+            continue
+        else:
+            assert label_string not in df.columns,\
+                'Oops, {} is a column but {} is not'.format(
+                    label_string,p_string)
+            top_k -= 1
+            break
+            
+    print('Found {} probability column pairs'.format(top_k))
+    
+    # Category names start after the fixed columns and the probability columns
+    category_names = []
+    column_names = list(df.columns)
+    first_category_name_index = 0
+    while('top_' in column_names[first_category_name_index] or \
+          column_names[first_category_name_index] in expected_columns):
+        first_category_name_index += 1
+        
+    i_column = first_category_name_index
+    while( (i_column < len(column_names)) and (column_names[i_column] != 'corrected_label') ):
+        category_names.append(column_names[i_column])
+        i_column += 1
+        
+    print('Found {} categories:\n'.format(len(category_names)))
+    
+    for s in category_names:
+        print(s)
+    
+    info = {}
+    info['format_version'] = '1.3'
+    info['detector'] = 'Zamba Cloud'
+    info['classifier'] = 'Zamba Cloud'    
+    
+    detection_category_id_to_name = {}
+    for category_id,category_name in enumerate(category_names):
+        detection_category_id_to_name[str(category_id)] = category_name
+    detection_category_name_to_id = {v: k for k, v in detection_category_id_to_name.items()}
+    
+    images = []
+    
+    # i_row = 0; row = df.iloc[i_row]
+    for i_row,row in df.iterrows():
+        
+        im = {}
+        images.append(im)
+        im['file'] = row['original_filename']
+        
+        detections = []
+        
+        # k = 1
+        for k in range(1,top_k+1):
+            label = row['top_{}_label'.format(k)]
+            confidence = row['top_{}_probability'.format(k)]
+            det = {}
+            det['category'] = detection_category_name_to_id[label]
+            det['conf'] = confidence
+            det['bbox'] = [0,0,1.0,1.0]
+            detections.append(det)
+            
+        im['detections'] = detections
+        
+    # ...for each row
+    
+    results = {}
+    results['info'] = info
+    results['detection_categories'] = detection_category_id_to_name
+    results['images'] = images
+    
+    with open(output_file,'w') as f:
+        json.dump(results,f,indent=1)
+        
+# ...zamba_results_to_md_results(...)
+        
+
+#%% Interactive driver
+
+if False:
+
+    pass
+
+    #%%
+    
+    input_file = r"G:\temp\labels-job-b95a4b76-e332-4e17-ab40-03469392d36a-2023-11-04_16-28-50.060130.csv"
+    output_file = None
+    zamba_results_to_md_results(input_file,output_file)
+    
+
+#%% Command-line driver
+
+import sys,argparse
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        description='Convert a Zamba-formatted .csv results file to a MD-formatted .json results file')
+    
+    parser.add_argument(
+        'input_file',
+        type=str,
+        help='input .csv file')
+    
+    parser.add_argument(
+        '--output_file',
+        type=str,
+        default=None,
+        help='output .json file (defaults to input file appended with ".json")')
+    
+    if len(sys.argv[1:]) == 0:
+        parser.print_help()
+        parser.exit()
+
+    args = parser.parse_args()
+
+    zamba_results_to_md_results(args.input_file,args.output_file)
+    
+if __name__ == '__main__':
+    main()
+    

data_management/labelme_to_coco.py

@@ -20,11 +20,19 @@ from tqdm import tqdm
 
 #%% Functions
 
-def labelme_to_coco(input_folder,output_file=None,category_id_to_category_name=None,
-                    empty_category_name='empty',empty_category_id=None,info_struct=None,
-                    relative_paths_to_include=None,relative_paths_to_exclude=None,
-                    use_folders_as_labels=False,recursive=True,no_json_handling='skip',
-                    validate_image_sizes=True):
+def labelme_to_coco(input_folder,
+                    output_file=None,
+                    category_id_to_category_name=None,
+                    empty_category_name='empty',
+                    empty_category_id=None,
+                    info_struct=None,
+                    relative_paths_to_include=None,
+                    relative_paths_to_exclude=None,
+                    use_folders_as_labels=False,
+                    recursive=True,
+                    no_json_handling='skip',
+                    validate_image_sizes=True,
+                    right_edge_quantization_threshold=None):
     """
     Find all images in [input_folder] that have corresponding .json files, and convert
     to a COCO .json file.
@@ -48,7 +56,13 @@ def labelme_to_coco(input_folder,output_file=None,category_id_to_category_name=N
         
     * 'skip': ignore image files with no corresponding .json files
     * 'empty': treat image files with no corresponding .json files as empty
-    * 'error': throw an error when an image file has no corresponding .json file 
+    * 'error': throw an error when an image file has no corresponding .json file
+    
+    right_edge_quantization_threshold is an off-by-default hack to handle cases where 
+    boxes that really should be running off the right side of the image only extend like 99%
+    of the way there, due to what appears to be a slight bias inherent to MD.  If a box extends
+    within [right_edge_quantization_threshold] (a small number, from 0 to 1, but probably around 
+    0.02) of the right edge of the image, it will be extended to the far right edge.    
     """
     
     if category_id_to_category_name is None:
@@ -86,6 +100,8 @@ def labelme_to_coco(input_folder,output_file=None,category_id_to_category_name=N
     images = []
     annotations = []
     
+    n_edges_quantized = 0
+    
     # image_fn_relative = image_filenames_relative[0]
     for image_fn_relative in tqdm(image_filenames_relative):
         
@@ -182,6 +198,14 @@ def labelme_to_coco(input_folder,output_file=None,category_id_to_category_name=N
                 x1 = max(p0[0],p1[0])
                 y0 = min(p0[1],p1[1])
                 y1 = max(p0[1],p1[1])
+                
+                if right_edge_quantization_threshold is not None:                    
+                    x1_rel = x1 / (im['width'] - 1)
+                    right_edge_distance = 1.0 - x1_rel
+                    if right_edge_distance < right_edge_quantization_threshold:
+                        n_edges_quantized += 1
+                        x1 = im['width'] - 1
+                        
                 bbox = [x0,y0,abs(x1-x0),abs(y1-y0)]
                 ann = {}
                 ann['id'] = str(uuid.uuid1())
@@ -197,6 +221,10 @@ def labelme_to_coco(input_folder,output_file=None,category_id_to_category_name=N
                   
     # ..for each image                
     
+    if n_edges_quantized > 0:
+        print('Quantized the right edge in {} of {} images'.format(
+            n_edges_quantized,len(image_filenames_relative)))
+        
     output_dict = {}
     output_dict['images'] = images
     output_dict['annotations'] = annotations
@@ -324,7 +352,7 @@ if False:
     options.viz_size = (900, -1)
     options.num_to_visualize = 5000
 
-    html_file,_ = visualize_db.process_images(output_file,os.path.expanduser('~/tmp/labelme_to_coco_preview'),
+    html_file,_ = visualize_db.visualize_db(output_file,os.path.expanduser('~/tmp/labelme_to_coco_preview'),
                                 input_folder,options)