megadetector 5.0.5__py3-none-any.whl → 5.0.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of megadetector might be problematic. Click here for more details.

Files changed (132) hide show
  1. api/batch_processing/data_preparation/manage_local_batch.py +302 -263
  2. api/batch_processing/data_preparation/manage_video_batch.py +81 -2
  3. api/batch_processing/postprocessing/add_max_conf.py +1 -0
  4. api/batch_processing/postprocessing/categorize_detections_by_size.py +50 -19
  5. api/batch_processing/postprocessing/compare_batch_results.py +110 -60
  6. api/batch_processing/postprocessing/load_api_results.py +56 -70
  7. api/batch_processing/postprocessing/md_to_coco.py +1 -1
  8. api/batch_processing/postprocessing/md_to_labelme.py +2 -1
  9. api/batch_processing/postprocessing/postprocess_batch_results.py +240 -81
  10. api/batch_processing/postprocessing/render_detection_confusion_matrix.py +625 -0
  11. api/batch_processing/postprocessing/repeat_detection_elimination/find_repeat_detections.py +71 -23
  12. api/batch_processing/postprocessing/repeat_detection_elimination/remove_repeat_detections.py +1 -1
  13. api/batch_processing/postprocessing/repeat_detection_elimination/repeat_detections_core.py +227 -75
  14. api/batch_processing/postprocessing/subset_json_detector_output.py +132 -5
  15. api/batch_processing/postprocessing/top_folders_to_bottom.py +1 -1
  16. api/synchronous/api_core/animal_detection_api/detection/run_detector_batch.py +2 -2
  17. classification/prepare_classification_script.py +191 -191
  18. data_management/coco_to_yolo.py +68 -45
  19. data_management/databases/integrity_check_json_db.py +7 -5
  20. data_management/generate_crops_from_cct.py +3 -3
  21. data_management/get_image_sizes.py +8 -6
  22. data_management/importers/add_timestamps_to_icct.py +79 -0
  23. data_management/importers/animl_results_to_md_results.py +160 -0
  24. data_management/importers/auckland_doc_test_to_json.py +4 -4
  25. data_management/importers/auckland_doc_to_json.py +1 -1
  26. data_management/importers/awc_to_json.py +5 -5
  27. data_management/importers/bellevue_to_json.py +5 -5
  28. data_management/importers/carrizo_shrubfree_2018.py +5 -5
  29. data_management/importers/carrizo_trail_cam_2017.py +5 -5
  30. data_management/importers/cct_field_adjustments.py +2 -3
  31. data_management/importers/channel_islands_to_cct.py +4 -4
  32. data_management/importers/ena24_to_json.py +5 -5
  33. data_management/importers/helena_to_cct.py +10 -10
  34. data_management/importers/idaho-camera-traps.py +12 -12
  35. data_management/importers/idfg_iwildcam_lila_prep.py +8 -8
  36. data_management/importers/jb_csv_to_json.py +4 -4
  37. data_management/importers/missouri_to_json.py +1 -1
  38. data_management/importers/noaa_seals_2019.py +1 -1
  39. data_management/importers/pc_to_json.py +5 -5
  40. data_management/importers/prepare-noaa-fish-data-for-lila.py +4 -4
  41. data_management/importers/prepare_zsl_imerit.py +5 -5
  42. data_management/importers/rspb_to_json.py +4 -4
  43. data_management/importers/save_the_elephants_survey_A.py +5 -5
  44. data_management/importers/save_the_elephants_survey_B.py +6 -6
  45. data_management/importers/snapshot_safari_importer.py +9 -9
  46. data_management/importers/snapshot_serengeti_lila.py +9 -9
  47. data_management/importers/timelapse_csv_set_to_json.py +5 -7
  48. data_management/importers/ubc_to_json.py +4 -4
  49. data_management/importers/umn_to_json.py +4 -4
  50. data_management/importers/wellington_to_json.py +1 -1
  51. data_management/importers/wi_to_json.py +2 -2
  52. data_management/importers/zamba_results_to_md_results.py +181 -0
  53. data_management/labelme_to_coco.py +35 -7
  54. data_management/labelme_to_yolo.py +229 -0
  55. data_management/lila/add_locations_to_island_camera_traps.py +1 -1
  56. data_management/lila/add_locations_to_nacti.py +147 -0
  57. data_management/lila/create_lila_blank_set.py +474 -0
  58. data_management/lila/create_lila_test_set.py +2 -1
  59. data_management/lila/create_links_to_md_results_files.py +106 -0
  60. data_management/lila/download_lila_subset.py +46 -21
  61. data_management/lila/generate_lila_per_image_labels.py +23 -14
  62. data_management/lila/get_lila_annotation_counts.py +17 -11
  63. data_management/lila/lila_common.py +14 -11
  64. data_management/lila/test_lila_metadata_urls.py +116 -0
  65. data_management/ocr_tools.py +829 -0
  66. data_management/resize_coco_dataset.py +13 -11
  67. data_management/yolo_output_to_md_output.py +84 -12
  68. data_management/yolo_to_coco.py +38 -20
  69. detection/process_video.py +36 -14
  70. detection/pytorch_detector.py +23 -8
  71. detection/run_detector.py +76 -19
  72. detection/run_detector_batch.py +178 -63
  73. detection/run_inference_with_yolov5_val.py +326 -57
  74. detection/run_tiled_inference.py +153 -43
  75. detection/video_utils.py +34 -8
  76. md_utils/ct_utils.py +172 -1
  77. md_utils/md_tests.py +372 -51
  78. md_utils/path_utils.py +167 -39
  79. md_utils/process_utils.py +26 -7
  80. md_utils/split_locations_into_train_val.py +215 -0
  81. md_utils/string_utils.py +10 -0
  82. md_utils/url_utils.py +0 -2
  83. md_utils/write_html_image_list.py +9 -26
  84. md_visualization/plot_utils.py +12 -8
  85. md_visualization/visualization_utils.py +106 -7
  86. md_visualization/visualize_db.py +16 -8
  87. md_visualization/visualize_detector_output.py +208 -97
  88. {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/METADATA +3 -6
  89. {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/RECORD +98 -121
  90. {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/WHEEL +1 -1
  91. taxonomy_mapping/map_lila_taxonomy_to_wi_taxonomy.py +1 -1
  92. taxonomy_mapping/map_new_lila_datasets.py +43 -39
  93. taxonomy_mapping/prepare_lila_taxonomy_release.py +5 -2
  94. taxonomy_mapping/preview_lila_taxonomy.py +27 -27
  95. taxonomy_mapping/species_lookup.py +33 -13
  96. taxonomy_mapping/taxonomy_csv_checker.py +7 -5
  97. api/synchronous/api_core/yolov5/detect.py +0 -252
  98. api/synchronous/api_core/yolov5/export.py +0 -607
  99. api/synchronous/api_core/yolov5/hubconf.py +0 -146
  100. api/synchronous/api_core/yolov5/models/__init__.py +0 -0
  101. api/synchronous/api_core/yolov5/models/common.py +0 -738
  102. api/synchronous/api_core/yolov5/models/experimental.py +0 -104
  103. api/synchronous/api_core/yolov5/models/tf.py +0 -574
  104. api/synchronous/api_core/yolov5/models/yolo.py +0 -338
  105. api/synchronous/api_core/yolov5/train.py +0 -670
  106. api/synchronous/api_core/yolov5/utils/__init__.py +0 -36
  107. api/synchronous/api_core/yolov5/utils/activations.py +0 -103
  108. api/synchronous/api_core/yolov5/utils/augmentations.py +0 -284
  109. api/synchronous/api_core/yolov5/utils/autoanchor.py +0 -170
  110. api/synchronous/api_core/yolov5/utils/autobatch.py +0 -66
  111. api/synchronous/api_core/yolov5/utils/aws/__init__.py +0 -0
  112. api/synchronous/api_core/yolov5/utils/aws/resume.py +0 -40
  113. api/synchronous/api_core/yolov5/utils/benchmarks.py +0 -148
  114. api/synchronous/api_core/yolov5/utils/callbacks.py +0 -71
  115. api/synchronous/api_core/yolov5/utils/dataloaders.py +0 -1087
  116. api/synchronous/api_core/yolov5/utils/downloads.py +0 -178
  117. api/synchronous/api_core/yolov5/utils/flask_rest_api/example_request.py +0 -19
  118. api/synchronous/api_core/yolov5/utils/flask_rest_api/restapi.py +0 -46
  119. api/synchronous/api_core/yolov5/utils/general.py +0 -1018
  120. api/synchronous/api_core/yolov5/utils/loggers/__init__.py +0 -187
  121. api/synchronous/api_core/yolov5/utils/loggers/wandb/__init__.py +0 -0
  122. api/synchronous/api_core/yolov5/utils/loggers/wandb/log_dataset.py +0 -27
  123. api/synchronous/api_core/yolov5/utils/loggers/wandb/sweep.py +0 -41
  124. api/synchronous/api_core/yolov5/utils/loggers/wandb/wandb_utils.py +0 -577
  125. api/synchronous/api_core/yolov5/utils/loss.py +0 -234
  126. api/synchronous/api_core/yolov5/utils/metrics.py +0 -355
  127. api/synchronous/api_core/yolov5/utils/plots.py +0 -489
  128. api/synchronous/api_core/yolov5/utils/torch_utils.py +0 -314
  129. api/synchronous/api_core/yolov5/val.py +0 -394
  130. md_utils/matlab_porting_tools.py +0 -97
  131. {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/LICENSE +0 -0
  132. {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/top_level.txt +0 -0
@@ -256,20 +256,20 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
256
256
 
257
257
  #%% Validate output
258
258
 
259
- from data_management.databases import sanity_check_json_db
259
+ from data_management.databases import integrity_check_json_db
260
260
 
261
- options = sanity_check_json_db.SanityCheckOptions()
261
+ options = integrity_check_json_db.IntegrityCheckOptions()
262
262
  options.baseDir = image_directory
263
263
  options.bCheckImageSizes = False
264
264
  options.bCheckImageExistence = False
265
265
  options.bFindUnusedImages = False
266
- data = sanity_check_json_db.sanity_check_json_db(output_json_file,options)
266
+ data = integrity_check_json_db.integrity_check_json_db(output_json_file,options)
267
267
 
268
268
 
269
269
  #%% Preview labels
270
270
 
271
271
  from md_visualization import visualize_db
272
- from data_management.databases import sanity_check_json_db
272
+ from data_management.databases import integrity_check_json_db
273
273
 
274
274
  viz_options = visualize_db.DbVizOptions()
275
275
  viz_options.num_to_visualize = None
@@ -278,7 +278,7 @@ viz_options.add_search_links = False
278
278
  viz_options.sort_by_filename = False
279
279
  viz_options.parallelize_rendering = True
280
280
  viz_options.classes_to_exclude = ['empty']
281
- html_output_file,image_db = visualize_db.process_images(db_path=output_json_file,
281
+ html_output_file,image_db = visualize_db.visualize_db(db_path=output_json_file,
282
282
  output_dir=os.path.join(
283
283
  output_base, 'carrizo trail cam 2017/preview'),
284
284
  image_base_dir=image_directory,
@@ -10,7 +10,7 @@
10
10
 
11
11
  #%% Constants and environment
12
12
 
13
- from data_management.databases import sanity_check_json_db
13
+ from data_management.databases import integrity_check_json_db
14
14
  import json
15
15
  import os
16
16
 
@@ -54,5 +54,4 @@ print('Finished writing output .json to {}'.format(outputJsonFile))
54
54
 
55
55
  #%% Check output data file
56
56
 
57
- sanity_check_json_db.sanity_check_json_db(outputJsonFile)
58
-
57
+ integrity_check_json_db.integrity_check_json_db(outputJsonFile)
@@ -882,16 +882,16 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
882
882
 
883
883
  #%% Validate output
884
884
 
885
- from data_management.databases import sanity_check_json_db
885
+ from data_management.databases import integrity_check_json_db
886
886
 
887
887
  fn = output_file
888
- options = sanity_check_json_db.SanityCheckOptions()
888
+ options = integrity_check_json_db.IntegrityCheckOptions()
889
889
  options.baseDir = output_image_folder
890
890
  options.bCheckImageSizes = False
891
891
  options.bCheckImageExistence = False
892
892
  options.bFindUnusedImages = False
893
893
 
894
- sortedCategories, data, error = sanity_check_json_db.sanity_check_json_db(fn,options)
894
+ sortedCategories, data, error = integrity_check_json_db.integrity_check_json_db(fn,options)
895
895
 
896
896
 
897
897
  #%% Preview labels
@@ -906,7 +906,7 @@ viz_options.trim_to_images_with_bboxes = False
906
906
  viz_options.add_search_links = False
907
907
  viz_options.sort_by_filename = False
908
908
  viz_options.parallelize_rendering = True
909
- html_output_file,image_db = visualize_db.process_images(db_path=output_file,
909
+ html_output_file,image_db = visualize_db.visualize_db(db_path=output_file,
910
910
  output_dir=os.path.join(output_base,'preview'),
911
911
  image_base_dir=output_image_folder,
912
912
  options=viz_options)
@@ -245,22 +245,22 @@ zipdir(non_human_dir,non_human_zipfile)
245
245
 
246
246
  #%% Validate output
247
247
 
248
- from data_management.databases import sanity_check_json_db
248
+ from data_management.databases import integrity_check_json_db
249
249
 
250
250
  fn = output_file
251
- options = sanity_check_json_db.SanityCheckOptions()
251
+ options = integrity_check_json_db.IntegrityCheckOptions()
252
252
  options.baseDir = image_directory
253
253
  options.bCheckImageSizes = False
254
254
  options.bCheckImageExistence = True
255
255
  options.bFindUnusedImages = True
256
256
 
257
- sortedCategories, data = sanity_check_json_db.sanity_check_json_db(fn,options)
257
+ sortedCategories, data = integrity_check_json_db.integrity_check_json_db(fn,options)
258
258
 
259
259
 
260
260
  #%% Preview labels
261
261
 
262
262
  from md_visualization import visualize_db
263
- from data_management.databases import sanity_check_json_db
263
+ from data_management.databases import integrity_check_json_db
264
264
 
265
265
  viz_options = visualize_db.DbVizOptions()
266
266
  viz_options.num_to_visualize = None
@@ -268,7 +268,7 @@ viz_options.trim_to_images_with_bboxes = False
268
268
  viz_options.add_search_links = True
269
269
  viz_options.sort_by_filename = False
270
270
  viz_options.parallelize_rendering = True
271
- html_output_file,image_db = visualize_db.process_images(db_path=output_file,
271
+ html_output_file,image_db = visualize_db.visualize_db(db_path=output_file,
272
272
  output_dir=os.path.join(base_directory,'preview'),
273
273
  image_base_dir=image_directory,
274
274
  options=viz_options)
@@ -251,20 +251,20 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
251
251
 
252
252
  #%% Validate output
253
253
 
254
- from data_management.databases import sanity_check_json_db
254
+ from data_management.databases import integrity_check_json_db
255
255
 
256
- options = sanity_check_json_db.SanityCheckOptions()
256
+ options = integrity_check_json_db.IntegrityCheckOptions()
257
257
  options.baseDir = image_directory
258
258
  options.bCheckImageSizes = False
259
259
  options.bCheckImageExistence = False
260
260
  options.bFindUnusedImages = False
261
- data = sanity_check_json_db.sanity_check_json_db(output_json_file,options)
261
+ data = integrity_check_json_db.integrity_check_json_db(output_json_file,options)
262
262
 
263
263
 
264
264
  #%% Preview labels
265
265
 
266
266
  from md_visualization import visualize_db
267
- from data_management.databases import sanity_check_json_db
267
+ from data_management.databases import integrity_check_json_db
268
268
 
269
269
  viz_options = visualize_db.DbVizOptions()
270
270
  viz_options.num_to_visualize = None
@@ -273,10 +273,10 @@ viz_options.add_search_links = True
273
273
  viz_options.sort_by_filename = False
274
274
  viz_options.parallelize_rendering = True
275
275
  viz_options.classes_to_exclude = ['empty']
276
- html_output_file,image_db = visualize_db.process_images(db_path=output_json_file,
277
- output_dir=os.path.join(
278
- output_directory, 'RSPB/preview'),
279
- image_base_dir=image_directory,
280
- options=viz_options)
281
- os.startfile(html_output_file))
276
+ html_output_file,image_db = visualize_db.visualize_db(db_path=output_json_file,
277
+ output_dir=os.path.join(
278
+ output_directory, 'RSPB/preview'),
279
+ image_base_dir=image_directory,
280
+ options=viz_options)
281
+ os.startfile(html_output_file)
282
282
 
@@ -837,15 +837,15 @@ if __name__ == "__main__":
837
837
 
838
838
  #%% Validate .json file
839
839
 
840
- from data_management.databases import sanity_check_json_db
840
+ from data_management.databases import integrity_check_json_db
841
841
 
842
- options = sanity_check_json_db.SanityCheckOptions()
842
+ options = integrity_check_json_db.IntegrityCheckOptions()
843
843
  options.baseDir = input_base
844
844
  options.bCheckImageSizes = False
845
845
  options.bCheckImageExistence = False
846
846
  options.bFindUnusedImages = False
847
847
 
848
- _, _, _ = sanity_check_json_db.sanity_check_json_db(output_json_original_strings, options)
848
+ _, _, _ = integrity_check_json_db.integrity_check_json_db(output_json_original_strings, options)
849
849
 
850
850
 
851
851
  #%% Preview labels
@@ -861,7 +861,7 @@ if __name__ == "__main__":
861
861
  viz_options.include_filename_links = True
862
862
 
863
863
  viz_options.classes_to_exclude = ['empty','deer','elk']
864
- html_output_file, _ = visualize_db.process_images(db_path=output_json_original_strings,
864
+ html_output_file, _ = visualize_db.visualize_db(db_path=output_json_original_strings,
865
865
  output_dir=os.path.join(
866
866
  output_base,'preview'),
867
867
  image_base_dir=input_base,
@@ -1119,15 +1119,15 @@ if __name__ == "__main__":
1119
1119
 
1120
1120
  #%% Validate .json file (modified strings, original files)
1121
1121
 
1122
- from data_management.databases import sanity_check_json_db
1122
+ from data_management.databases import integrity_check_json_db
1123
1123
 
1124
- options = sanity_check_json_db.SanityCheckOptions()
1124
+ options = integrity_check_json_db.IntegrityCheckOptions()
1125
1125
  options.baseDir = input_base
1126
1126
  options.bCheckImageSizes = False
1127
1127
  options.bCheckImageExistence = False
1128
1128
  options.bFindUnusedImages = False
1129
1129
 
1130
- _, _, _ = sanity_check_json_db.sanity_check_json_db(output_json_remapped_ids, options)
1130
+ _, _, _ = integrity_check_json_db.integrity_check_json_db(output_json_remapped_ids, options)
1131
1131
 
1132
1132
 
1133
1133
  #%% Preview labels (original files)
@@ -1146,7 +1146,7 @@ if __name__ == "__main__":
1146
1146
  # viz_options.classes_to_include = ['bobcat']
1147
1147
  viz_options.classes_to_include = [viz_options.multiple_categories_tag]
1148
1148
 
1149
- html_output_file, _ = visualize_db.process_images(db_path=output_json_remapped_ids,
1149
+ html_output_file, _ = visualize_db.visualize_db(db_path=output_json_remapped_ids,
1150
1150
  output_dir=os.path.join(
1151
1151
  output_base,'preview'),
1152
1152
  image_base_dir=input_base,
@@ -1252,15 +1252,15 @@ if __name__ == "__main__":
1252
1252
 
1253
1253
  #%% Validate .json file (final filenames)
1254
1254
 
1255
- from data_management.databases import sanity_check_json_db
1255
+ from data_management.databases import integrity_check_json_db
1256
1256
 
1257
- options = sanity_check_json_db.SanityCheckOptions()
1257
+ options = integrity_check_json_db.IntegrityCheckOptions()
1258
1258
  options.baseDir = input_base
1259
1259
  options.bCheckImageSizes = False
1260
1260
  options.bCheckImageExistence = False
1261
1261
  options.bFindUnusedImages = False
1262
1262
 
1263
- _, _, _ = sanity_check_json_db.sanity_check_json_db(output_json, options)
1263
+ _, _, _ = integrity_check_json_db.integrity_check_json_db(output_json, options)
1264
1264
 
1265
1265
 
1266
1266
  #%% Preview labels (final filenames)
@@ -1281,7 +1281,7 @@ if __name__ == "__main__":
1281
1281
  # viz_options.classes_to_include = [viz_options.multiple_categories_tag]
1282
1282
  # viz_options.classes_to_include = ['human','vehicle','domestic dog']
1283
1283
 
1284
- html_output_file, _ = visualize_db.process_images(db_path=output_json,
1284
+ html_output_file, _ = visualize_db.visualize_db(db_path=output_json,
1285
1285
  output_dir=os.path.join(
1286
1286
  output_base,'final-preview-01'),
1287
1287
  image_base_dir=output_image_base_public,
@@ -146,15 +146,15 @@ with open(output_json,'w') as f:
146
146
 
147
147
  #%% Validate .json file
148
148
 
149
- from data_management.databases import sanity_check_json_db
149
+ from data_management.databases import integrity_check_json_db
150
150
 
151
- options = sanity_check_json_db.SanityCheckOptions()
151
+ options = integrity_check_json_db.IntegrityCheckOptions()
152
152
  options.baseDir = os.path.join(base_folder,'images'); assert os.path.isdir(options.baseDir)
153
153
  options.bCheckImageSizes = False
154
154
  options.bCheckImageExistence = False
155
155
  options.bFindUnusedImages = False
156
156
 
157
- _, _, _ = sanity_check_json_db.sanity_check_json_db(output_json, options)
157
+ _, _, _ = integrity_check_json_db.integrity_check_json_db(output_json, options)
158
158
 
159
159
 
160
160
  #%% Preview labels
@@ -170,7 +170,7 @@ viz_options.parallelize_rendering = True
170
170
  viz_options.include_filename_links = True
171
171
 
172
172
  # viz_options.classes_to_exclude = ['test']
173
- html_output_file, _ = visualize_db.process_images(db_path=output_json,
173
+ html_output_file, _ = visualize_db.visualize_db(db_path=output_json,
174
174
  output_dir=os.path.join(
175
175
  base_folder,'preview'),
176
176
  image_base_dir=os.path.join(base_folder,'images'),
@@ -262,15 +262,15 @@ with open(output_json,'w') as f:
262
262
 
263
263
  #%% Validate .json file
264
264
 
265
- from data_management.databases import sanity_check_json_db
265
+ from data_management.databases import integrity_check_json_db
266
266
 
267
- options = sanity_check_json_db.SanityCheckOptions()
267
+ options = integrity_check_json_db.IntegrityCheckOptions()
268
268
  options.baseDir = remote_image_base_dir
269
269
  options.bCheckImageSizes = False
270
270
  options.bCheckImageExistence = False
271
271
  options.bFindUnusedImages = False
272
272
 
273
- _, _, _ = sanity_check_json_db.sanity_check_json_db(output_json, options)
273
+ _, _, _ = integrity_check_json_db.integrity_check_json_db(output_json, options)
274
274
 
275
275
 
276
276
  #%% Preview labels
@@ -286,7 +286,7 @@ viz_options.parallelize_rendering = True
286
286
  viz_options.include_filename_links = True
287
287
 
288
288
  # viz_options.classes_to_exclude = ['test']
289
- html_output_file, _ = visualize_db.process_images(db_path=output_json,
289
+ html_output_file, _ = visualize_db.visualize_db(db_path=output_json,
290
290
  output_dir=os.path.join(
291
291
  base_folder,'preview'),
292
292
  image_base_dir=remote_image_base_dir,
@@ -140,11 +140,11 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
140
140
  len(images),len(annotations),len(categories)))
141
141
 
142
142
 
143
- #%% Sanity-check
143
+ #%% Validate
144
144
 
145
- from data_management.databases import sanity_check_json_db
145
+ from data_management.databases import integrity_check_json_db
146
146
 
147
- options = sanity_check_json_db.SanityCheckOptions()
148
- sortedCategories,data = sanity_check_json_db.sanity_check_json_db(output_file, options)
147
+ options = integrity_check_json_db.IntegrityCheckOptions()
148
+ sortedCategories,data = integrity_check_json_db.integrity_check_json_db(output_file, options)
149
149
 
150
150
 
@@ -483,7 +483,7 @@ options.classes_to_exclude = None
483
483
  options.trim_to_images_with_bboxes = False
484
484
  options.parallelize_rendering = True
485
485
 
486
- htmlOutputFile,_ = visualize_db.process_images(outputJsonFilenameSet1,output_dir,imageDir,options)
486
+ htmlOutputFile,_ = visualize_db.visualize_db(outputJsonFilenameSet1,output_dir,imageDir,options)
487
487
 
488
488
  from md_utils.path_utils import open_file
489
489
  open_file(htmlOutputFile)
@@ -2,7 +2,7 @@
2
2
  #
3
3
  # noaa_seals_2019.py
4
4
  #
5
- # Prepare the NOAA Artic Seals 2019 metadata for LILA.
5
+ # Prepare the NOAA Arctic Seals 2019 metadata for LILA.
6
6
  #
7
7
  ########
8
8
 
@@ -20,7 +20,7 @@ import humanfriendly
20
20
  import os
21
21
  import PIL
22
22
 
23
- from data_management.databases import sanity_check_json_db
23
+ from data_management.databases import integrity_check_json_db
24
24
  from data_management.cct_json_utils import IndexedJsonDb
25
25
  from data_management import cct_json_to_filename_json
26
26
  from md_visualization import visualize_db
@@ -337,11 +337,11 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
337
337
  len(images),len(annotations),len(categories)))
338
338
 
339
339
 
340
- #%% Sanity-check the database's integrity
340
+ #%% Validate the database's integrity
341
341
 
342
342
  json_data = json.load(open(output_file))
343
- options = sanity_check_json_db.SanityCheckOptions()
344
- sortedCategories,data = sanity_check_json_db.sanity_check_json_db(json_data, options)
343
+ options = integrity_check_json_db.IntegrityCheckOptions()
344
+ sortedCategories,data = integrity_check_json_db.integrity_check_json_db(json_data, options)
345
345
 
346
346
 
347
347
  #%% Render a bunch of images to make sure the labels got carried along correctly
@@ -354,7 +354,7 @@ options.sort_by_filename = False
354
354
  # options.classes_to_exclude = ['unlabeled']
355
355
  options.classes_to_exclude = None
356
356
 
357
- htmlOutputFile,_ = visualize_db.process_images(json_data,output_dir,input_base,options)
357
+ htmlOutputFile,_ = visualize_db.visualize_db(json_data,output_dir,input_base,options)
358
358
  os.startfile(htmlOutputFile)
359
359
 
360
360
 
@@ -335,10 +335,10 @@ viz_options.sort_by_filename = False
335
335
  viz_options.parallelize_rendering = True
336
336
  viz_options.include_filename_links = True
337
337
 
338
- html_output_file, _ = visualize_db.process_images(db_path=output_json_fn,
339
- output_dir=os.path.join(base_folder,'preview'),
340
- image_base_dir=os.path.join(base_folder,jpeg_image_folder),
341
- options=viz_options)
338
+ html_output_file, _ = visualize_db.visualize_db(db_path=output_json_fn,
339
+ output_dir=os.path.join(base_folder,'preview'),
340
+ image_base_dir=os.path.join(base_folder,jpeg_image_folder),
341
+ options=viz_options)
342
342
  open_file(html_output_file)
343
343
 
344
344
 
@@ -16,7 +16,7 @@ from operator import itemgetter
16
16
  from shutil import copyfile
17
17
 
18
18
  from md_visualization import visualize_db
19
- from data_management.databases import sanity_check_json_db
19
+ from data_management.databases import integrity_check_json_db
20
20
  from data_management.cct_json_utils import IndexedJsonDb
21
21
 
22
22
  annotation_list_filename = r'd:\wildlife_data\zsl_borneo\all_img_ids_to_bbox.json'
@@ -39,15 +39,15 @@ indexedData = IndexedJsonDb(image_json)
39
39
  print('Done loading data')
40
40
 
41
41
 
42
- #%% Sanity-check data
42
+ #%% Validate data
43
43
 
44
- options = sanity_check_json_db.SanityCheckOptions()
44
+ options = integrity_check_json_db.IntegrityCheckOptions()
45
45
  options.baseDir = image_base
46
46
  options.bCheckImageSizes = False
47
47
  options.bCheckImageExistence = True
48
48
  options.bFindUnusedImages = False
49
49
 
50
- sortedCategories = sanity_check_json_db.sanity_check_json_db(indexedData.db,options)
50
+ sortedCategories = integrity_check_json_db.integrity_check_json_db(indexedData.db,options)
51
51
 
52
52
 
53
53
  #%% Label previews
@@ -57,7 +57,7 @@ viz_options.num_to_visualize = 500
57
57
  viz_options.trim_to_images_with_bboxes = False
58
58
  viz_options.add_search_links = True
59
59
  viz_options.sort_by_filename = False
60
- html_output_file,image_db = visualize_db.process_images(indexedData.db,
60
+ html_output_file,image_db = visualize_db.visualize_db(indexedData.db,
61
61
  os.path.join(output_base,'preview'),
62
62
  image_base,viz_options)
63
63
  os.startfile(html_output_file)
@@ -21,7 +21,7 @@ import ntpath
21
21
  import humanfriendly
22
22
  import PIL
23
23
 
24
- from data_management.databases import sanity_check_json_db
24
+ from data_management.databases import integrity_check_json_db
25
25
  from md_visualization import visualize_db
26
26
 
27
27
  # [location] is an obfuscation
@@ -320,11 +320,11 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
320
320
 
321
321
  #%% Check database integrity
322
322
 
323
- options = sanity_check_json_db.SanityCheckOptions()
323
+ options = integrity_check_json_db.IntegrityCheckOptions()
324
324
  options.baseDir = imageBaseDir
325
325
  options.bCheckImageSizes = False
326
326
  options.bFindUnusedImages = False
327
- sanity_check_json_db.sanity_check_json_db(outputFile, options)
327
+ integrity_check_json_db.integrity_check_json_db(outputFile, options)
328
328
 
329
329
 
330
330
  #%% Preview a few images to make sure labels were passed along sensibly
@@ -334,7 +334,7 @@ output_dir = os.path.join(baseDir,'label_preview')
334
334
  image_base_dir = imageBaseDir
335
335
  options = visualize_db.DbVizOptions()
336
336
  options.num_to_visualize = 100
337
- htmlOutputFile = visualize_db.process_images(db_path,output_dir,image_base_dir,options)
337
+ htmlOutputFile = visualize_db.visualize_db(db_path,output_dir,image_base_dir,options)
338
338
 
339
339
 
340
340
  #%% One-time processing step: copy images to a flat directory for annotation
@@ -277,21 +277,21 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
277
277
 
278
278
  #%% Validate output
279
279
 
280
- from data_management.databases import sanity_check_json_db
280
+ from data_management.databases import integrity_check_json_db
281
281
 
282
- options = sanity_check_json_db.SanityCheckOptions()
282
+ options = integrity_check_json_db.IntegrityCheckOptions()
283
283
  options.baseDir = image_directory
284
284
  options.bCheckImageSizes = False
285
285
  options.bCheckImageExistence = False
286
286
  options.bFindUnusedImages = False
287
287
 
288
- sortedCategories, data = sanity_check_json_db.sanity_check_json_db(output_json_file,options)
288
+ sortedCategories, data = integrity_check_json_db.integrity_check_json_db(output_json_file,options)
289
289
 
290
290
 
291
291
  #%% Preview labels
292
292
 
293
293
  from md_visualization import visualize_db
294
- from data_management.databases import sanity_check_json_db
294
+ from data_management.databases import integrity_check_json_db
295
295
 
296
296
  viz_options = visualize_db.DbVizOptions()
297
297
  viz_options.num_to_visualize = 1000
@@ -299,7 +299,7 @@ viz_options.trim_to_images_with_bboxes = False
299
299
  viz_options.add_search_links = True
300
300
  viz_options.sort_by_filename = False
301
301
  viz_options.parallelize_rendering = True
302
- html_output_file,image_db = visualize_db.process_images(db_path=output_json_file,
302
+ html_output_file,image_db = visualize_db.visualize_db(db_path=output_json_file,
303
303
  output_dir=os.path.join(output_base,'preview'),
304
304
  image_base_dir=image_directory,
305
305
  options=viz_options)
@@ -10,7 +10,7 @@
10
10
  #%% Constants and environment
11
11
 
12
12
  from md_visualization import visualize_db
13
- from data_management.databases import sanity_check_json_db
13
+ from data_management.databases import integrity_check_json_db
14
14
  import pandas as pd
15
15
  import os
16
16
  import glob
@@ -289,21 +289,21 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
289
289
 
290
290
  #%% Validate output
291
291
 
292
- from data_management.databases import sanity_check_json_db
292
+ from data_management.databases import integrity_check_json_db
293
293
 
294
- options = sanity_check_json_db.SanityCheckOptions()
294
+ options = integrity_check_json_db.IntegrityCheckOptions()
295
295
  options.baseDir = image_directory
296
296
  options.bCheckImageSizes = False
297
297
  options.bCheckImageExistence = False
298
298
  options.bFindUnusedImages = False
299
299
 
300
- sortedCategories, data = sanity_check_json_db.sanity_check_json_db(output_json_file,options)
300
+ sortedCategories, data = integrity_check_json_db.integrity_check_json_db(output_json_file,options)
301
301
 
302
302
 
303
303
  #%% Preview labels
304
304
 
305
305
  from md_visualization import visualize_db
306
- from data_management.databases import sanity_check_json_db
306
+ from data_management.databases import integrity_check_json_db
307
307
 
308
308
  viz_options = visualize_db.DbVizOptions()
309
309
  viz_options.num_to_visualize = 1000
@@ -311,7 +311,7 @@ viz_options.trim_to_images_with_bboxes = False
311
311
  viz_options.add_search_links = True
312
312
  viz_options.sort_by_filename = False
313
313
  viz_options.parallelize_rendering = True
314
- html_output_file,image_db = visualize_db.process_images(db_path=output_json_file,
314
+ html_output_file,image_db = visualize_db.visualize_db(db_path=output_json_file,
315
315
  output_dir=os.path.join(output_base,'preview'),
316
316
  image_base_dir=image_directory,
317
317
  options=viz_options)
@@ -262,7 +262,7 @@ print('Converted {} annotations, {} images, {} categories ({} invalid dates)'.fo
262
262
  len(annotations),len(images),len(categories),n_invalid_dates))
263
263
 
264
264
 
265
- #%% Take a look at categories (just sanity-checking)
265
+ #%% Take a look at categories
266
266
 
267
267
  assert(len(im_id_to_image)==len(images))
268
268
  print('Loaded metadata about {} images and {} sequences'.format(len(images),len(seq_id_to_annotations)))
@@ -389,7 +389,7 @@ print('\nOf {} images: {} missing, {} corrupt, {} no annotation'.format(len(imag
389
389
  n_missing, n_corrupt, n_no_annotation))
390
390
 
391
391
 
392
- #%% Print distribution of sequence lengths (sanity-check)
392
+ #%% Print distribution of sequence lengths
393
393
 
394
394
  seq_id_to_sequence_length = {}
395
395
 
@@ -445,7 +445,7 @@ print('{} files not in the database (of {})'.format(len(files_not_in_db),len(ima
445
445
  del fn
446
446
 
447
447
 
448
- #%% Sanity-check image and annotation uniqueness
448
+ #%% Validate image and annotation uniqueness
449
449
 
450
450
  tmp_img_ids = set()
451
451
  tmp_ann_ids = set()
@@ -458,7 +458,7 @@ for ann in tqdm(annotations):
458
458
  assert ann['id'] not in tmp_ann_ids
459
459
  tmp_ann_ids.add(ann['id'])
460
460
 
461
- print('Finished uniqueness sanity-check')
461
+ print('Finished uniqueness check')
462
462
 
463
463
 
464
464
  #%% Minor updates to fields
@@ -579,15 +579,15 @@ print('\nFinished writing {}, added {} files'.format(zipfilename,n_images_added)
579
579
 
580
580
  #%% Consistency-check-check .json file
581
581
 
582
- from data_management.databases import sanity_check_json_db
582
+ from data_management.databases import integrity_check_json_db
583
583
 
584
- options = sanity_check_json_db.SanityCheckOptions()
584
+ options = integrity_check_json_db.IntegrityCheckOptions()
585
585
  options.baseDir = output_public_folder
586
586
  options.bCheckImageSizes = False
587
587
  options.bCheckImageExistence = True
588
588
  options.bFindUnusedImages = False
589
589
 
590
- sortedCategories, data, errorInfo = sanity_check_json_db.sanity_check_json_db(json_filename,options)
590
+ sortedCategories, data, errorInfo = integrity_check_json_db.integrity_check_json_db(json_filename,options)
591
591
 
592
592
  # This will produce some validation errors, because this zipfile doesn't include humans
593
593
  assert(len(errorInfo['validationErrors']) == len(human_image_ids))
@@ -718,7 +718,7 @@ with open(summary_info_filename,'w') as f:
718
718
  f.write(s)
719
719
 
720
720
 
721
- #%% Generate preview, sanity-check labels
721
+ #%% Generate preview, integrity-check labels
722
722
 
723
723
  viz_options = visualize_db.DbVizOptions()
724
724
  viz_options.num_to_visualize = 5000
@@ -728,7 +728,7 @@ viz_options.sort_by_filename = False
728
728
  viz_options.parallelize_rendering = True
729
729
  viz_options.classes_to_exclude = ['test','empty']
730
730
  # viz_options.classes_to_include = ['jackalblackbacked','bustardkori']
731
- html_output_file, image_db = visualize_db.process_images(db_path=json_filename,
731
+ html_output_file, image_db = visualize_db.visualize_db(db_path=json_filename,
732
732
  output_dir=output_preview_folder,
733
733
  image_base_dir=output_public_folder,
734
734
  options=viz_options)