megadetector 5.0.5__py3-none-any.whl → 5.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of megadetector might be problematic. Click here for more details.
- api/batch_processing/data_preparation/manage_local_batch.py +302 -263
- api/batch_processing/data_preparation/manage_video_batch.py +81 -2
- api/batch_processing/postprocessing/add_max_conf.py +1 -0
- api/batch_processing/postprocessing/categorize_detections_by_size.py +50 -19
- api/batch_processing/postprocessing/compare_batch_results.py +110 -60
- api/batch_processing/postprocessing/load_api_results.py +56 -70
- api/batch_processing/postprocessing/md_to_coco.py +1 -1
- api/batch_processing/postprocessing/md_to_labelme.py +2 -1
- api/batch_processing/postprocessing/postprocess_batch_results.py +240 -81
- api/batch_processing/postprocessing/render_detection_confusion_matrix.py +625 -0
- api/batch_processing/postprocessing/repeat_detection_elimination/find_repeat_detections.py +71 -23
- api/batch_processing/postprocessing/repeat_detection_elimination/remove_repeat_detections.py +1 -1
- api/batch_processing/postprocessing/repeat_detection_elimination/repeat_detections_core.py +227 -75
- api/batch_processing/postprocessing/subset_json_detector_output.py +132 -5
- api/batch_processing/postprocessing/top_folders_to_bottom.py +1 -1
- api/synchronous/api_core/animal_detection_api/detection/run_detector_batch.py +2 -2
- classification/prepare_classification_script.py +191 -191
- data_management/coco_to_yolo.py +68 -45
- data_management/databases/integrity_check_json_db.py +7 -5
- data_management/generate_crops_from_cct.py +3 -3
- data_management/get_image_sizes.py +8 -6
- data_management/importers/add_timestamps_to_icct.py +79 -0
- data_management/importers/animl_results_to_md_results.py +160 -0
- data_management/importers/auckland_doc_test_to_json.py +4 -4
- data_management/importers/auckland_doc_to_json.py +1 -1
- data_management/importers/awc_to_json.py +5 -5
- data_management/importers/bellevue_to_json.py +5 -5
- data_management/importers/carrizo_shrubfree_2018.py +5 -5
- data_management/importers/carrizo_trail_cam_2017.py +5 -5
- data_management/importers/cct_field_adjustments.py +2 -3
- data_management/importers/channel_islands_to_cct.py +4 -4
- data_management/importers/ena24_to_json.py +5 -5
- data_management/importers/helena_to_cct.py +10 -10
- data_management/importers/idaho-camera-traps.py +12 -12
- data_management/importers/idfg_iwildcam_lila_prep.py +8 -8
- data_management/importers/jb_csv_to_json.py +4 -4
- data_management/importers/missouri_to_json.py +1 -1
- data_management/importers/noaa_seals_2019.py +1 -1
- data_management/importers/pc_to_json.py +5 -5
- data_management/importers/prepare-noaa-fish-data-for-lila.py +4 -4
- data_management/importers/prepare_zsl_imerit.py +5 -5
- data_management/importers/rspb_to_json.py +4 -4
- data_management/importers/save_the_elephants_survey_A.py +5 -5
- data_management/importers/save_the_elephants_survey_B.py +6 -6
- data_management/importers/snapshot_safari_importer.py +9 -9
- data_management/importers/snapshot_serengeti_lila.py +9 -9
- data_management/importers/timelapse_csv_set_to_json.py +5 -7
- data_management/importers/ubc_to_json.py +4 -4
- data_management/importers/umn_to_json.py +4 -4
- data_management/importers/wellington_to_json.py +1 -1
- data_management/importers/wi_to_json.py +2 -2
- data_management/importers/zamba_results_to_md_results.py +181 -0
- data_management/labelme_to_coco.py +35 -7
- data_management/labelme_to_yolo.py +229 -0
- data_management/lila/add_locations_to_island_camera_traps.py +1 -1
- data_management/lila/add_locations_to_nacti.py +147 -0
- data_management/lila/create_lila_blank_set.py +474 -0
- data_management/lila/create_lila_test_set.py +2 -1
- data_management/lila/create_links_to_md_results_files.py +106 -0
- data_management/lila/download_lila_subset.py +46 -21
- data_management/lila/generate_lila_per_image_labels.py +23 -14
- data_management/lila/get_lila_annotation_counts.py +17 -11
- data_management/lila/lila_common.py +14 -11
- data_management/lila/test_lila_metadata_urls.py +116 -0
- data_management/ocr_tools.py +829 -0
- data_management/resize_coco_dataset.py +13 -11
- data_management/yolo_output_to_md_output.py +84 -12
- data_management/yolo_to_coco.py +38 -20
- detection/process_video.py +36 -14
- detection/pytorch_detector.py +23 -8
- detection/run_detector.py +76 -19
- detection/run_detector_batch.py +178 -63
- detection/run_inference_with_yolov5_val.py +326 -57
- detection/run_tiled_inference.py +153 -43
- detection/video_utils.py +34 -8
- md_utils/ct_utils.py +172 -1
- md_utils/md_tests.py +372 -51
- md_utils/path_utils.py +167 -39
- md_utils/process_utils.py +26 -7
- md_utils/split_locations_into_train_val.py +215 -0
- md_utils/string_utils.py +10 -0
- md_utils/url_utils.py +0 -2
- md_utils/write_html_image_list.py +9 -26
- md_visualization/plot_utils.py +12 -8
- md_visualization/visualization_utils.py +106 -7
- md_visualization/visualize_db.py +16 -8
- md_visualization/visualize_detector_output.py +208 -97
- {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/METADATA +3 -6
- {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/RECORD +98 -121
- {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/WHEEL +1 -1
- taxonomy_mapping/map_lila_taxonomy_to_wi_taxonomy.py +1 -1
- taxonomy_mapping/map_new_lila_datasets.py +43 -39
- taxonomy_mapping/prepare_lila_taxonomy_release.py +5 -2
- taxonomy_mapping/preview_lila_taxonomy.py +27 -27
- taxonomy_mapping/species_lookup.py +33 -13
- taxonomy_mapping/taxonomy_csv_checker.py +7 -5
- api/synchronous/api_core/yolov5/detect.py +0 -252
- api/synchronous/api_core/yolov5/export.py +0 -607
- api/synchronous/api_core/yolov5/hubconf.py +0 -146
- api/synchronous/api_core/yolov5/models/__init__.py +0 -0
- api/synchronous/api_core/yolov5/models/common.py +0 -738
- api/synchronous/api_core/yolov5/models/experimental.py +0 -104
- api/synchronous/api_core/yolov5/models/tf.py +0 -574
- api/synchronous/api_core/yolov5/models/yolo.py +0 -338
- api/synchronous/api_core/yolov5/train.py +0 -670
- api/synchronous/api_core/yolov5/utils/__init__.py +0 -36
- api/synchronous/api_core/yolov5/utils/activations.py +0 -103
- api/synchronous/api_core/yolov5/utils/augmentations.py +0 -284
- api/synchronous/api_core/yolov5/utils/autoanchor.py +0 -170
- api/synchronous/api_core/yolov5/utils/autobatch.py +0 -66
- api/synchronous/api_core/yolov5/utils/aws/__init__.py +0 -0
- api/synchronous/api_core/yolov5/utils/aws/resume.py +0 -40
- api/synchronous/api_core/yolov5/utils/benchmarks.py +0 -148
- api/synchronous/api_core/yolov5/utils/callbacks.py +0 -71
- api/synchronous/api_core/yolov5/utils/dataloaders.py +0 -1087
- api/synchronous/api_core/yolov5/utils/downloads.py +0 -178
- api/synchronous/api_core/yolov5/utils/flask_rest_api/example_request.py +0 -19
- api/synchronous/api_core/yolov5/utils/flask_rest_api/restapi.py +0 -46
- api/synchronous/api_core/yolov5/utils/general.py +0 -1018
- api/synchronous/api_core/yolov5/utils/loggers/__init__.py +0 -187
- api/synchronous/api_core/yolov5/utils/loggers/wandb/__init__.py +0 -0
- api/synchronous/api_core/yolov5/utils/loggers/wandb/log_dataset.py +0 -27
- api/synchronous/api_core/yolov5/utils/loggers/wandb/sweep.py +0 -41
- api/synchronous/api_core/yolov5/utils/loggers/wandb/wandb_utils.py +0 -577
- api/synchronous/api_core/yolov5/utils/loss.py +0 -234
- api/synchronous/api_core/yolov5/utils/metrics.py +0 -355
- api/synchronous/api_core/yolov5/utils/plots.py +0 -489
- api/synchronous/api_core/yolov5/utils/torch_utils.py +0 -314
- api/synchronous/api_core/yolov5/val.py +0 -394
- md_utils/matlab_porting_tools.py +0 -97
- {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/LICENSE +0 -0
- {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/top_level.txt +0 -0
|
@@ -256,20 +256,20 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
|
|
|
256
256
|
|
|
257
257
|
#%% Validate output
|
|
258
258
|
|
|
259
|
-
from data_management.databases import
|
|
259
|
+
from data_management.databases import integrity_check_json_db
|
|
260
260
|
|
|
261
|
-
options =
|
|
261
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
262
262
|
options.baseDir = image_directory
|
|
263
263
|
options.bCheckImageSizes = False
|
|
264
264
|
options.bCheckImageExistence = False
|
|
265
265
|
options.bFindUnusedImages = False
|
|
266
|
-
data =
|
|
266
|
+
data = integrity_check_json_db.integrity_check_json_db(output_json_file,options)
|
|
267
267
|
|
|
268
268
|
|
|
269
269
|
#%% Preview labels
|
|
270
270
|
|
|
271
271
|
from md_visualization import visualize_db
|
|
272
|
-
from data_management.databases import
|
|
272
|
+
from data_management.databases import integrity_check_json_db
|
|
273
273
|
|
|
274
274
|
viz_options = visualize_db.DbVizOptions()
|
|
275
275
|
viz_options.num_to_visualize = None
|
|
@@ -278,7 +278,7 @@ viz_options.add_search_links = False
|
|
|
278
278
|
viz_options.sort_by_filename = False
|
|
279
279
|
viz_options.parallelize_rendering = True
|
|
280
280
|
viz_options.classes_to_exclude = ['empty']
|
|
281
|
-
html_output_file,image_db = visualize_db.
|
|
281
|
+
html_output_file,image_db = visualize_db.visualize_db(db_path=output_json_file,
|
|
282
282
|
output_dir=os.path.join(
|
|
283
283
|
output_base, 'carrizo trail cam 2017/preview'),
|
|
284
284
|
image_base_dir=image_directory,
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
|
|
11
11
|
#%% Constants and environment
|
|
12
12
|
|
|
13
|
-
from data_management.databases import
|
|
13
|
+
from data_management.databases import integrity_check_json_db
|
|
14
14
|
import json
|
|
15
15
|
import os
|
|
16
16
|
|
|
@@ -54,5 +54,4 @@ print('Finished writing output .json to {}'.format(outputJsonFile))
|
|
|
54
54
|
|
|
55
55
|
#%% Check output data file
|
|
56
56
|
|
|
57
|
-
|
|
58
|
-
|
|
57
|
+
integrity_check_json_db.integrity_check_json_db(outputJsonFile)
|
|
@@ -882,16 +882,16 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
|
|
|
882
882
|
|
|
883
883
|
#%% Validate output
|
|
884
884
|
|
|
885
|
-
from data_management.databases import
|
|
885
|
+
from data_management.databases import integrity_check_json_db
|
|
886
886
|
|
|
887
887
|
fn = output_file
|
|
888
|
-
options =
|
|
888
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
889
889
|
options.baseDir = output_image_folder
|
|
890
890
|
options.bCheckImageSizes = False
|
|
891
891
|
options.bCheckImageExistence = False
|
|
892
892
|
options.bFindUnusedImages = False
|
|
893
893
|
|
|
894
|
-
sortedCategories, data, error =
|
|
894
|
+
sortedCategories, data, error = integrity_check_json_db.integrity_check_json_db(fn,options)
|
|
895
895
|
|
|
896
896
|
|
|
897
897
|
#%% Preview labels
|
|
@@ -906,7 +906,7 @@ viz_options.trim_to_images_with_bboxes = False
|
|
|
906
906
|
viz_options.add_search_links = False
|
|
907
907
|
viz_options.sort_by_filename = False
|
|
908
908
|
viz_options.parallelize_rendering = True
|
|
909
|
-
html_output_file,image_db = visualize_db.
|
|
909
|
+
html_output_file,image_db = visualize_db.visualize_db(db_path=output_file,
|
|
910
910
|
output_dir=os.path.join(output_base,'preview'),
|
|
911
911
|
image_base_dir=output_image_folder,
|
|
912
912
|
options=viz_options)
|
|
@@ -245,22 +245,22 @@ zipdir(non_human_dir,non_human_zipfile)
|
|
|
245
245
|
|
|
246
246
|
#%% Validate output
|
|
247
247
|
|
|
248
|
-
from data_management.databases import
|
|
248
|
+
from data_management.databases import integrity_check_json_db
|
|
249
249
|
|
|
250
250
|
fn = output_file
|
|
251
|
-
options =
|
|
251
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
252
252
|
options.baseDir = image_directory
|
|
253
253
|
options.bCheckImageSizes = False
|
|
254
254
|
options.bCheckImageExistence = True
|
|
255
255
|
options.bFindUnusedImages = True
|
|
256
256
|
|
|
257
|
-
sortedCategories, data =
|
|
257
|
+
sortedCategories, data = integrity_check_json_db.integrity_check_json_db(fn,options)
|
|
258
258
|
|
|
259
259
|
|
|
260
260
|
#%% Preview labels
|
|
261
261
|
|
|
262
262
|
from md_visualization import visualize_db
|
|
263
|
-
from data_management.databases import
|
|
263
|
+
from data_management.databases import integrity_check_json_db
|
|
264
264
|
|
|
265
265
|
viz_options = visualize_db.DbVizOptions()
|
|
266
266
|
viz_options.num_to_visualize = None
|
|
@@ -268,7 +268,7 @@ viz_options.trim_to_images_with_bboxes = False
|
|
|
268
268
|
viz_options.add_search_links = True
|
|
269
269
|
viz_options.sort_by_filename = False
|
|
270
270
|
viz_options.parallelize_rendering = True
|
|
271
|
-
html_output_file,image_db = visualize_db.
|
|
271
|
+
html_output_file,image_db = visualize_db.visualize_db(db_path=output_file,
|
|
272
272
|
output_dir=os.path.join(base_directory,'preview'),
|
|
273
273
|
image_base_dir=image_directory,
|
|
274
274
|
options=viz_options)
|
|
@@ -251,20 +251,20 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
|
|
|
251
251
|
|
|
252
252
|
#%% Validate output
|
|
253
253
|
|
|
254
|
-
from data_management.databases import
|
|
254
|
+
from data_management.databases import integrity_check_json_db
|
|
255
255
|
|
|
256
|
-
options =
|
|
256
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
257
257
|
options.baseDir = image_directory
|
|
258
258
|
options.bCheckImageSizes = False
|
|
259
259
|
options.bCheckImageExistence = False
|
|
260
260
|
options.bFindUnusedImages = False
|
|
261
|
-
data =
|
|
261
|
+
data = integrity_check_json_db.integrity_check_json_db(output_json_file,options)
|
|
262
262
|
|
|
263
263
|
|
|
264
264
|
#%% Preview labels
|
|
265
265
|
|
|
266
266
|
from md_visualization import visualize_db
|
|
267
|
-
from data_management.databases import
|
|
267
|
+
from data_management.databases import integrity_check_json_db
|
|
268
268
|
|
|
269
269
|
viz_options = visualize_db.DbVizOptions()
|
|
270
270
|
viz_options.num_to_visualize = None
|
|
@@ -273,10 +273,10 @@ viz_options.add_search_links = True
|
|
|
273
273
|
viz_options.sort_by_filename = False
|
|
274
274
|
viz_options.parallelize_rendering = True
|
|
275
275
|
viz_options.classes_to_exclude = ['empty']
|
|
276
|
-
html_output_file,image_db = visualize_db.
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
os.startfile(html_output_file)
|
|
276
|
+
html_output_file,image_db = visualize_db.visualize_db(db_path=output_json_file,
|
|
277
|
+
output_dir=os.path.join(
|
|
278
|
+
output_directory, 'RSPB/preview'),
|
|
279
|
+
image_base_dir=image_directory,
|
|
280
|
+
options=viz_options)
|
|
281
|
+
os.startfile(html_output_file)
|
|
282
282
|
|
|
@@ -837,15 +837,15 @@ if __name__ == "__main__":
|
|
|
837
837
|
|
|
838
838
|
#%% Validate .json file
|
|
839
839
|
|
|
840
|
-
from data_management.databases import
|
|
840
|
+
from data_management.databases import integrity_check_json_db
|
|
841
841
|
|
|
842
|
-
options =
|
|
842
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
843
843
|
options.baseDir = input_base
|
|
844
844
|
options.bCheckImageSizes = False
|
|
845
845
|
options.bCheckImageExistence = False
|
|
846
846
|
options.bFindUnusedImages = False
|
|
847
847
|
|
|
848
|
-
_, _, _ =
|
|
848
|
+
_, _, _ = integrity_check_json_db.integrity_check_json_db(output_json_original_strings, options)
|
|
849
849
|
|
|
850
850
|
|
|
851
851
|
#%% Preview labels
|
|
@@ -861,7 +861,7 @@ if __name__ == "__main__":
|
|
|
861
861
|
viz_options.include_filename_links = True
|
|
862
862
|
|
|
863
863
|
viz_options.classes_to_exclude = ['empty','deer','elk']
|
|
864
|
-
html_output_file, _ = visualize_db.
|
|
864
|
+
html_output_file, _ = visualize_db.visualize_db(db_path=output_json_original_strings,
|
|
865
865
|
output_dir=os.path.join(
|
|
866
866
|
output_base,'preview'),
|
|
867
867
|
image_base_dir=input_base,
|
|
@@ -1119,15 +1119,15 @@ if __name__ == "__main__":
|
|
|
1119
1119
|
|
|
1120
1120
|
#%% Validate .json file (modified strings, original files)
|
|
1121
1121
|
|
|
1122
|
-
from data_management.databases import
|
|
1122
|
+
from data_management.databases import integrity_check_json_db
|
|
1123
1123
|
|
|
1124
|
-
options =
|
|
1124
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
1125
1125
|
options.baseDir = input_base
|
|
1126
1126
|
options.bCheckImageSizes = False
|
|
1127
1127
|
options.bCheckImageExistence = False
|
|
1128
1128
|
options.bFindUnusedImages = False
|
|
1129
1129
|
|
|
1130
|
-
_, _, _ =
|
|
1130
|
+
_, _, _ = integrity_check_json_db.integrity_check_json_db(output_json_remapped_ids, options)
|
|
1131
1131
|
|
|
1132
1132
|
|
|
1133
1133
|
#%% Preview labels (original files)
|
|
@@ -1146,7 +1146,7 @@ if __name__ == "__main__":
|
|
|
1146
1146
|
# viz_options.classes_to_include = ['bobcat']
|
|
1147
1147
|
viz_options.classes_to_include = [viz_options.multiple_categories_tag]
|
|
1148
1148
|
|
|
1149
|
-
html_output_file, _ = visualize_db.
|
|
1149
|
+
html_output_file, _ = visualize_db.visualize_db(db_path=output_json_remapped_ids,
|
|
1150
1150
|
output_dir=os.path.join(
|
|
1151
1151
|
output_base,'preview'),
|
|
1152
1152
|
image_base_dir=input_base,
|
|
@@ -1252,15 +1252,15 @@ if __name__ == "__main__":
|
|
|
1252
1252
|
|
|
1253
1253
|
#%% Validate .json file (final filenames)
|
|
1254
1254
|
|
|
1255
|
-
from data_management.databases import
|
|
1255
|
+
from data_management.databases import integrity_check_json_db
|
|
1256
1256
|
|
|
1257
|
-
options =
|
|
1257
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
1258
1258
|
options.baseDir = input_base
|
|
1259
1259
|
options.bCheckImageSizes = False
|
|
1260
1260
|
options.bCheckImageExistence = False
|
|
1261
1261
|
options.bFindUnusedImages = False
|
|
1262
1262
|
|
|
1263
|
-
_, _, _ =
|
|
1263
|
+
_, _, _ = integrity_check_json_db.integrity_check_json_db(output_json, options)
|
|
1264
1264
|
|
|
1265
1265
|
|
|
1266
1266
|
#%% Preview labels (final filenames)
|
|
@@ -1281,7 +1281,7 @@ if __name__ == "__main__":
|
|
|
1281
1281
|
# viz_options.classes_to_include = [viz_options.multiple_categories_tag]
|
|
1282
1282
|
# viz_options.classes_to_include = ['human','vehicle','domestic dog']
|
|
1283
1283
|
|
|
1284
|
-
html_output_file, _ = visualize_db.
|
|
1284
|
+
html_output_file, _ = visualize_db.visualize_db(db_path=output_json,
|
|
1285
1285
|
output_dir=os.path.join(
|
|
1286
1286
|
output_base,'final-preview-01'),
|
|
1287
1287
|
image_base_dir=output_image_base_public,
|
|
@@ -146,15 +146,15 @@ with open(output_json,'w') as f:
|
|
|
146
146
|
|
|
147
147
|
#%% Validate .json file
|
|
148
148
|
|
|
149
|
-
from data_management.databases import
|
|
149
|
+
from data_management.databases import integrity_check_json_db
|
|
150
150
|
|
|
151
|
-
options =
|
|
151
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
152
152
|
options.baseDir = os.path.join(base_folder,'images'); assert os.path.isdir(options.baseDir)
|
|
153
153
|
options.bCheckImageSizes = False
|
|
154
154
|
options.bCheckImageExistence = False
|
|
155
155
|
options.bFindUnusedImages = False
|
|
156
156
|
|
|
157
|
-
_, _, _ =
|
|
157
|
+
_, _, _ = integrity_check_json_db.integrity_check_json_db(output_json, options)
|
|
158
158
|
|
|
159
159
|
|
|
160
160
|
#%% Preview labels
|
|
@@ -170,7 +170,7 @@ viz_options.parallelize_rendering = True
|
|
|
170
170
|
viz_options.include_filename_links = True
|
|
171
171
|
|
|
172
172
|
# viz_options.classes_to_exclude = ['test']
|
|
173
|
-
html_output_file, _ = visualize_db.
|
|
173
|
+
html_output_file, _ = visualize_db.visualize_db(db_path=output_json,
|
|
174
174
|
output_dir=os.path.join(
|
|
175
175
|
base_folder,'preview'),
|
|
176
176
|
image_base_dir=os.path.join(base_folder,'images'),
|
|
@@ -262,15 +262,15 @@ with open(output_json,'w') as f:
|
|
|
262
262
|
|
|
263
263
|
#%% Validate .json file
|
|
264
264
|
|
|
265
|
-
from data_management.databases import
|
|
265
|
+
from data_management.databases import integrity_check_json_db
|
|
266
266
|
|
|
267
|
-
options =
|
|
267
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
268
268
|
options.baseDir = remote_image_base_dir
|
|
269
269
|
options.bCheckImageSizes = False
|
|
270
270
|
options.bCheckImageExistence = False
|
|
271
271
|
options.bFindUnusedImages = False
|
|
272
272
|
|
|
273
|
-
_, _, _ =
|
|
273
|
+
_, _, _ = integrity_check_json_db.integrity_check_json_db(output_json, options)
|
|
274
274
|
|
|
275
275
|
|
|
276
276
|
#%% Preview labels
|
|
@@ -286,7 +286,7 @@ viz_options.parallelize_rendering = True
|
|
|
286
286
|
viz_options.include_filename_links = True
|
|
287
287
|
|
|
288
288
|
# viz_options.classes_to_exclude = ['test']
|
|
289
|
-
html_output_file, _ = visualize_db.
|
|
289
|
+
html_output_file, _ = visualize_db.visualize_db(db_path=output_json,
|
|
290
290
|
output_dir=os.path.join(
|
|
291
291
|
base_folder,'preview'),
|
|
292
292
|
image_base_dir=remote_image_base_dir,
|
|
@@ -140,11 +140,11 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
|
|
|
140
140
|
len(images),len(annotations),len(categories)))
|
|
141
141
|
|
|
142
142
|
|
|
143
|
-
#%%
|
|
143
|
+
#%% Validate
|
|
144
144
|
|
|
145
|
-
from data_management.databases import
|
|
145
|
+
from data_management.databases import integrity_check_json_db
|
|
146
146
|
|
|
147
|
-
options =
|
|
148
|
-
sortedCategories,data =
|
|
147
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
148
|
+
sortedCategories,data = integrity_check_json_db.integrity_check_json_db(output_file, options)
|
|
149
149
|
|
|
150
150
|
|
|
@@ -483,7 +483,7 @@ options.classes_to_exclude = None
|
|
|
483
483
|
options.trim_to_images_with_bboxes = False
|
|
484
484
|
options.parallelize_rendering = True
|
|
485
485
|
|
|
486
|
-
htmlOutputFile,_ = visualize_db.
|
|
486
|
+
htmlOutputFile,_ = visualize_db.visualize_db(outputJsonFilenameSet1,output_dir,imageDir,options)
|
|
487
487
|
|
|
488
488
|
from md_utils.path_utils import open_file
|
|
489
489
|
open_file(htmlOutputFile)
|
|
@@ -20,7 +20,7 @@ import humanfriendly
|
|
|
20
20
|
import os
|
|
21
21
|
import PIL
|
|
22
22
|
|
|
23
|
-
from data_management.databases import
|
|
23
|
+
from data_management.databases import integrity_check_json_db
|
|
24
24
|
from data_management.cct_json_utils import IndexedJsonDb
|
|
25
25
|
from data_management import cct_json_to_filename_json
|
|
26
26
|
from md_visualization import visualize_db
|
|
@@ -337,11 +337,11 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
|
|
|
337
337
|
len(images),len(annotations),len(categories)))
|
|
338
338
|
|
|
339
339
|
|
|
340
|
-
#%%
|
|
340
|
+
#%% Validate the database's integrity
|
|
341
341
|
|
|
342
342
|
json_data = json.load(open(output_file))
|
|
343
|
-
options =
|
|
344
|
-
sortedCategories,data =
|
|
343
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
344
|
+
sortedCategories,data = integrity_check_json_db.integrity_check_json_db(json_data, options)
|
|
345
345
|
|
|
346
346
|
|
|
347
347
|
#%% Render a bunch of images to make sure the labels got carried along correctly
|
|
@@ -354,7 +354,7 @@ options.sort_by_filename = False
|
|
|
354
354
|
# options.classes_to_exclude = ['unlabeled']
|
|
355
355
|
options.classes_to_exclude = None
|
|
356
356
|
|
|
357
|
-
htmlOutputFile,_ = visualize_db.
|
|
357
|
+
htmlOutputFile,_ = visualize_db.visualize_db(json_data,output_dir,input_base,options)
|
|
358
358
|
os.startfile(htmlOutputFile)
|
|
359
359
|
|
|
360
360
|
|
|
@@ -335,10 +335,10 @@ viz_options.sort_by_filename = False
|
|
|
335
335
|
viz_options.parallelize_rendering = True
|
|
336
336
|
viz_options.include_filename_links = True
|
|
337
337
|
|
|
338
|
-
html_output_file, _ = visualize_db.
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
338
|
+
html_output_file, _ = visualize_db.visualize_db(db_path=output_json_fn,
|
|
339
|
+
output_dir=os.path.join(base_folder,'preview'),
|
|
340
|
+
image_base_dir=os.path.join(base_folder,jpeg_image_folder),
|
|
341
|
+
options=viz_options)
|
|
342
342
|
open_file(html_output_file)
|
|
343
343
|
|
|
344
344
|
|
|
@@ -16,7 +16,7 @@ from operator import itemgetter
|
|
|
16
16
|
from shutil import copyfile
|
|
17
17
|
|
|
18
18
|
from md_visualization import visualize_db
|
|
19
|
-
from data_management.databases import
|
|
19
|
+
from data_management.databases import integrity_check_json_db
|
|
20
20
|
from data_management.cct_json_utils import IndexedJsonDb
|
|
21
21
|
|
|
22
22
|
annotation_list_filename = r'd:\wildlife_data\zsl_borneo\all_img_ids_to_bbox.json'
|
|
@@ -39,15 +39,15 @@ indexedData = IndexedJsonDb(image_json)
|
|
|
39
39
|
print('Done loading data')
|
|
40
40
|
|
|
41
41
|
|
|
42
|
-
#%%
|
|
42
|
+
#%% Validate data
|
|
43
43
|
|
|
44
|
-
options =
|
|
44
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
45
45
|
options.baseDir = image_base
|
|
46
46
|
options.bCheckImageSizes = False
|
|
47
47
|
options.bCheckImageExistence = True
|
|
48
48
|
options.bFindUnusedImages = False
|
|
49
49
|
|
|
50
|
-
sortedCategories =
|
|
50
|
+
sortedCategories = integrity_check_json_db.integrity_check_json_db(indexedData.db,options)
|
|
51
51
|
|
|
52
52
|
|
|
53
53
|
#%% Label previews
|
|
@@ -57,7 +57,7 @@ viz_options.num_to_visualize = 500
|
|
|
57
57
|
viz_options.trim_to_images_with_bboxes = False
|
|
58
58
|
viz_options.add_search_links = True
|
|
59
59
|
viz_options.sort_by_filename = False
|
|
60
|
-
html_output_file,image_db = visualize_db.
|
|
60
|
+
html_output_file,image_db = visualize_db.visualize_db(indexedData.db,
|
|
61
61
|
os.path.join(output_base,'preview'),
|
|
62
62
|
image_base,viz_options)
|
|
63
63
|
os.startfile(html_output_file)
|
|
@@ -21,7 +21,7 @@ import ntpath
|
|
|
21
21
|
import humanfriendly
|
|
22
22
|
import PIL
|
|
23
23
|
|
|
24
|
-
from data_management.databases import
|
|
24
|
+
from data_management.databases import integrity_check_json_db
|
|
25
25
|
from md_visualization import visualize_db
|
|
26
26
|
|
|
27
27
|
# [location] is an obfuscation
|
|
@@ -320,11 +320,11 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
|
|
|
320
320
|
|
|
321
321
|
#%% Check database integrity
|
|
322
322
|
|
|
323
|
-
options =
|
|
323
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
324
324
|
options.baseDir = imageBaseDir
|
|
325
325
|
options.bCheckImageSizes = False
|
|
326
326
|
options.bFindUnusedImages = False
|
|
327
|
-
|
|
327
|
+
integrity_check_json_db.integrity_check_json_db(outputFile, options)
|
|
328
328
|
|
|
329
329
|
|
|
330
330
|
#%% Preview a few images to make sure labels were passed along sensibly
|
|
@@ -334,7 +334,7 @@ output_dir = os.path.join(baseDir,'label_preview')
|
|
|
334
334
|
image_base_dir = imageBaseDir
|
|
335
335
|
options = visualize_db.DbVizOptions()
|
|
336
336
|
options.num_to_visualize = 100
|
|
337
|
-
htmlOutputFile = visualize_db.
|
|
337
|
+
htmlOutputFile = visualize_db.visualize_db(db_path,output_dir,image_base_dir,options)
|
|
338
338
|
|
|
339
339
|
|
|
340
340
|
#%% One-time processing step: copy images to a flat directory for annotation
|
|
@@ -277,21 +277,21 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
|
|
|
277
277
|
|
|
278
278
|
#%% Validate output
|
|
279
279
|
|
|
280
|
-
from data_management.databases import
|
|
280
|
+
from data_management.databases import integrity_check_json_db
|
|
281
281
|
|
|
282
|
-
options =
|
|
282
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
283
283
|
options.baseDir = image_directory
|
|
284
284
|
options.bCheckImageSizes = False
|
|
285
285
|
options.bCheckImageExistence = False
|
|
286
286
|
options.bFindUnusedImages = False
|
|
287
287
|
|
|
288
|
-
sortedCategories, data =
|
|
288
|
+
sortedCategories, data = integrity_check_json_db.integrity_check_json_db(output_json_file,options)
|
|
289
289
|
|
|
290
290
|
|
|
291
291
|
#%% Preview labels
|
|
292
292
|
|
|
293
293
|
from md_visualization import visualize_db
|
|
294
|
-
from data_management.databases import
|
|
294
|
+
from data_management.databases import integrity_check_json_db
|
|
295
295
|
|
|
296
296
|
viz_options = visualize_db.DbVizOptions()
|
|
297
297
|
viz_options.num_to_visualize = 1000
|
|
@@ -299,7 +299,7 @@ viz_options.trim_to_images_with_bboxes = False
|
|
|
299
299
|
viz_options.add_search_links = True
|
|
300
300
|
viz_options.sort_by_filename = False
|
|
301
301
|
viz_options.parallelize_rendering = True
|
|
302
|
-
html_output_file,image_db = visualize_db.
|
|
302
|
+
html_output_file,image_db = visualize_db.visualize_db(db_path=output_json_file,
|
|
303
303
|
output_dir=os.path.join(output_base,'preview'),
|
|
304
304
|
image_base_dir=image_directory,
|
|
305
305
|
options=viz_options)
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
#%% Constants and environment
|
|
11
11
|
|
|
12
12
|
from md_visualization import visualize_db
|
|
13
|
-
from data_management.databases import
|
|
13
|
+
from data_management.databases import integrity_check_json_db
|
|
14
14
|
import pandas as pd
|
|
15
15
|
import os
|
|
16
16
|
import glob
|
|
@@ -289,21 +289,21 @@ print('Finished writing .json file with {} images, {} annotations, and {} catego
|
|
|
289
289
|
|
|
290
290
|
#%% Validate output
|
|
291
291
|
|
|
292
|
-
from data_management.databases import
|
|
292
|
+
from data_management.databases import integrity_check_json_db
|
|
293
293
|
|
|
294
|
-
options =
|
|
294
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
295
295
|
options.baseDir = image_directory
|
|
296
296
|
options.bCheckImageSizes = False
|
|
297
297
|
options.bCheckImageExistence = False
|
|
298
298
|
options.bFindUnusedImages = False
|
|
299
299
|
|
|
300
|
-
sortedCategories, data =
|
|
300
|
+
sortedCategories, data = integrity_check_json_db.integrity_check_json_db(output_json_file,options)
|
|
301
301
|
|
|
302
302
|
|
|
303
303
|
#%% Preview labels
|
|
304
304
|
|
|
305
305
|
from md_visualization import visualize_db
|
|
306
|
-
from data_management.databases import
|
|
306
|
+
from data_management.databases import integrity_check_json_db
|
|
307
307
|
|
|
308
308
|
viz_options = visualize_db.DbVizOptions()
|
|
309
309
|
viz_options.num_to_visualize = 1000
|
|
@@ -311,7 +311,7 @@ viz_options.trim_to_images_with_bboxes = False
|
|
|
311
311
|
viz_options.add_search_links = True
|
|
312
312
|
viz_options.sort_by_filename = False
|
|
313
313
|
viz_options.parallelize_rendering = True
|
|
314
|
-
html_output_file,image_db = visualize_db.
|
|
314
|
+
html_output_file,image_db = visualize_db.visualize_db(db_path=output_json_file,
|
|
315
315
|
output_dir=os.path.join(output_base,'preview'),
|
|
316
316
|
image_base_dir=image_directory,
|
|
317
317
|
options=viz_options)
|
|
@@ -262,7 +262,7 @@ print('Converted {} annotations, {} images, {} categories ({} invalid dates)'.fo
|
|
|
262
262
|
len(annotations),len(images),len(categories),n_invalid_dates))
|
|
263
263
|
|
|
264
264
|
|
|
265
|
-
#%% Take a look at categories
|
|
265
|
+
#%% Take a look at categories
|
|
266
266
|
|
|
267
267
|
assert(len(im_id_to_image)==len(images))
|
|
268
268
|
print('Loaded metadata about {} images and {} sequences'.format(len(images),len(seq_id_to_annotations)))
|
|
@@ -389,7 +389,7 @@ print('\nOf {} images: {} missing, {} corrupt, {} no annotation'.format(len(imag
|
|
|
389
389
|
n_missing, n_corrupt, n_no_annotation))
|
|
390
390
|
|
|
391
391
|
|
|
392
|
-
#%% Print distribution of sequence lengths
|
|
392
|
+
#%% Print distribution of sequence lengths
|
|
393
393
|
|
|
394
394
|
seq_id_to_sequence_length = {}
|
|
395
395
|
|
|
@@ -445,7 +445,7 @@ print('{} files not in the database (of {})'.format(len(files_not_in_db),len(ima
|
|
|
445
445
|
del fn
|
|
446
446
|
|
|
447
447
|
|
|
448
|
-
#%%
|
|
448
|
+
#%% Validate image and annotation uniqueness
|
|
449
449
|
|
|
450
450
|
tmp_img_ids = set()
|
|
451
451
|
tmp_ann_ids = set()
|
|
@@ -458,7 +458,7 @@ for ann in tqdm(annotations):
|
|
|
458
458
|
assert ann['id'] not in tmp_ann_ids
|
|
459
459
|
tmp_ann_ids.add(ann['id'])
|
|
460
460
|
|
|
461
|
-
print('Finished uniqueness
|
|
461
|
+
print('Finished uniqueness check')
|
|
462
462
|
|
|
463
463
|
|
|
464
464
|
#%% Minor updates to fields
|
|
@@ -579,15 +579,15 @@ print('\nFinished writing {}, added {} files'.format(zipfilename,n_images_added)
|
|
|
579
579
|
|
|
580
580
|
#%% Consistency-check-check .json file
|
|
581
581
|
|
|
582
|
-
from data_management.databases import
|
|
582
|
+
from data_management.databases import integrity_check_json_db
|
|
583
583
|
|
|
584
|
-
options =
|
|
584
|
+
options = integrity_check_json_db.IntegrityCheckOptions()
|
|
585
585
|
options.baseDir = output_public_folder
|
|
586
586
|
options.bCheckImageSizes = False
|
|
587
587
|
options.bCheckImageExistence = True
|
|
588
588
|
options.bFindUnusedImages = False
|
|
589
589
|
|
|
590
|
-
sortedCategories, data, errorInfo =
|
|
590
|
+
sortedCategories, data, errorInfo = integrity_check_json_db.integrity_check_json_db(json_filename,options)
|
|
591
591
|
|
|
592
592
|
# This will produce some validation errors, because this zipfile doesn't include humans
|
|
593
593
|
assert(len(errorInfo['validationErrors']) == len(human_image_ids))
|
|
@@ -718,7 +718,7 @@ with open(summary_info_filename,'w') as f:
|
|
|
718
718
|
f.write(s)
|
|
719
719
|
|
|
720
720
|
|
|
721
|
-
#%% Generate preview,
|
|
721
|
+
#%% Generate preview, integrity-check labels
|
|
722
722
|
|
|
723
723
|
viz_options = visualize_db.DbVizOptions()
|
|
724
724
|
viz_options.num_to_visualize = 5000
|
|
@@ -728,7 +728,7 @@ viz_options.sort_by_filename = False
|
|
|
728
728
|
viz_options.parallelize_rendering = True
|
|
729
729
|
viz_options.classes_to_exclude = ['test','empty']
|
|
730
730
|
# viz_options.classes_to_include = ['jackalblackbacked','bustardkori']
|
|
731
|
-
html_output_file, image_db = visualize_db.
|
|
731
|
+
html_output_file, image_db = visualize_db.visualize_db(db_path=json_filename,
|
|
732
732
|
output_dir=output_preview_folder,
|
|
733
733
|
image_base_dir=output_public_folder,
|
|
734
734
|
options=viz_options)
|