megadetector 5.0.28__py3-none-any.whl → 5.0.29__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of megadetector might be problematic. Click here for more details.

Files changed (176) hide show
  1. megadetector/api/batch_processing/api_core/batch_service/score.py +4 -5
  2. megadetector/api/batch_processing/api_core_support/aggregate_results_manually.py +1 -1
  3. megadetector/api/batch_processing/api_support/summarize_daily_activity.py +1 -1
  4. megadetector/api/batch_processing/integration/digiKam/xmp_integration.py +2 -2
  5. megadetector/api/batch_processing/integration/eMammal/test_scripts/push_annotations_to_emammal.py +1 -1
  6. megadetector/api/batch_processing/integration/eMammal/test_scripts/select_images_for_testing.py +1 -1
  7. megadetector/api/synchronous/api_core/tests/load_test.py +2 -3
  8. megadetector/classification/aggregate_classifier_probs.py +3 -3
  9. megadetector/classification/analyze_failed_images.py +5 -5
  10. megadetector/classification/cache_batchapi_outputs.py +5 -5
  11. megadetector/classification/create_classification_dataset.py +11 -12
  12. megadetector/classification/crop_detections.py +10 -10
  13. megadetector/classification/csv_to_json.py +8 -8
  14. megadetector/classification/detect_and_crop.py +13 -15
  15. megadetector/classification/evaluate_model.py +7 -7
  16. megadetector/classification/identify_mislabeled_candidates.py +6 -6
  17. megadetector/classification/json_to_azcopy_list.py +1 -1
  18. megadetector/classification/json_validator.py +29 -32
  19. megadetector/classification/map_classification_categories.py +9 -9
  20. megadetector/classification/merge_classification_detection_output.py +12 -9
  21. megadetector/classification/prepare_classification_script.py +19 -19
  22. megadetector/classification/prepare_classification_script_mc.py +23 -23
  23. megadetector/classification/run_classifier.py +4 -4
  24. megadetector/classification/save_mislabeled.py +6 -6
  25. megadetector/classification/train_classifier.py +1 -1
  26. megadetector/classification/train_classifier_tf.py +9 -9
  27. megadetector/classification/train_utils.py +10 -10
  28. megadetector/data_management/annotations/annotation_constants.py +1 -1
  29. megadetector/data_management/camtrap_dp_to_coco.py +45 -45
  30. megadetector/data_management/cct_json_utils.py +101 -101
  31. megadetector/data_management/cct_to_md.py +49 -49
  32. megadetector/data_management/cct_to_wi.py +33 -33
  33. megadetector/data_management/coco_to_labelme.py +75 -75
  34. megadetector/data_management/coco_to_yolo.py +189 -189
  35. megadetector/data_management/databases/add_width_and_height_to_db.py +3 -2
  36. megadetector/data_management/databases/combine_coco_camera_traps_files.py +38 -38
  37. megadetector/data_management/databases/integrity_check_json_db.py +202 -188
  38. megadetector/data_management/databases/subset_json_db.py +33 -33
  39. megadetector/data_management/generate_crops_from_cct.py +38 -38
  40. megadetector/data_management/get_image_sizes.py +54 -49
  41. megadetector/data_management/labelme_to_coco.py +130 -124
  42. megadetector/data_management/labelme_to_yolo.py +78 -72
  43. megadetector/data_management/lila/create_lila_blank_set.py +81 -83
  44. megadetector/data_management/lila/create_lila_test_set.py +32 -31
  45. megadetector/data_management/lila/create_links_to_md_results_files.py +18 -18
  46. megadetector/data_management/lila/download_lila_subset.py +21 -24
  47. megadetector/data_management/lila/generate_lila_per_image_labels.py +91 -91
  48. megadetector/data_management/lila/get_lila_annotation_counts.py +30 -30
  49. megadetector/data_management/lila/get_lila_image_counts.py +22 -22
  50. megadetector/data_management/lila/lila_common.py +70 -70
  51. megadetector/data_management/lila/test_lila_metadata_urls.py +13 -14
  52. megadetector/data_management/mewc_to_md.py +339 -340
  53. megadetector/data_management/ocr_tools.py +258 -252
  54. megadetector/data_management/read_exif.py +231 -224
  55. megadetector/data_management/remap_coco_categories.py +26 -26
  56. megadetector/data_management/remove_exif.py +31 -20
  57. megadetector/data_management/rename_images.py +187 -187
  58. megadetector/data_management/resize_coco_dataset.py +41 -41
  59. megadetector/data_management/speciesnet_to_md.py +41 -41
  60. megadetector/data_management/wi_download_csv_to_coco.py +55 -55
  61. megadetector/data_management/yolo_output_to_md_output.py +117 -120
  62. megadetector/data_management/yolo_to_coco.py +195 -188
  63. megadetector/detection/change_detection.py +831 -0
  64. megadetector/detection/process_video.py +340 -337
  65. megadetector/detection/pytorch_detector.py +304 -262
  66. megadetector/detection/run_detector.py +177 -164
  67. megadetector/detection/run_detector_batch.py +364 -363
  68. megadetector/detection/run_inference_with_yolov5_val.py +328 -325
  69. megadetector/detection/run_tiled_inference.py +256 -249
  70. megadetector/detection/tf_detector.py +24 -24
  71. megadetector/detection/video_utils.py +290 -282
  72. megadetector/postprocessing/add_max_conf.py +15 -11
  73. megadetector/postprocessing/categorize_detections_by_size.py +44 -44
  74. megadetector/postprocessing/classification_postprocessing.py +415 -415
  75. megadetector/postprocessing/combine_batch_outputs.py +20 -21
  76. megadetector/postprocessing/compare_batch_results.py +528 -517
  77. megadetector/postprocessing/convert_output_format.py +97 -97
  78. megadetector/postprocessing/create_crop_folder.py +219 -146
  79. megadetector/postprocessing/detector_calibration.py +173 -168
  80. megadetector/postprocessing/generate_csv_report.py +508 -499
  81. megadetector/postprocessing/load_api_results.py +23 -20
  82. megadetector/postprocessing/md_to_coco.py +129 -98
  83. megadetector/postprocessing/md_to_labelme.py +89 -83
  84. megadetector/postprocessing/md_to_wi.py +40 -40
  85. megadetector/postprocessing/merge_detections.py +87 -114
  86. megadetector/postprocessing/postprocess_batch_results.py +313 -298
  87. megadetector/postprocessing/remap_detection_categories.py +36 -36
  88. megadetector/postprocessing/render_detection_confusion_matrix.py +205 -199
  89. megadetector/postprocessing/repeat_detection_elimination/find_repeat_detections.py +57 -57
  90. megadetector/postprocessing/repeat_detection_elimination/remove_repeat_detections.py +27 -28
  91. megadetector/postprocessing/repeat_detection_elimination/repeat_detections_core.py +702 -677
  92. megadetector/postprocessing/separate_detections_into_folders.py +226 -211
  93. megadetector/postprocessing/subset_json_detector_output.py +265 -262
  94. megadetector/postprocessing/top_folders_to_bottom.py +45 -45
  95. megadetector/postprocessing/validate_batch_results.py +70 -70
  96. megadetector/taxonomy_mapping/map_lila_taxonomy_to_wi_taxonomy.py +52 -52
  97. megadetector/taxonomy_mapping/map_new_lila_datasets.py +15 -15
  98. megadetector/taxonomy_mapping/prepare_lila_taxonomy_release.py +14 -14
  99. megadetector/taxonomy_mapping/preview_lila_taxonomy.py +66 -66
  100. megadetector/taxonomy_mapping/retrieve_sample_image.py +16 -16
  101. megadetector/taxonomy_mapping/simple_image_download.py +8 -8
  102. megadetector/taxonomy_mapping/species_lookup.py +33 -33
  103. megadetector/taxonomy_mapping/taxonomy_csv_checker.py +14 -14
  104. megadetector/taxonomy_mapping/taxonomy_graph.py +10 -10
  105. megadetector/taxonomy_mapping/validate_lila_category_mappings.py +13 -13
  106. megadetector/utils/azure_utils.py +22 -22
  107. megadetector/utils/ct_utils.py +1018 -200
  108. megadetector/utils/directory_listing.py +21 -77
  109. megadetector/utils/gpu_test.py +22 -22
  110. megadetector/utils/md_tests.py +541 -518
  111. megadetector/utils/path_utils.py +1457 -398
  112. megadetector/utils/process_utils.py +41 -41
  113. megadetector/utils/sas_blob_utils.py +53 -49
  114. megadetector/utils/split_locations_into_train_val.py +61 -61
  115. megadetector/utils/string_utils.py +147 -26
  116. megadetector/utils/url_utils.py +463 -173
  117. megadetector/utils/wi_utils.py +2629 -2526
  118. megadetector/utils/write_html_image_list.py +137 -137
  119. megadetector/visualization/plot_utils.py +21 -21
  120. megadetector/visualization/render_images_with_thumbnails.py +37 -73
  121. megadetector/visualization/visualization_utils.py +401 -397
  122. megadetector/visualization/visualize_db.py +197 -190
  123. megadetector/visualization/visualize_detector_output.py +79 -73
  124. {megadetector-5.0.28.dist-info → megadetector-5.0.29.dist-info}/METADATA +135 -132
  125. megadetector-5.0.29.dist-info/RECORD +163 -0
  126. {megadetector-5.0.28.dist-info → megadetector-5.0.29.dist-info}/WHEEL +1 -1
  127. {megadetector-5.0.28.dist-info → megadetector-5.0.29.dist-info}/licenses/LICENSE +0 -0
  128. {megadetector-5.0.28.dist-info → megadetector-5.0.29.dist-info}/top_level.txt +0 -0
  129. megadetector/data_management/importers/add_nacti_sizes.py +0 -52
  130. megadetector/data_management/importers/add_timestamps_to_icct.py +0 -79
  131. megadetector/data_management/importers/animl_results_to_md_results.py +0 -158
  132. megadetector/data_management/importers/auckland_doc_test_to_json.py +0 -373
  133. megadetector/data_management/importers/auckland_doc_to_json.py +0 -201
  134. megadetector/data_management/importers/awc_to_json.py +0 -191
  135. megadetector/data_management/importers/bellevue_to_json.py +0 -272
  136. megadetector/data_management/importers/cacophony-thermal-importer.py +0 -793
  137. megadetector/data_management/importers/carrizo_shrubfree_2018.py +0 -269
  138. megadetector/data_management/importers/carrizo_trail_cam_2017.py +0 -289
  139. megadetector/data_management/importers/cct_field_adjustments.py +0 -58
  140. megadetector/data_management/importers/channel_islands_to_cct.py +0 -913
  141. megadetector/data_management/importers/eMammal/copy_and_unzip_emammal.py +0 -180
  142. megadetector/data_management/importers/eMammal/eMammal_helpers.py +0 -249
  143. megadetector/data_management/importers/eMammal/make_eMammal_json.py +0 -223
  144. megadetector/data_management/importers/ena24_to_json.py +0 -276
  145. megadetector/data_management/importers/filenames_to_json.py +0 -386
  146. megadetector/data_management/importers/helena_to_cct.py +0 -283
  147. megadetector/data_management/importers/idaho-camera-traps.py +0 -1407
  148. megadetector/data_management/importers/idfg_iwildcam_lila_prep.py +0 -294
  149. megadetector/data_management/importers/import_desert_lion_conservation_camera_traps.py +0 -387
  150. megadetector/data_management/importers/jb_csv_to_json.py +0 -150
  151. megadetector/data_management/importers/mcgill_to_json.py +0 -250
  152. megadetector/data_management/importers/missouri_to_json.py +0 -490
  153. megadetector/data_management/importers/nacti_fieldname_adjustments.py +0 -79
  154. megadetector/data_management/importers/noaa_seals_2019.py +0 -181
  155. megadetector/data_management/importers/osu-small-animals-to-json.py +0 -364
  156. megadetector/data_management/importers/pc_to_json.py +0 -365
  157. megadetector/data_management/importers/plot_wni_giraffes.py +0 -123
  158. megadetector/data_management/importers/prepare_zsl_imerit.py +0 -131
  159. megadetector/data_management/importers/raic_csv_to_md_results.py +0 -416
  160. megadetector/data_management/importers/rspb_to_json.py +0 -356
  161. megadetector/data_management/importers/save_the_elephants_survey_A.py +0 -320
  162. megadetector/data_management/importers/save_the_elephants_survey_B.py +0 -329
  163. megadetector/data_management/importers/snapshot_safari_importer.py +0 -758
  164. megadetector/data_management/importers/snapshot_serengeti_lila.py +0 -1067
  165. megadetector/data_management/importers/snapshotserengeti/make_full_SS_json.py +0 -150
  166. megadetector/data_management/importers/snapshotserengeti/make_per_season_SS_json.py +0 -153
  167. megadetector/data_management/importers/sulross_get_exif.py +0 -65
  168. megadetector/data_management/importers/timelapse_csv_set_to_json.py +0 -490
  169. megadetector/data_management/importers/ubc_to_json.py +0 -399
  170. megadetector/data_management/importers/umn_to_json.py +0 -507
  171. megadetector/data_management/importers/wellington_to_json.py +0 -263
  172. megadetector/data_management/importers/wi_to_json.py +0 -442
  173. megadetector/data_management/importers/zamba_results_to_md_results.py +0 -180
  174. megadetector/data_management/lila/add_locations_to_island_camera_traps.py +0 -101
  175. megadetector/data_management/lila/add_locations_to_nacti.py +0 -151
  176. megadetector-5.0.28.dist-info/RECORD +0 -209
@@ -32,7 +32,7 @@ from megadetector.utils.write_html_image_list import write_html_image_list
32
32
  from megadetector.data_management.cct_json_utils import IndexedJsonDb
33
33
  from megadetector.visualization import visualization_utils as vis_utils
34
34
 
35
- def isnan(x):
35
+ def _isnan(x):
36
36
  return (isinstance(x,float) and np.isnan(x))
37
37
 
38
38
 
@@ -42,104 +42,104 @@ class DbVizOptions:
42
42
  """
43
43
  Parameters controlling the behavior of visualize_db().
44
44
  """
45
-
45
+
46
46
  def __init__(self):
47
-
47
+
48
48
  #: Number of images to sample from the database, or None to visualize all images
49
49
  self.num_to_visualize = None
50
-
50
+
51
51
  #: Target size for rendering; set either dimension to -1 to preserve aspect ratio.
52
52
  #:
53
53
  #: If viz_size is None or (-1,-1), the original image size is used.
54
54
  self.viz_size = (1000, -1)
55
-
55
+
56
56
  #: HTML rendering options; see write_html_image_list for details
57
57
  #:
58
58
  #:The most relevant option one might want to set here is:
59
59
  #:
60
- #: htmlOptions['maxFiguresPerHtmlFile']
60
+ #: html_options['maxFiguresPerHtmlFile']
61
61
  #:
62
62
  #: ...which can be used to paginate previews to a number of images that will load well
63
63
  #: in a browser (5000 is a reasonable limit).
64
- self.htmlOptions = write_html_image_list()
65
-
64
+ self.html_options = write_html_image_list()
65
+
66
66
  #: Whether to sort images by filename (True) or randomly (False)
67
67
  self.sort_by_filename = True
68
-
68
+
69
69
  #: Only show images that contain bounding boxes
70
70
  self.trim_to_images_with_bboxes = False
71
-
71
+
72
72
  #: Random seed to use for sampling images
73
73
  self.random_seed = 0
74
-
75
- #: Should we include Web search links for each category name?
74
+
75
+ #: Should we include Web search links for each category name?
76
76
  self.add_search_links = False
77
-
77
+
78
78
  #: Should each thumbnail image link back to the original image?
79
79
  self.include_image_links = False
80
-
80
+
81
81
  #: Should there be a text link back to each original image?
82
82
  self.include_filename_links = False
83
-
83
+
84
84
  #: Line width in pixels
85
85
  self.box_thickness = 4
86
-
86
+
87
87
  #: Number of pixels to expand each bounding box
88
88
  self.box_expansion = 0
89
-
89
+
90
90
  #: Only include images that contain annotations with these class names (not IDs) (list)
91
91
  #:
92
92
  #: Mutually exclusive with classes_to_exclude
93
93
  self.classes_to_include = None
94
-
94
+
95
95
  #: Exclude images that contain annotations with these class names (not IDs) (list)
96
96
  #:
97
97
  #: Mutually exclusive with classes_to_include
98
98
  self.classes_to_exclude = None
99
-
99
+
100
100
  #: Special tag used to say "show me all images with multiple categories"
101
101
  #:
102
102
  #: :meta private:
103
103
  self.multiple_categories_tag = '*multiple*'
104
-
105
- #: We sometimes flatten image directories by replacing a path separator with
104
+
105
+ #: We sometimes flatten image directories by replacing a path separator with
106
106
  #: another character. Leave blank for the typical case where this isn't necessary.
107
107
  self.pathsep_replacement = '' # '~'
108
-
108
+
109
109
  #: Parallelize rendering across multiple workers
110
110
  self.parallelize_rendering = False
111
-
111
+
112
112
  #: In theory, whether to parallelize with threads (True) or processes (False), but
113
113
  #: process-based parallelization in this function is currently unsupported
114
114
  self.parallelize_rendering_with_threads = True
115
-
115
+
116
116
  #: Number of workers to use for parallelization; ignored if parallelize_rendering
117
117
  #: is False
118
118
  self.parallelize_rendering_n_cores = 25
119
-
119
+
120
120
  #: Should we show absolute (True) or relative (False) paths for each image?
121
121
  self.show_full_paths = False
122
-
122
+
123
123
  #: List of additional fields in the image struct that we should print in image headers
124
124
  self.extra_image_fields_to_print = None
125
-
125
+
126
126
  #: List of additional fields in the annotation struct that we should print in image headers
127
127
  self.extra_annotation_fields_to_print = None
128
-
128
+
129
129
  #: Set to False to skip existing images
130
130
  self.force_rendering = True
131
-
131
+
132
132
  #: Enable additionald debug console output
133
133
  self.verbose = False
134
-
134
+
135
135
  #: COCO files used for evaluation may contain confidence scores, this
136
136
  #: determines the field name used for confidence scores
137
137
  self.confidence_field_name = 'score'
138
-
139
- #: Optionally apply a confidence threshold; this requires that [confidence_field_name]
138
+
139
+ #: Optionally apply a confidence threshold; this requires that [confidence_field_name]
140
140
  #: be present in all detections.
141
141
  self.confidence_threshold = None
142
-
142
+
143
143
 
144
144
  #%% Helper functions
145
145
 
@@ -148,9 +148,9 @@ def _image_filename_to_path(image_file_name, image_base_dir, pathsep_replacement
148
148
  Translates the file name in an image entry in the json database to a path, possibly doing
149
149
  some manipulation of path separators.
150
150
  """
151
-
151
+
152
152
  if len(pathsep_replacement) > 0:
153
- image_file_name = os.path.normpath(image_file_name).replace(os.pathsep,pathsep_replacement)
153
+ image_file_name = os.path.normpath(image_file_name).replace(os.pathsep,pathsep_replacement)
154
154
  return os.path.join(image_base_dir, image_file_name)
155
155
 
156
156
 
@@ -159,40 +159,40 @@ def _image_filename_to_path(image_file_name, image_base_dir, pathsep_replacement
159
159
  def visualize_db(db_path, output_dir, image_base_dir, options=None):
160
160
  """
161
161
  Writes images and html to output_dir to visualize the annotations in a .json file.
162
-
162
+
163
163
  Args:
164
164
  db_path (str or dict): the .json filename to load, or a previously-loaded database
165
165
  image_base_dir (str): the folder where the images live; filenames in [db_path] should
166
166
  be relative to this folder.
167
167
  options (DbVizOptions, optional): See DbVizOptions for details
168
-
168
+
169
169
  Returns:
170
170
  tuple: A length-two tuple containing (the html filename) and (the loaded database).
171
- """
172
-
171
+ """
172
+
173
173
  if options is None:
174
174
  options = DbVizOptions()
175
-
175
+
176
176
  # Consistency checking for fields with specific format requirements
177
-
177
+
178
178
  # This should be a list, but if someone specifies a string, do a reasonable thing
179
179
  if isinstance(options.extra_image_fields_to_print,str):
180
180
  options.extra_image_fields_to_print = [options.extra_image_fields_to_print]
181
-
181
+
182
182
  if not options.parallelize_rendering_with_threads:
183
183
  print('Warning: process-based parallelization is not yet supported by visualize_db')
184
184
  options.parallelize_rendering_with_threads = True
185
-
185
+
186
186
  if image_base_dir.startswith('http'):
187
187
  if not image_base_dir.endswith('/'):
188
188
  image_base_dir += '/'
189
189
  else:
190
190
  assert(os.path.isdir(image_base_dir))
191
-
191
+
192
192
  os.makedirs(os.path.join(output_dir, 'rendered_images'), exist_ok=True)
193
-
193
+
194
194
  if isinstance(db_path,str):
195
- assert(os.path.isfile(db_path))
195
+ assert(os.path.isfile(db_path))
196
196
  print('Loading database from {}...'.format(db_path))
197
197
  image_db = json.load(open(db_path))
198
198
  print('...done')
@@ -200,52 +200,52 @@ def visualize_db(db_path, output_dir, image_base_dir, options=None):
200
200
  print('Using previously-loaded DB')
201
201
  image_db = db_path
202
202
  else:
203
- raise ValueError('Illegal dictionary or filename')
204
-
203
+ raise ValueError('Illegal dictionary or filename')
204
+
205
205
  annotations = image_db['annotations']
206
206
  images = image_db['images']
207
207
  categories = image_db['categories']
208
-
208
+
209
209
  # Optionally remove all images without bounding boxes, *before* sampling
210
210
  if options.trim_to_images_with_bboxes:
211
-
211
+
212
212
  b_has_bbox = [False] * len(annotations)
213
213
  for i_ann,ann in enumerate(annotations):
214
214
  if 'bbox' in ann:
215
215
  assert isinstance(ann['bbox'],list)
216
216
  b_has_bbox[i_ann] = True
217
217
  annotations_with_boxes = list(compress(annotations, b_has_bbox))
218
-
218
+
219
219
  image_ids_with_boxes = [x['image_id'] for x in annotations_with_boxes]
220
220
  image_ids_with_boxes = set(image_ids_with_boxes)
221
-
221
+
222
222
  image_has_box = [False] * len(images)
223
223
  for i_image,image in enumerate(images):
224
- imageID = image['id']
225
- if imageID in image_ids_with_boxes:
224
+ image_id = image['id']
225
+ if image_id in image_ids_with_boxes:
226
226
  image_has_box[i_image] = True
227
227
  images_with_bboxes = list(compress(images, image_has_box))
228
228
  images = images_with_bboxes
229
-
229
+
230
230
  # Optionally include/remove images with specific labels, *before* sampling
231
-
231
+
232
232
  assert (not ((options.classes_to_exclude is not None) and \
233
233
  (options.classes_to_include is not None))), \
234
234
  'Cannot specify an inclusion and exclusion list'
235
-
235
+
236
236
  if options.classes_to_exclude is not None:
237
237
  assert isinstance(options.classes_to_exclude,list), \
238
238
  'If supplied, classes_to_exclude should be a list'
239
-
239
+
240
240
  if options.classes_to_include is not None:
241
241
  assert isinstance(options.classes_to_include,list), \
242
242
  'If supplied, classes_to_include should be a list'
243
-
243
+
244
244
  if (options.classes_to_exclude is not None) or (options.classes_to_include is not None):
245
-
245
+
246
246
  print('Indexing database')
247
247
  indexed_db = IndexedJsonDb(image_db)
248
- b_valid_class = [True] * len(images)
248
+ b_valid_class = [True] * len(images)
249
249
  for i_image,image in enumerate(images):
250
250
  classes = indexed_db.get_classes_for_image(image)
251
251
  if options.classes_to_exclude is not None:
@@ -257,85 +257,85 @@ def visualize_db(db_path, output_dir, image_base_dir, options=None):
257
257
  b_valid_class[i_image] = False
258
258
  if options.multiple_categories_tag in options.classes_to_include:
259
259
  if len(classes) > 1:
260
- b_valid_class[i_image] = True
260
+ b_valid_class[i_image] = True
261
261
  if not b_valid_class[i_image]:
262
262
  for c in classes:
263
263
  if c in options.classes_to_include:
264
264
  b_valid_class[i_image] = True
265
- break
265
+ break
266
266
  else:
267
267
  raise ValueError('Illegal include/exclude combination')
268
-
268
+
269
269
  images_with_valid_classes = list(compress(images, b_valid_class))
270
- images = images_with_valid_classes
271
-
270
+ images = images_with_valid_classes
271
+
272
272
  # ...if we need to include/exclude categories
273
-
273
+
274
274
  # Put the annotations in a dataframe so we can select all annotations for a given image
275
275
  print('Creating data frames')
276
276
  df_anno = pd.DataFrame(annotations)
277
277
  df_img = pd.DataFrame(images)
278
-
278
+
279
279
  # Construct label map
280
280
  label_map = {}
281
281
  for cat in categories:
282
282
  label_map[int(cat['id'])] = cat['name']
283
-
283
+
284
284
  # Take a sample of images
285
285
  if options.num_to_visualize is not None:
286
286
  if options.num_to_visualize > len(df_img):
287
287
  print('Warning: asked to visualize {} images, but only {} are available, keeping them all'.\
288
288
  format(options.num_to_visualize,len(df_img)))
289
- else:
289
+ else:
290
290
  df_img = df_img.sample(n=options.num_to_visualize,random_state=options.random_seed)
291
-
291
+
292
292
  images_html = []
293
-
293
+
294
294
  # Set of dicts representing inputs to render_db_bounding_boxes:
295
295
  #
296
296
  # bboxes, box_classes, image_path
297
297
  rendering_info = []
298
-
298
+
299
299
  print('Preparing rendering list')
300
-
300
+
301
301
  for i_image,img in tqdm(df_img.iterrows(),total=len(df_img)):
302
-
302
+
303
303
  img_id = img['id']
304
304
  assert img_id is not None
305
-
305
+
306
306
  img_relative_path = img['file_name']
307
-
307
+
308
308
  if image_base_dir.startswith('http'):
309
309
  img_path = image_base_dir + img_relative_path
310
310
  else:
311
- img_path = os.path.join(image_base_dir,
311
+ img_path = os.path.join(image_base_dir,
312
312
  _image_filename_to_path(img_relative_path, image_base_dir))
313
-
313
+
314
314
  annos_i = df_anno.loc[df_anno['image_id'] == img_id, :] # all annotations on this image
315
-
315
+
316
316
  bboxes = []
317
317
  box_classes = []
318
318
  box_score_strings = []
319
-
319
+
320
320
  # All the class labels we've seen for this image (with out without bboxes)
321
321
  image_categories = set()
322
-
322
+
323
323
  extra_annotation_field_string = ''
324
324
  annotation_level_for_image = ''
325
-
325
+
326
326
  # Iterate over annotations for this image
327
327
  # i_ann = 0; anno = annos_i.iloc[i_ann]
328
328
  for i_ann,anno in annos_i.iterrows():
329
-
329
+
330
330
  if options.extra_annotation_fields_to_print is not None:
331
331
  field_names = list(anno.index)
332
332
  for field_name in field_names:
333
333
  if field_name in options.extra_annotation_fields_to_print:
334
334
  field_value = anno[field_name]
335
- if (field_value is not None) and (not isnan(field_value)):
335
+ if (field_value is not None) and (not _isnan(field_value)):
336
336
  extra_annotation_field_string += ' ({}:{})'.format(
337
- field_name,field_value)
338
-
337
+ field_name,field_value)
338
+
339
339
  if options.confidence_threshold is not None:
340
340
  assert options.confidence_field_name in anno, \
341
341
  'Error: confidence thresholding requested, ' + \
@@ -343,18 +343,18 @@ def visualize_db(db_path, output_dir, image_base_dir, options=None):
343
343
  options.confidence_field_name)
344
344
  if anno[options.confidence_field_name] < options.confidence_threshold:
345
345
  continue
346
-
346
+
347
347
  if 'sequence_level_annotation' in anno:
348
- bSequenceLevelAnnotation = anno['sequence_level_annotation']
349
- if bSequenceLevelAnnotation:
350
- annLevel = 'sequence'
348
+ b_sequence_level_annotation = anno['sequence_level_annotation']
349
+ if b_sequence_level_annotation:
350
+ annotation_level = 'sequence'
351
351
  else:
352
- annLevel = 'image'
352
+ annotation_level = 'image'
353
353
  if annotation_level_for_image == '':
354
- annotation_level_for_image = annLevel
355
- elif annotation_level_for_image != annLevel:
354
+ annotation_level_for_image = annotation_level
355
+ elif annotation_level_for_image != annotation_level:
356
356
  annotation_level_for_image = 'mixed'
357
-
357
+
358
358
  category_id = anno['category_id']
359
359
  category_name = label_map[category_id]
360
360
  if options.add_search_links:
@@ -362,66 +362,66 @@ def visualize_db(db_path, output_dir, image_base_dir, options=None):
362
362
  category_name = '<a href="https://www.google.com/search?tbm=isch&q={}">{}</a>'.format(
363
363
  category_name,category_name)
364
364
  image_categories.add(category_name)
365
-
365
+
366
366
  if 'bbox' in anno:
367
- bbox = anno['bbox']
367
+ bbox = anno['bbox']
368
368
  if isinstance(bbox,float):
369
369
  assert math.isnan(bbox), "I shouldn't see a bbox that's neither a box nor NaN"
370
370
  continue
371
371
  bboxes.append(bbox)
372
372
  box_classes.append(anno['category_id'])
373
-
373
+
374
374
  box_score_string = ''
375
375
  if options.confidence_field_name is not None and \
376
376
  options.confidence_field_name in anno:
377
377
  score = anno[options.confidence_field_name]
378
378
  box_score_string = '({}%)'.format(round(100 * score))
379
379
  box_score_strings.append(box_score_string)
380
-
380
+
381
381
  # ...for each of this image's annotations
382
-
382
+
383
383
  image_classes = ', '.join(image_categories)
384
-
385
- img_id_string = str(img_id).lower()
384
+
385
+ img_id_string = str(img_id).lower()
386
386
  file_name = '{}_gt.jpg'.format(os.path.splitext(img_id_string)[0])
387
-
387
+
388
388
  # Replace characters that muck up image links
389
389
  illegal_characters = ['/','\\',':','\t','#',' ','%']
390
390
  for c in illegal_characters:
391
391
  file_name = file_name.replace(c,'~')
392
-
392
+
393
393
  rendering_info_this_image = {'bboxes':bboxes,
394
394
  'box_classes':box_classes,
395
395
  'tags':box_score_strings,
396
396
  'img_path':img_path,
397
397
  'output_file_name':file_name}
398
398
  rendering_info.append(rendering_info_this_image)
399
-
399
+
400
400
  label_level_string = ''
401
401
  if len(annotation_level_for_image) > 0:
402
402
  label_level_string = ' (annotation level: {})'.format(annotation_level_for_image)
403
-
403
+
404
404
  if 'frame_num' in img and 'seq_num_frames' in img:
405
405
  frame_string = ' frame: {} of {},'.format(img['frame_num'],img['seq_num_frames'])
406
406
  elif 'frame_num' in img:
407
407
  frame_string = ' frame: {},'.format(img['frame_num'])
408
408
  else:
409
409
  frame_string = ''
410
-
410
+
411
411
  if options.show_full_paths:
412
412
  filename_text = img_path
413
413
  else:
414
414
  filename_text = img_relative_path
415
415
  if options.include_filename_links:
416
416
  filename_text = '<a href="{}">{}</a>'.format(img_path,filename_text)
417
-
417
+
418
418
  flag_string = ''
419
-
420
- if ('flags' in img) and (not isnan(img['flags'])):
419
+
420
+ if ('flags' in img) and (not _isnan(img['flags'])):
421
421
  flag_string = ', flags: {}'.format(str(img['flags']))
422
-
422
+
423
423
  extra_field_string = ''
424
-
424
+
425
425
  if options.extra_image_fields_to_print is not None:
426
426
  for field_name in options.extra_image_fields_to_print:
427
427
  if field_name in img:
@@ -429,7 +429,7 @@ def visualize_db(db_path, output_dir, image_base_dir, options=None):
429
429
  # previous field in [extra_fields_to_print] or from the rest of the string
430
430
  extra_field_string += ', {}: {}'.format(
431
431
  field_name,str(img[field_name]))
432
-
432
+
433
433
  # We're adding html for an image before we render it, so it's possible this image will
434
434
  # fail to render. For applications where this script is being used to debua a database
435
435
  # (the common case?), this is useful behavior, for other applications, this is annoying.
@@ -437,39 +437,39 @@ def visualize_db(db_path, output_dir, image_base_dir, options=None):
437
437
  {
438
438
  'filename': '{}/{}'.format('rendered_images', file_name),
439
439
  'title': '{}<br/>{}, num boxes: {},{} class labels: {}{}{}{}{}'.format(
440
- filename_text, img_id, len(bboxes), frame_string, image_classes,
440
+ filename_text, img_id, len(bboxes), frame_string, image_classes,
441
441
  label_level_string, flag_string, extra_field_string, extra_annotation_field_string),
442
442
  'textStyle': 'font-family:verdana,arial,calibri;font-size:80%;' + \
443
443
  'text-align:left;margin-top:20;margin-bottom:5'
444
444
  }
445
445
  if options.include_image_links:
446
446
  image_dict['linkTarget'] = img_path
447
-
447
+
448
448
  images_html.append(image_dict)
449
-
449
+
450
450
  # ...for each image
451
451
 
452
452
  def render_image_info(rendering_info):
453
-
453
+
454
454
  img_path = rendering_info['img_path']
455
455
  bboxes = rendering_info['bboxes']
456
456
  bbox_classes = rendering_info['box_classes']
457
457
  bbox_tags = None
458
458
  if 'tags' in rendering_info:
459
- bbox_tags = rendering_info['tags']
459
+ bbox_tags = rendering_info['tags']
460
460
  output_file_name = rendering_info['output_file_name']
461
461
  output_full_path = os.path.join(output_dir, 'rendered_images', output_file_name)
462
-
462
+
463
463
  if (os.path.isfile(output_full_path)) and (not options.force_rendering):
464
464
  if options.verbose:
465
465
  print('Skipping existing image {}'.format(output_full_path))
466
466
  return True
467
-
467
+
468
468
  if not img_path.startswith('http'):
469
469
  if not os.path.exists(img_path):
470
470
  print('Image {} cannot be found'.format(img_path))
471
471
  return False
472
-
472
+
473
473
  try:
474
474
  original_image = vis_utils.open_image(img_path)
475
475
  original_size = original_image.size
@@ -481,147 +481,154 @@ def visualize_db(db_path, output_dir, image_base_dir, options=None):
481
481
  except Exception as e:
482
482
  print('Image {} failed to open, error: {}'.format(img_path, e))
483
483
  return False
484
-
485
- vis_utils.render_db_bounding_boxes(boxes=bboxes,
484
+
485
+ vis_utils.render_db_bounding_boxes(boxes=bboxes,
486
486
  classes=bbox_classes,
487
- image=image,
487
+ image=image,
488
488
  original_size=original_size,
489
489
  label_map=label_map,
490
490
  thickness=options.box_thickness,
491
491
  expansion=options.box_expansion,
492
492
  tags=bbox_tags)
493
-
493
+
494
494
  image.save(output_full_path)
495
-
495
+
496
496
  return True
497
-
497
+
498
498
  # ...def render_image_info
499
-
499
+
500
500
  print('Rendering images')
501
501
  start_time = time.time()
502
-
502
+
503
503
  if options.parallelize_rendering:
504
-
504
+
505
505
  if options.parallelize_rendering_with_threads:
506
506
  worker_string = 'threads'
507
507
  else:
508
508
  worker_string = 'processes'
509
-
510
- if options.parallelize_rendering_n_cores is None:
511
- if options.parallelize_rendering_with_threads:
512
- pool = ThreadPool()
513
- else:
514
- pool = Pool()
515
- else:
516
- if options.parallelize_rendering_with_threads:
517
- pool = ThreadPool(options.parallelize_rendering_n_cores)
509
+
510
+ pool = None
511
+ try:
512
+ if options.parallelize_rendering_n_cores is None:
513
+ if options.parallelize_rendering_with_threads:
514
+ pool = ThreadPool()
515
+ else:
516
+ pool = Pool()
518
517
  else:
519
- pool = Pool(options.parallelize_rendering_n_cores)
520
- print('Rendering images with {} {}'.format(options.parallelize_rendering_n_cores,
521
- worker_string))
522
- rendering_success = list(tqdm(pool.imap(render_image_info, rendering_info),
523
- total=len(rendering_info)))
524
-
518
+ if options.parallelize_rendering_with_threads:
519
+ pool = ThreadPool(options.parallelize_rendering_n_cores)
520
+ else:
521
+ pool = Pool(options.parallelize_rendering_n_cores)
522
+ print('Rendering images with {} {}'.format(options.parallelize_rendering_n_cores,
523
+ worker_string))
524
+ rendering_success = list(tqdm(pool.imap(render_image_info, rendering_info),
525
+ total=len(rendering_info)))
526
+ finally:
527
+ if pool is not None:
528
+ pool.close()
529
+ pool.join()
530
+ print("Pool closed and joined for DB visualization")
531
+
525
532
  else:
526
-
533
+
527
534
  rendering_success = []
528
- for file_info in tqdm(rendering_info):
535
+ for file_info in tqdm(rendering_info):
529
536
  rendering_success.append(render_image_info(file_info))
530
-
537
+
531
538
  elapsed = time.time() - start_time
532
-
539
+
533
540
  print('Rendered {} images in {} ({} successful)'.format(
534
541
  len(rendering_info),humanfriendly.format_timespan(elapsed),sum(rendering_success)))
535
-
536
- if options.sort_by_filename:
542
+
543
+ if options.sort_by_filename:
537
544
  images_html = sorted(images_html, key=lambda x: x['filename'])
538
545
  else:
539
546
  random.shuffle(images_html)
540
-
541
- htmlOutputFile = os.path.join(output_dir, 'index.html')
542
-
543
- htmlOptions = options.htmlOptions
547
+
548
+ html_output_file = os.path.join(output_dir, 'index.html')
549
+
550
+ html_options = options.html_options
544
551
  if isinstance(db_path,str):
545
- htmlOptions['headerHtml'] = '<h1>Sample annotations from {}</h1>'.format(db_path)
552
+ html_options['headerHtml'] = '<h1>Sample annotations from {}</h1>'.format(db_path)
546
553
  else:
547
- htmlOptions['headerHtml'] = '<h1>Sample annotations</h1>'
548
-
554
+ html_options['headerHtml'] = '<h1>Sample annotations</h1>'
555
+
549
556
  write_html_image_list(
550
- filename=htmlOutputFile,
557
+ filename=html_output_file,
551
558
  images=images_html,
552
- options=htmlOptions)
559
+ options=html_options)
560
+
561
+ print('Visualized {} images, wrote results to {}'.format(len(images_html),html_output_file))
553
562
 
554
- print('Visualized {} images, wrote results to {}'.format(len(images_html),htmlOutputFile))
555
-
556
- return htmlOutputFile,image_db
563
+ return html_output_file,image_db
557
564
 
558
565
  # def visualize_db(...)
559
-
560
-
566
+
567
+
561
568
  #%% Command-line driver
562
-
563
- # Copy all fields from a Namespace (i.e., the output from parse_args) to an object.
569
+
570
+ # Copy all fields from a Namespace (i.e., the output from parse_args) to an object.
564
571
  #
565
572
  # Skips fields starting with _. Does not check existence in the target object.
566
- def args_to_object(args, obj):
567
-
573
+ def _args_to_object(args, obj):
574
+
568
575
  for n, v in inspect.getmembers(args):
569
576
  if not n.startswith('_'):
570
577
  setattr(obj, n, v)
571
578
 
572
579
 
573
- def main():
574
-
580
+ def main(): # noqa
581
+
575
582
  parser = argparse.ArgumentParser()
576
- parser.add_argument('db_path', action='store', type=str,
583
+ parser.add_argument('db_path', action='store', type=str,
577
584
  help='.json file to visualize')
578
- parser.add_argument('output_dir', action='store', type=str,
585
+ parser.add_argument('output_dir', action='store', type=str,
579
586
  help='Output directory for html and rendered images')
580
- parser.add_argument('image_base_dir', action='store', type=str,
587
+ parser.add_argument('image_base_dir', action='store', type=str,
581
588
  help='Base directory (or URL) for input images')
582
589
 
583
- parser.add_argument('--num_to_visualize', action='store', type=int, default=None,
590
+ parser.add_argument('--num_to_visualize', action='store', type=int, default=None,
584
591
  help='Number of images to visualize (randomly drawn) (defaults to all)')
585
- parser.add_argument('--random_sort', action='store_true',
592
+ parser.add_argument('--random_sort', action='store_true',
586
593
  help='Sort randomly (rather than by filename) in output html')
587
- parser.add_argument('--trim_to_images_with_bboxes', action='store_true',
594
+ parser.add_argument('--trim_to_images_with_bboxes', action='store_true',
588
595
  help='Only include images with bounding boxes (defaults to false)')
589
596
  parser.add_argument('--random_seed', action='store', type=int, default=None,
590
597
  help='Random seed for image selection')
591
598
  parser.add_argument('--pathsep_replacement', action='store', type=str, default='',
592
599
  help='Replace path separators in relative filenames with another ' + \
593
600
  'character (frequently ~)')
594
-
601
+
595
602
  if len(sys.argv[1:]) == 0:
596
603
  parser.print_help()
597
604
  parser.exit()
598
-
605
+
599
606
  args = parser.parse_args()
600
-
607
+
601
608
  # Convert to an options object
602
609
  options = DbVizOptions()
603
- args_to_object(args, options)
610
+ _args_to_object(args, options)
604
611
  if options.random_sort:
605
612
  options.sort_by_filename = False
606
-
607
- visualize_db(options.db_path,options.output_dir,options.image_base_dir,options)
608
613
 
609
- if __name__ == '__main__':
614
+ visualize_db(options.db_path,options.output_dir,options.image_base_dir,options)
615
+
616
+ if __name__ == '__main__':
610
617
  main()
611
618
 
612
619
 
613
620
  #%% Interactive driver
614
621
 
615
622
  if False:
616
-
623
+
617
624
  #%%
618
-
625
+
619
626
  db_path = r'e:\wildlife_data\missouri_camera_traps\missouri_camera_traps_set1.json'
620
627
  output_dir = r'e:\wildlife_data\missouri_camera_traps\preview'
621
628
  image_base_dir = r'e:\wildlife_data\missouri_camera_traps'
622
-
629
+
623
630
  options = DbVizOptions()
624
631
  options.num_to_visualize = 100
625
-
626
- htmlOutputFile,db = visualize_db(db_path,output_dir,image_base_dir,options)
627
- # os.startfile(htmlOutputFile)
632
+
633
+ html_output_file, db = visualize_db(db_path,output_dir,image_base_dir,options)
634
+ # os.startfile(html_output_file)