megadetector 10.0.9__py3-none-any.whl → 10.0.11__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of megadetector might be problematic. Click here for more details.

Files changed (84) hide show
  1. megadetector/data_management/animl_to_md.py +5 -2
  2. megadetector/data_management/cct_json_utils.py +4 -2
  3. megadetector/data_management/cct_to_md.py +5 -4
  4. megadetector/data_management/cct_to_wi.py +5 -1
  5. megadetector/data_management/coco_to_yolo.py +3 -2
  6. megadetector/data_management/databases/combine_coco_camera_traps_files.py +4 -4
  7. megadetector/data_management/databases/integrity_check_json_db.py +2 -2
  8. megadetector/data_management/databases/subset_json_db.py +0 -3
  9. megadetector/data_management/generate_crops_from_cct.py +6 -4
  10. megadetector/data_management/get_image_sizes.py +5 -35
  11. megadetector/data_management/labelme_to_coco.py +10 -6
  12. megadetector/data_management/labelme_to_yolo.py +19 -28
  13. megadetector/data_management/lila/create_lila_test_set.py +22 -2
  14. megadetector/data_management/lila/generate_lila_per_image_labels.py +7 -5
  15. megadetector/data_management/lila/lila_common.py +2 -2
  16. megadetector/data_management/lila/test_lila_metadata_urls.py +0 -1
  17. megadetector/data_management/ocr_tools.py +6 -10
  18. megadetector/data_management/read_exif.py +69 -13
  19. megadetector/data_management/remap_coco_categories.py +1 -1
  20. megadetector/data_management/remove_exif.py +10 -5
  21. megadetector/data_management/rename_images.py +20 -13
  22. megadetector/data_management/resize_coco_dataset.py +10 -4
  23. megadetector/data_management/speciesnet_to_md.py +3 -3
  24. megadetector/data_management/yolo_output_to_md_output.py +3 -1
  25. megadetector/data_management/yolo_to_coco.py +28 -19
  26. megadetector/detection/change_detection.py +26 -18
  27. megadetector/detection/process_video.py +1 -1
  28. megadetector/detection/pytorch_detector.py +5 -5
  29. megadetector/detection/run_detector.py +34 -10
  30. megadetector/detection/run_detector_batch.py +60 -42
  31. megadetector/detection/run_inference_with_yolov5_val.py +3 -1
  32. megadetector/detection/run_md_and_speciesnet.py +282 -110
  33. megadetector/detection/run_tiled_inference.py +7 -7
  34. megadetector/detection/tf_detector.py +4 -6
  35. megadetector/detection/video_utils.py +9 -6
  36. megadetector/postprocessing/add_max_conf.py +4 -4
  37. megadetector/postprocessing/categorize_detections_by_size.py +3 -2
  38. megadetector/postprocessing/classification_postprocessing.py +19 -21
  39. megadetector/postprocessing/combine_batch_outputs.py +3 -2
  40. megadetector/postprocessing/compare_batch_results.py +49 -27
  41. megadetector/postprocessing/convert_output_format.py +8 -6
  42. megadetector/postprocessing/create_crop_folder.py +13 -4
  43. megadetector/postprocessing/generate_csv_report.py +22 -8
  44. megadetector/postprocessing/load_api_results.py +8 -4
  45. megadetector/postprocessing/md_to_coco.py +2 -3
  46. megadetector/postprocessing/md_to_labelme.py +12 -8
  47. megadetector/postprocessing/md_to_wi.py +2 -1
  48. megadetector/postprocessing/merge_detections.py +4 -6
  49. megadetector/postprocessing/postprocess_batch_results.py +4 -3
  50. megadetector/postprocessing/remap_detection_categories.py +6 -3
  51. megadetector/postprocessing/render_detection_confusion_matrix.py +18 -10
  52. megadetector/postprocessing/repeat_detection_elimination/remove_repeat_detections.py +1 -1
  53. megadetector/postprocessing/repeat_detection_elimination/repeat_detections_core.py +5 -3
  54. megadetector/postprocessing/separate_detections_into_folders.py +10 -4
  55. megadetector/postprocessing/subset_json_detector_output.py +1 -1
  56. megadetector/postprocessing/top_folders_to_bottom.py +22 -7
  57. megadetector/postprocessing/validate_batch_results.py +1 -1
  58. megadetector/taxonomy_mapping/map_new_lila_datasets.py +59 -3
  59. megadetector/taxonomy_mapping/prepare_lila_taxonomy_release.py +1 -1
  60. megadetector/taxonomy_mapping/preview_lila_taxonomy.py +26 -17
  61. megadetector/taxonomy_mapping/species_lookup.py +51 -2
  62. megadetector/utils/ct_utils.py +9 -4
  63. megadetector/utils/directory_listing.py +3 -0
  64. megadetector/utils/extract_frames_from_video.py +4 -0
  65. megadetector/utils/gpu_test.py +6 -6
  66. megadetector/utils/md_tests.py +21 -21
  67. megadetector/utils/path_utils.py +171 -36
  68. megadetector/utils/split_locations_into_train_val.py +0 -4
  69. megadetector/utils/string_utils.py +21 -0
  70. megadetector/utils/url_utils.py +5 -3
  71. megadetector/utils/wi_platform_utils.py +168 -24
  72. megadetector/utils/wi_taxonomy_utils.py +38 -8
  73. megadetector/utils/write_html_image_list.py +1 -2
  74. megadetector/visualization/plot_utils.py +31 -19
  75. megadetector/visualization/render_images_with_thumbnails.py +3 -0
  76. megadetector/visualization/visualization_utils.py +18 -7
  77. megadetector/visualization/visualize_db.py +9 -26
  78. megadetector/visualization/visualize_detector_output.py +1 -0
  79. megadetector/visualization/visualize_video_output.py +14 -2
  80. {megadetector-10.0.9.dist-info → megadetector-10.0.11.dist-info}/METADATA +1 -1
  81. {megadetector-10.0.9.dist-info → megadetector-10.0.11.dist-info}/RECORD +84 -84
  82. {megadetector-10.0.9.dist-info → megadetector-10.0.11.dist-info}/WHEEL +0 -0
  83. {megadetector-10.0.9.dist-info → megadetector-10.0.11.dist-info}/licenses/LICENSE +0 -0
  84. {megadetector-10.0.9.dist-info → megadetector-10.0.11.dist-info}/top_level.txt +0 -0
@@ -32,6 +32,7 @@ from megadetector.utils.ct_utils import write_json
32
32
  from megadetector.utils.ct_utils import make_temp_folder
33
33
  from megadetector.utils.ct_utils import is_list_sorted
34
34
  from megadetector.utils.ct_utils import is_sphinx_build
35
+ from megadetector.utils.ct_utils import args_to_object
35
36
  from megadetector.utils import path_utils
36
37
  from megadetector.visualization import visualization_utils as vis_utils
37
38
  from megadetector.postprocessing.validate_batch_results import \
@@ -61,8 +62,21 @@ DEFAULT_DETECTION_CONFIDENCE_THRESHOLD_FOR_OUTPUT = DEFAULT_OUTPUT_CONFIDENCE_TH
61
62
  DEFAULT_DETECTOR_BATCH_SIZE = 1
62
63
  DEFAULT_CLASSIFIER_BATCH_SIZE = 8
63
64
  DEFAULT_LOADER_WORKERS = 4
64
- MAX_QUEUE_SIZE_IMAGES_PER_WORKER = 10
65
- DEAFULT_SECONDS_PER_VIDEO_FRAME = 1.0
65
+
66
+ # This determines the maximum number of image filenames that can be assigned to
67
+ # each of the producer workers before blocking. The actual size of the queue
68
+ # will be MAX_IMAGE_QUEUE_SIZE_PER_WORKER * n_workers. This is only used for
69
+ # the classification step.
70
+ MAX_IMAGE_QUEUE_SIZE_PER_WORKER = 30
71
+
72
+ # This determines the maximum number of crops that can accumulate in the queue
73
+ # used to communicate between the producers (which read and crop images) and the
74
+ # consumer (which runs the classifier). This is only used for the classification step.
75
+ MAX_BATCH_QUEUE_SIZE = 300
76
+
77
+ # Default interval between frames we should process when processing video.
78
+ # This is only used for the detection step.
79
+ DEFAULT_SECONDS_PER_VIDEO_FRAME = 1.0
66
80
 
67
81
  # Max number of classification scores to include per detection
68
82
  DEFAULT_TOP_N_SCORES = 2
@@ -71,6 +85,11 @@ DEFAULT_TOP_N_SCORES = 2
71
85
  # cumulative confidence is above this value
72
86
  ROLLUP_TARGET_CONFIDENCE = 0.5
73
87
 
88
+ # When the called supplies an existing MD results file, should we validate it before
89
+ # starting classification? This tends
90
+ VALIDATE_DETECTION_FILE = False
91
+
92
+
74
93
  verbose = False
75
94
 
76
95
 
@@ -109,10 +128,10 @@ class CropBatch:
109
128
  """
110
129
 
111
130
  def __init__(self):
112
- # List of preprocessed images
131
+ #: List of preprocessed images
113
132
  self.crops = []
114
133
 
115
- # List of CropMetadata objects
134
+ #: List of CropMetadata objects
116
135
  self.metadata = []
117
136
 
118
137
  def add_crop(self, crop_data, metadata):
@@ -152,6 +171,12 @@ def _process_image_detections(file_path: str,
152
171
 
153
172
  detections = detection_results['detections']
154
173
 
174
+ # Don't bother loading images that have no above-threshold detections
175
+ detections_above_threshold = \
176
+ [d for d in detections if d['conf'] >= detection_confidence_threshold]
177
+ if len(detections_above_threshold) == 0:
178
+ return
179
+
155
180
  # Load the image
156
181
  try:
157
182
  image = vis_utils.load_image(absolute_file_path)
@@ -173,6 +198,11 @@ def _process_image_detections(file_path: str,
173
198
  return
174
199
 
175
200
  # Process each detection above threshold
201
+ #
202
+ # detection_index needs to index into the original list of detections
203
+ # (this is how classification results will be associated with detections
204
+ # later), so iterate over "detections" here, rather than
205
+ # "detections_above_threshold".
176
206
  for detection_index, detection in enumerate(detections):
177
207
 
178
208
  conf = detection['conf']
@@ -192,6 +222,7 @@ def _process_image_detections(file_path: str,
192
222
 
193
223
  # Preprocess the crop
194
224
  try:
225
+
195
226
  preprocessed_crop = classifier.preprocess(
196
227
  image,
197
228
  bboxes=[speciesnet_bbox],
@@ -199,6 +230,7 @@ def _process_image_detections(file_path: str,
199
230
  )
200
231
 
201
232
  if preprocessed_crop is not None:
233
+
202
234
  metadata = CropMetadata(
203
235
  image_file=file_path,
204
236
  detection_index=detection_index,
@@ -207,10 +239,11 @@ def _process_image_detections(file_path: str,
207
239
  original_height=original_height
208
240
  )
209
241
 
210
- # Send individual crop immediately to consumer
242
+ # Send individual crop to the consumer
211
243
  batch_queue.put(('crop', preprocessed_crop, metadata))
212
244
 
213
245
  except Exception as e:
246
+
214
247
  print('Warning: failed to preprocess crop from {}, detection {}: {}'.format(
215
248
  file_path, detection_index, str(e)))
216
249
 
@@ -226,6 +259,8 @@ def _process_image_detections(file_path: str,
226
259
  'Failed to preprocess crop: {}'.format(str(e)),
227
260
  failure_metadata))
228
261
 
262
+ # ...try/except
263
+
229
264
  # ...for each detection in this image
230
265
 
231
266
  # ...def _process_image_detections(...)
@@ -256,6 +291,7 @@ def _process_video_detections(file_path: str,
256
291
  frame_to_detections = {}
257
292
 
258
293
  for detection_index, detection in enumerate(detections):
294
+
259
295
  conf = detection['conf']
260
296
  if conf < detection_confidence_threshold:
261
297
  continue
@@ -267,6 +303,8 @@ def _process_video_detections(file_path: str,
267
303
  frame_to_detections[frame_number] = []
268
304
  frame_to_detections[frame_number].append((detection_index, detection))
269
305
 
306
+ # ...for each detection in this video
307
+
270
308
  if len(frames_with_detections) == 0:
271
309
  return
272
310
 
@@ -290,6 +328,7 @@ def _process_video_detections(file_path: str,
290
328
  return
291
329
  frame_number = int(match.group(1))
292
330
 
331
+ # Only process frames for which we have detection results
293
332
  if frame_number not in frame_to_detections:
294
333
  return
295
334
 
@@ -360,13 +399,16 @@ def _process_video_detections(file_path: str,
360
399
 
361
400
  # Process the video frames
362
401
  try:
402
+
363
403
  run_callback_on_frames(
364
404
  input_video_file=absolute_file_path,
365
405
  frame_callback=frame_callback,
366
406
  frames_to_process=frames_to_process,
367
407
  verbose=verbose
368
408
  )
409
+
369
410
  except Exception as e:
411
+
370
412
  print('Warning: failed to process video {}: {}'.format(file_path, str(e)))
371
413
 
372
414
  # Send failure information to consumer for the whole video
@@ -448,6 +490,7 @@ def _crop_producer_func(image_queue: JoinableQueue,
448
490
  is_video = is_video_file(file_path)
449
491
 
450
492
  if is_video:
493
+
451
494
  # Process video
452
495
  _process_video_detections(
453
496
  file_path=file_path,
@@ -457,7 +500,9 @@ def _crop_producer_func(image_queue: JoinableQueue,
457
500
  detection_confidence_threshold=detection_confidence_threshold,
458
501
  batch_queue=batch_queue
459
502
  )
503
+
460
504
  else:
505
+
461
506
  # Process image
462
507
  _process_image_detections(
463
508
  file_path=file_path,
@@ -531,7 +576,7 @@ def _crop_consumer_func(batch_queue: Queue,
531
576
  taxonomy_map = {}
532
577
  geofence_map = {}
533
578
 
534
- if (enable_rollup is not None) or (country is not None):
579
+ if (enable_rollup) or (country is not None):
535
580
 
536
581
  # Note to self: there are a number of reasons loading the ensemble
537
582
  # could fail here; don't catch this exception, this should be a
@@ -571,9 +616,9 @@ def _crop_consumer_func(batch_queue: Queue,
571
616
  item_type, data, metadata = item
572
617
 
573
618
  if metadata.image_file not in all_results:
574
- all_results[metadata.image_file] = {}
619
+ all_results[metadata.image_file] = {}
575
620
 
576
- # We should never be processing the same detetion twice
621
+ # We should never be processing the same detection twice
577
622
  assert metadata.detection_index not in all_results[metadata.image_file]
578
623
 
579
624
  if item_type == 'failure':
@@ -601,6 +646,7 @@ def _crop_consumer_func(batch_queue: Queue,
601
646
 
602
647
  # ...while (we have items to process)
603
648
 
649
+ # Send all the results at once back to the main process
604
650
  results_queue.put(all_results)
605
651
 
606
652
  if verbose:
@@ -828,7 +874,7 @@ def _run_detection_step(source_folder: str,
828
874
  batch_size=detector_batch_size,
829
875
  include_image_size=False,
830
876
  include_image_timestamp=False,
831
- include_exif_data=False,
877
+ include_exif_tags=None,
832
878
  loader_workers=detector_worker_threads,
833
879
  preprocess_on_image_queue=True
834
880
  )
@@ -914,9 +960,11 @@ def _run_classification_step(detector_results_file: str,
914
960
  top_n_scores (int, optional): maximum number of scores to include for each detection
915
961
  """
916
962
 
917
- print('Starting SpeciesNet classification step...')
963
+ print('Starting classification step...')
918
964
 
919
965
  # Load MegaDetector results
966
+ print('Reading detection results from {}'.format(detector_results_file))
967
+
920
968
  with open(detector_results_file, 'r') as f:
921
969
  detector_results = json.load(f)
922
970
 
@@ -936,10 +984,22 @@ def _run_classification_step(detector_results_file: str,
936
984
  print('Set multiprocessing start method to spawn (was {})'.format(
937
985
  original_start_method))
938
986
 
939
- # Set up multiprocessing queues
940
- max_queue_size = classifier_worker_threads * MAX_QUEUE_SIZE_IMAGES_PER_WORKER
941
- image_queue = JoinableQueue(max_queue_size)
942
- batch_queue = Queue()
987
+ ## Set up multiprocessing queues
988
+
989
+ # This queue receives lists of image filenames (and associated detection results)
990
+ # from the "main" thread (the one you're reading right now). Items are pulled off
991
+ # of this queue by producer workers (on _crop_producer_func), where the corresponding
992
+ # images are loaded from disk and preprocessed into crops.
993
+ image_queue = JoinableQueue(maxsize= \
994
+ classifier_worker_threads * MAX_IMAGE_QUEUE_SIZE_PER_WORKER)
995
+
996
+ # This queue receives cropped images from producers (on _crop_producer_func); those
997
+ # crops are pulled off of this queue by the consumer (on _crop_consumer_func).
998
+ batch_queue = Queue(maxsize=MAX_BATCH_QUEUE_SIZE)
999
+
1000
+ # This is not really used as a queue, rather it's just used to send all the results
1001
+ # at once from the consumer process to the main process (the one you're reading right
1002
+ # now).
943
1003
  results_queue = Queue()
944
1004
 
945
1005
  # Start producer workers
@@ -951,7 +1011,9 @@ def _run_classification_step(detector_results_file: str,
951
1011
  p.start()
952
1012
  producers.append(p)
953
1013
 
954
- # Start consumer worker
1014
+
1015
+ ## Start consumer worker
1016
+
955
1017
  consumer = Process(target=_crop_consumer_func,
956
1018
  args=(batch_queue, results_queue, classifier_model,
957
1019
  classifier_batch_size, classifier_worker_threads,
@@ -974,16 +1036,23 @@ def _run_classification_step(detector_results_file: str,
974
1036
 
975
1037
  print('Finished waiting for input queue')
976
1038
 
977
- # Wait for results
1039
+
1040
+ ## Wait for results
1041
+
978
1042
  classification_results = results_queue.get()
979
1043
 
980
- # Clean up processes
1044
+
1045
+ ## Clean up processes
1046
+
981
1047
  for p in producers:
982
1048
  p.join()
983
1049
  consumer.join()
984
1050
 
985
1051
  print('Finished waiting for workers')
986
1052
 
1053
+
1054
+ ## Format results and write output
1055
+
987
1056
  class CategoryState:
988
1057
  """
989
1058
  Helper class to manage classification category IDs.
@@ -1091,7 +1160,7 @@ def _run_classification_step(detector_results_file: str,
1091
1160
 
1092
1161
  # Add classifications to the detection
1093
1162
  detection['classifications'] = classification_pairs
1094
- detection['raw_classifications'] = raw_classification_pairs
1163
+ # detection['raw_classifications'] = raw_classification_pairs
1095
1164
 
1096
1165
  # ...if this classification contains a failure
1097
1166
 
@@ -1126,6 +1195,196 @@ def _run_classification_step(detector_results_file: str,
1126
1195
  # ...def _run_classification_step(...)
1127
1196
 
1128
1197
 
1198
+ #%% Options class
1199
+
1200
+ class RunMDSpeciesNetOptions:
1201
+ """
1202
+ Class controlling the behavior of run_md_and_speciesnet()
1203
+ """
1204
+
1205
+ def __init__(self):
1206
+
1207
+ #: Folder containing images and/or videos to process
1208
+ self.source = None
1209
+
1210
+ #: Output file for results (JSON format)
1211
+ self.output_file = None
1212
+
1213
+ #: MegaDetector model identifier (MDv5a, MDv5b, MDv1000-redwood, etc.)
1214
+ self.detector_model = DEFAULT_DETECTOR_MODEL
1215
+
1216
+ #: SpeciesNet classifier model identifier (e.g. kaggle:google/speciesnet/pyTorch/v4.0.1a)
1217
+ self.classification_model = DEFAULT_CLASSIFIER_MODEL
1218
+
1219
+ #: Batch size for MegaDetector inference
1220
+ self.detector_batch_size = DEFAULT_DETECTOR_BATCH_SIZE
1221
+
1222
+ #: Batch size for SpeciesNet classification
1223
+ self.classifier_batch_size = DEFAULT_CLASSIFIER_BATCH_SIZE
1224
+
1225
+ #: Number of worker threads for preprocessing
1226
+ self.loader_workers = DEFAULT_LOADER_WORKERS
1227
+
1228
+ #: Classify detections above this threshold
1229
+ self.detection_confidence_threshold_for_classification = \
1230
+ DEFAULT_DETECTION_CONFIDENCE_THRESHOLD_FOR_CLASSIFICATION
1231
+
1232
+ #: Include detections above this threshold in the output
1233
+ self.detection_confidence_threshold_for_output = \
1234
+ DEFAULT_DETECTION_CONFIDENCE_THRESHOLD_FOR_OUTPUT
1235
+
1236
+ #: Folder for intermediate files (default: system temp)
1237
+ self.intermediate_file_folder = None
1238
+
1239
+ #: Keep intermediate files (e.g. detection-only results file)
1240
+ self.keep_intermediate_files = False
1241
+
1242
+ #: Disable taxonomic rollup
1243
+ self.norollup = False
1244
+
1245
+ #: Country code (ISO 3166-1 alpha-3) for geofencing (default None, no geoferencing)
1246
+ self.country = None
1247
+
1248
+ #: Admin1 region/state code for geofencing
1249
+ self.admin1_region = None
1250
+
1251
+ #: Path to existing MegaDetector output file (skips detection step)
1252
+ self.detections_file = None
1253
+
1254
+ #: Ignore videos, only process images
1255
+ self.skip_video = False
1256
+
1257
+ #: Ignore images, only process videos
1258
+ self.skip_images = False
1259
+
1260
+ #: Sample every Nth frame from videos
1261
+ #:
1262
+ #: Mutually exclusive with time_sample
1263
+ self.frame_sample = None
1264
+
1265
+ #: Sample frames every N seconds from videos
1266
+ #:
1267
+ #: Mutually exclusive with frame_sample
1268
+ self.time_sample = DEFAULT_SECONDS_PER_VIDEO_FRAME
1269
+
1270
+ #: Enable additional debug output
1271
+ self.verbose = False
1272
+
1273
+ if self.time_sample is None and self.frame_sample is None:
1274
+ self.time_sample = DEFAULT_SECONDS_PER_VIDEO_FRAME
1275
+
1276
+ # ...class RunMDSpeciesNetOptions
1277
+
1278
+
1279
+ #%% Main function
1280
+
1281
+ def run_md_and_speciesnet(options):
1282
+ """
1283
+ Main entry point, runs MegaDetector and SpeciesNet on a folder. See
1284
+ RunMDSpeciesNetOptions for available arguments.
1285
+
1286
+ Args:
1287
+ options (RunMDSpeciesNetOptions): options controlling MD and SN inference
1288
+ """
1289
+
1290
+ # Set global verbose flag
1291
+ global verbose
1292
+ verbose = options.verbose
1293
+
1294
+ # Also set the run_detector_batch verbose flag
1295
+ run_detector_batch.verbose = verbose
1296
+
1297
+ # Validate arguments
1298
+ if not os.path.isdir(options.source):
1299
+ raise ValueError(
1300
+ 'Source folder does not exist: {}'.format(options.source))
1301
+
1302
+ if (options.admin1_region is not None) and (options.country is None):
1303
+ raise ValueError('--admin1_region requires --country to be specified')
1304
+
1305
+ if options.skip_images and options.skip_video:
1306
+ raise ValueError('Cannot skip both images and videos')
1307
+
1308
+ if (options.frame_sample is not None) and (options.time_sample is not None):
1309
+ raise ValueError('--frame_sample and --time_sample are mutually exclusive')
1310
+ if (options.frame_sample is None) and (options.time_sample is None):
1311
+ options.time_sample = DEFAULT_SECONDS_PER_VIDEO_FRAME
1312
+
1313
+ # Set up intermediate file folder
1314
+ if options.intermediate_file_folder:
1315
+ temp_folder = options.intermediate_file_folder
1316
+ os.makedirs(temp_folder, exist_ok=True)
1317
+ else:
1318
+ temp_folder = make_temp_folder(subfolder='run_md_and_speciesnet')
1319
+
1320
+ start_time = time.time()
1321
+
1322
+ print('Processing folder: {}'.format(options.source))
1323
+ print('Output file: {}'.format(options.output_file))
1324
+ print('Intermediate files: {}'.format(temp_folder))
1325
+
1326
+ # Determine detector output file path
1327
+ if options.detections_file is not None:
1328
+ detector_output_file = options.detections_file
1329
+ if VALIDATE_DETECTION_FILE:
1330
+ print('Using existing detections file: {}'.format(detector_output_file))
1331
+ validation_options = ValidateBatchResultsOptions()
1332
+ validation_options.check_image_existence = True
1333
+ validation_options.relative_path_base = options.source
1334
+ validation_options.raise_errors = True
1335
+ validate_batch_results(detector_output_file,options=validation_options)
1336
+ print('Validated detections file')
1337
+ else:
1338
+ print('Bypassing validation of {}'.format(options.detections_file))
1339
+ else:
1340
+ detector_output_file = os.path.join(temp_folder, 'detector_output.json')
1341
+
1342
+ # Run MegaDetector
1343
+ _run_detection_step(
1344
+ source_folder=options.source,
1345
+ detector_output_file=detector_output_file,
1346
+ detector_model=options.detector_model,
1347
+ detector_batch_size=options.detector_batch_size,
1348
+ detection_confidence_threshold=options.detection_confidence_threshold_for_output,
1349
+ detector_worker_threads=options.loader_workers,
1350
+ skip_images=options.skip_images,
1351
+ skip_video=options.skip_video,
1352
+ frame_sample=options.frame_sample,
1353
+ time_sample=options.time_sample
1354
+ )
1355
+
1356
+ # Run SpeciesNet
1357
+ _run_classification_step(
1358
+ detector_results_file=detector_output_file,
1359
+ merged_results_file=options.output_file,
1360
+ source_folder=options.source,
1361
+ classifier_model=options.classification_model,
1362
+ classifier_batch_size=options.classifier_batch_size,
1363
+ classifier_worker_threads=options.loader_workers,
1364
+ detection_confidence_threshold=options.detection_confidence_threshold_for_classification,
1365
+ enable_rollup=(not options.norollup),
1366
+ country=options.country,
1367
+ admin1_region=options.admin1_region,
1368
+ )
1369
+
1370
+ elapsed_time = time.time() - start_time
1371
+ print(
1372
+ 'Processing complete in {}'.format(humanfriendly.format_timespan(elapsed_time)))
1373
+ print('Results written to: {}'.format(options.output_file))
1374
+
1375
+ # Clean up intermediate files if requested
1376
+ if (not options.keep_intermediate_files) and \
1377
+ (not options.intermediate_file_folder) and \
1378
+ (not options.detections_file):
1379
+ try:
1380
+ os.remove(detector_output_file)
1381
+ except Exception as e:
1382
+ print('Warning: error removing temporary output file {}: {}'.format(
1383
+ detector_output_file, str(e)))
1384
+
1385
+ # ...def run_md_and_speciesnet(...)
1386
+
1387
+
1129
1388
  #%% Command-line driver
1130
1389
 
1131
1390
  def main():
@@ -1181,7 +1440,7 @@ def main():
1181
1440
  help='Folder for intermediate files (default: system temp)')
1182
1441
  parser.add_argument('--keep_intermediate_files',
1183
1442
  action='store_true',
1184
- help='Keep intermediate files for debugging')
1443
+ help='Keep intermediate files (e.g. detection-only results file)')
1185
1444
  parser.add_argument('--norollup',
1186
1445
  action='store_true',
1187
1446
  help='Disable taxonomic rollup')
@@ -1208,7 +1467,7 @@ def main():
1208
1467
  type=float,
1209
1468
  default=None,
1210
1469
  help='Sample frames every N seconds from videos (default {})'.\
1211
- format(DEAFULT_SECONDS_PER_VIDEO_FRAME) + \
1470
+ format(DEFAULT_SECONDS_PER_VIDEO_FRAME) + \
1212
1471
  ' (mutually exclusive with --frame_sample)')
1213
1472
  parser.add_argument('--verbose',
1214
1473
  action='store_true',
@@ -1220,97 +1479,10 @@ def main():
1220
1479
 
1221
1480
  args = parser.parse_args()
1222
1481
 
1223
- # Set global verbose flag
1224
- global verbose
1225
- verbose = args.verbose
1226
-
1227
- # Also set the run_detector_batch verbose flag
1228
- run_detector_batch.verbose = verbose
1482
+ options = RunMDSpeciesNetOptions()
1483
+ args_to_object(args,options)
1229
1484
 
1230
- # Validate arguments
1231
- if not os.path.isdir(args.source):
1232
- raise ValueError(
1233
- 'Source folder does not exist: {}'.format(args.source))
1234
-
1235
- if args.admin1_region and not args.country:
1236
- raise ValueError('--admin1_region requires --country to be specified')
1237
-
1238
- if args.skip_images and args.skip_video:
1239
- raise ValueError('Cannot skip both images and videos')
1240
-
1241
- if (args.frame_sample is not None) and (args.time_sample is not None):
1242
- raise ValueError('--frame_sample and --time_sample are mutually exclusive')
1243
- if (args.frame_sample is None) and (args.time_sample is None):
1244
- args.time_sample = DEAFULT_SECONDS_PER_VIDEO_FRAME
1245
-
1246
- # Set up intermediate file folder
1247
- if args.intermediate_file_folder:
1248
- temp_folder = args.intermediate_file_folder
1249
- os.makedirs(temp_folder, exist_ok=True)
1250
- else:
1251
- temp_folder = make_temp_folder(subfolder='run_md_and_speciesnet')
1252
-
1253
- start_time = time.time()
1254
-
1255
- print('Processing folder: {}'.format(args.source))
1256
- print('Output file: {}'.format(args.output_file))
1257
- print('Intermediate files: {}'.format(temp_folder))
1258
-
1259
- # Determine detector output file path
1260
- if args.detections_file:
1261
- detector_output_file = args.detections_file
1262
- print('Using existing detections file: {}'.format(detector_output_file))
1263
- validation_options = ValidateBatchResultsOptions()
1264
- validation_options.check_image_existence = True
1265
- validation_options.relative_path_base = args.source
1266
- validation_options.raise_errors = True
1267
- validate_batch_results(detector_output_file,options=validation_options)
1268
- print('Validated detections file')
1269
- else:
1270
- detector_output_file = os.path.join(temp_folder, 'detector_output.json')
1271
-
1272
- # Run MegaDetector
1273
- _run_detection_step(
1274
- source_folder=args.source,
1275
- detector_output_file=detector_output_file,
1276
- detector_model=args.detector_model,
1277
- detector_batch_size=args.detector_batch_size,
1278
- detection_confidence_threshold=args.detection_confidence_threshold_for_output,
1279
- detector_worker_threads=args.loader_workers,
1280
- skip_images=args.skip_images,
1281
- skip_video=args.skip_video,
1282
- frame_sample=args.frame_sample,
1283
- time_sample=args.time_sample
1284
- )
1285
-
1286
- # Run SpeciesNet
1287
- _run_classification_step(
1288
- detector_results_file=detector_output_file,
1289
- merged_results_file=args.output_file,
1290
- source_folder=args.source,
1291
- classifier_model=args.classification_model,
1292
- classifier_batch_size=args.classifier_batch_size,
1293
- classifier_worker_threads=args.loader_workers,
1294
- detection_confidence_threshold=args.detection_confidence_threshold_for_classification,
1295
- enable_rollup=(not args.norollup),
1296
- country=args.country,
1297
- admin1_region=args.admin1_region,
1298
- )
1299
-
1300
- elapsed_time = time.time() - start_time
1301
- print(
1302
- 'Processing complete in {}'.format(humanfriendly.format_timespan(elapsed_time)))
1303
- print('Results written to: {}'.format(args.output_file))
1304
-
1305
- # Clean up intermediate files if requested
1306
- if (not args.keep_intermediate_files) and \
1307
- (not args.intermediate_file_folder) and \
1308
- (not args.detections_file):
1309
- try:
1310
- os.remove(detector_output_file)
1311
- except Exception as e:
1312
- print('Warning: error removing temporary output file {}: {}'.format(
1313
- detector_output_file, str(e)))
1485
+ run_md_and_speciesnet(options)
1314
1486
 
1315
1487
  # ...def main(...)
1316
1488
 
@@ -510,9 +510,9 @@ def run_tiled_inference(model_file,
510
510
  try:
511
511
  fn_relative = os.path.relpath(fn,image_folder)
512
512
  except ValueError:
513
- 'Illegal absolute path supplied to run_tiled_inference, {} is outside of {}'.format(
514
- fn,image_folder)
515
- raise
513
+ raise ValueError(
514
+ 'Illegal absolute path supplied to run_tiled_inference, {} is outside of {}'.format(
515
+ fn,image_folder))
516
516
  assert not fn_relative.startswith('..'), \
517
517
  'Illegal absolute path supplied to run_tiled_inference, {} is outside of {}'.format(
518
518
  fn,image_folder)
@@ -619,12 +619,12 @@ def run_tiled_inference(model_file,
619
619
  else:
620
620
 
621
621
  patch_file_names = []
622
- for im in all_image_patch_info:
622
+ for patch_info in all_image_patch_info:
623
623
  # If there was a patch generation error, don't run inference
624
624
  if patch_info['error'] is not None:
625
- assert im['image_fn'] in images_with_patch_errors
625
+ assert patch_info['image_fn'] in images_with_patch_errors
626
626
  continue
627
- for patch in im['patches']:
627
+ for patch in patch_info['patches']:
628
628
  patch_file_names.append(patch['patch_fn'])
629
629
 
630
630
  inference_results = load_and_run_detector_batch(model_file,
@@ -952,7 +952,7 @@ def main():
952
952
  parser.add_argument(
953
953
  '--tile_size_y',
954
954
  type=int,
955
- default=default_tile_size[0],
955
+ default=default_tile_size[1],
956
956
  help=('Tile height (defaults to {})'.format(default_tile_size[1])))
957
957
  parser.add_argument(
958
958
  '--tile_overlap',
@@ -42,7 +42,7 @@ class TFDetector:
42
42
  input and output tensor handles.
43
43
 
44
44
  Args:
45
- model_path (str): path to .pdb file
45
+ model_path (str): path to .pb file
46
46
  detector_options (dict, optional): key-value pairs that control detector
47
47
  options; currently not used by TFDetector
48
48
  """
@@ -138,8 +138,8 @@ class TFDetector:
138
138
  image_id,
139
139
  detection_threshold,
140
140
  image_size=None,
141
- skip_image_resizing=False,
142
- augment=False):
141
+ augment=False,
142
+ verbose=False):
143
143
  """
144
144
  Runs the detector on an image.
145
145
 
@@ -152,10 +152,9 @@ class TFDetector:
152
152
  image_size (tuple, optional): image size to use for inference, only mess with this
153
153
  if (a) you're using a model other than MegaDetector or (b) you know what you're
154
154
  doing
155
- skip_image_resizing (bool, optional): whether to skip internal image resizing (and rely on external
156
- resizing). Not currently supported, but included here for compatibility with PTDetector.
157
155
  augment (bool, optional): enable image augmentation. Not currently supported, but included
158
156
  here for compatibility with PTDetector.
157
+ verbose (bool, optional): enable additional debug output
159
158
 
160
159
  Returns:
161
160
  dict: a dictionary with the following fields:
@@ -166,7 +165,6 @@ class TFDetector:
166
165
  """
167
166
 
168
167
  assert image_size is None, 'Image sizing not supported for TF detectors'
169
- assert not skip_image_resizing, 'Image sizing not supported for TF detectors'
170
168
  assert not augment, 'Image augmentation is not supported for TF detectors'
171
169
 
172
170
  if detection_threshold is None: