megadetector 10.0.10__py3-none-any.whl → 10.0.11__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of megadetector might be problematic. Click here for more details.

Files changed (80) hide show
  1. megadetector/data_management/animl_to_md.py +5 -2
  2. megadetector/data_management/cct_json_utils.py +4 -2
  3. megadetector/data_management/cct_to_md.py +5 -4
  4. megadetector/data_management/cct_to_wi.py +5 -1
  5. megadetector/data_management/coco_to_yolo.py +3 -2
  6. megadetector/data_management/databases/combine_coco_camera_traps_files.py +4 -4
  7. megadetector/data_management/databases/integrity_check_json_db.py +2 -2
  8. megadetector/data_management/databases/subset_json_db.py +0 -3
  9. megadetector/data_management/generate_crops_from_cct.py +6 -4
  10. megadetector/data_management/get_image_sizes.py +5 -35
  11. megadetector/data_management/labelme_to_coco.py +10 -6
  12. megadetector/data_management/labelme_to_yolo.py +19 -28
  13. megadetector/data_management/lila/create_lila_test_set.py +22 -2
  14. megadetector/data_management/lila/generate_lila_per_image_labels.py +7 -5
  15. megadetector/data_management/lila/lila_common.py +2 -2
  16. megadetector/data_management/lila/test_lila_metadata_urls.py +0 -1
  17. megadetector/data_management/ocr_tools.py +6 -10
  18. megadetector/data_management/read_exif.py +59 -16
  19. megadetector/data_management/remap_coco_categories.py +1 -1
  20. megadetector/data_management/remove_exif.py +10 -5
  21. megadetector/data_management/rename_images.py +20 -13
  22. megadetector/data_management/resize_coco_dataset.py +10 -4
  23. megadetector/data_management/speciesnet_to_md.py +3 -3
  24. megadetector/data_management/yolo_output_to_md_output.py +3 -1
  25. megadetector/data_management/yolo_to_coco.py +28 -19
  26. megadetector/detection/change_detection.py +26 -18
  27. megadetector/detection/process_video.py +1 -1
  28. megadetector/detection/pytorch_detector.py +5 -5
  29. megadetector/detection/run_detector.py +34 -10
  30. megadetector/detection/run_detector_batch.py +2 -1
  31. megadetector/detection/run_inference_with_yolov5_val.py +3 -1
  32. megadetector/detection/run_md_and_speciesnet.py +215 -101
  33. megadetector/detection/run_tiled_inference.py +7 -7
  34. megadetector/detection/tf_detector.py +1 -1
  35. megadetector/detection/video_utils.py +9 -6
  36. megadetector/postprocessing/add_max_conf.py +4 -4
  37. megadetector/postprocessing/categorize_detections_by_size.py +3 -2
  38. megadetector/postprocessing/classification_postprocessing.py +7 -8
  39. megadetector/postprocessing/combine_batch_outputs.py +3 -2
  40. megadetector/postprocessing/compare_batch_results.py +49 -27
  41. megadetector/postprocessing/convert_output_format.py +8 -6
  42. megadetector/postprocessing/create_crop_folder.py +13 -4
  43. megadetector/postprocessing/generate_csv_report.py +22 -8
  44. megadetector/postprocessing/load_api_results.py +8 -4
  45. megadetector/postprocessing/md_to_coco.py +2 -3
  46. megadetector/postprocessing/md_to_labelme.py +12 -8
  47. megadetector/postprocessing/md_to_wi.py +2 -1
  48. megadetector/postprocessing/merge_detections.py +4 -6
  49. megadetector/postprocessing/postprocess_batch_results.py +4 -3
  50. megadetector/postprocessing/remap_detection_categories.py +6 -3
  51. megadetector/postprocessing/render_detection_confusion_matrix.py +18 -10
  52. megadetector/postprocessing/repeat_detection_elimination/remove_repeat_detections.py +1 -1
  53. megadetector/postprocessing/repeat_detection_elimination/repeat_detections_core.py +5 -3
  54. megadetector/postprocessing/separate_detections_into_folders.py +10 -4
  55. megadetector/postprocessing/subset_json_detector_output.py +1 -1
  56. megadetector/postprocessing/top_folders_to_bottom.py +22 -7
  57. megadetector/postprocessing/validate_batch_results.py +1 -1
  58. megadetector/taxonomy_mapping/map_new_lila_datasets.py +59 -3
  59. megadetector/taxonomy_mapping/prepare_lila_taxonomy_release.py +1 -1
  60. megadetector/taxonomy_mapping/preview_lila_taxonomy.py +26 -17
  61. megadetector/taxonomy_mapping/species_lookup.py +51 -2
  62. megadetector/utils/ct_utils.py +9 -4
  63. megadetector/utils/extract_frames_from_video.py +4 -0
  64. megadetector/utils/gpu_test.py +6 -6
  65. megadetector/utils/md_tests.py +21 -21
  66. megadetector/utils/path_utils.py +112 -44
  67. megadetector/utils/split_locations_into_train_val.py +0 -4
  68. megadetector/utils/url_utils.py +5 -3
  69. megadetector/utils/wi_taxonomy_utils.py +37 -8
  70. megadetector/utils/write_html_image_list.py +1 -2
  71. megadetector/visualization/plot_utils.py +31 -19
  72. megadetector/visualization/render_images_with_thumbnails.py +3 -0
  73. megadetector/visualization/visualization_utils.py +18 -7
  74. megadetector/visualization/visualize_db.py +9 -26
  75. megadetector/visualization/visualize_video_output.py +14 -2
  76. {megadetector-10.0.10.dist-info → megadetector-10.0.11.dist-info}/METADATA +1 -1
  77. {megadetector-10.0.10.dist-info → megadetector-10.0.11.dist-info}/RECORD +80 -80
  78. {megadetector-10.0.10.dist-info → megadetector-10.0.11.dist-info}/WHEEL +0 -0
  79. {megadetector-10.0.10.dist-info → megadetector-10.0.11.dist-info}/licenses/LICENSE +0 -0
  80. {megadetector-10.0.10.dist-info → megadetector-10.0.11.dist-info}/top_level.txt +0 -0
@@ -13,8 +13,7 @@ check out run_detector_batch.py**.
13
13
  That said, this script (run_detector.py) is a good way to test our detector on a handful of images
14
14
  and get super-satisfying, graphical results.
15
15
 
16
- If you would like to *not* use the GPU on the machine, set the environment
17
- variable CUDA_VISIBLE_DEVICES to "-1".
16
+ If you would like to *not* use the GPU, set the environment variable CUDA_VISIBLE_DEVICES to "-1".
18
17
 
19
18
  This script will only consider detections with > 0.005 confidence at all times.
20
19
  The threshold you provide is only for rendering the results. If you need to
@@ -79,6 +78,10 @@ DEFAULT_DETECTOR_LABEL_MAP = {
79
78
  # models other than MegaDetector.
80
79
  USE_MODEL_NATIVE_CLASSES = False
81
80
 
81
+ # Detection threshold to recommend to callers when all other mechanisms for choosing
82
+ # a model-specific threshold fail
83
+ fallback_detection_threshold = 0.2
84
+
82
85
  # Maps a variety of strings that might occur in filenames to canonical version numbers.
83
86
  #
84
87
  # Order matters here.
@@ -215,7 +218,8 @@ known_models = {
215
218
  {
216
219
  'url':model_url_base + 'md_v1000.0.0-redwood.pt',
217
220
  'normalized_typical_inference_speed':1.0,
218
- 'md5':'74474b3aec9cf1a990da38b37ddf9197'
221
+ 'md5':'74474b3aec9cf1a990da38b37ddf9197',
222
+ 'typical_detection_threshold':0.3
219
223
  },
220
224
  'v1000.0.0-spruce':
221
225
  {
@@ -515,19 +519,34 @@ def get_typical_confidence_threshold_from_results(results):
515
519
  with open(results,'r') as f:
516
520
  results = json.load(f)
517
521
 
522
+ default_threshold = None
523
+
524
+ # Best case: the .json file tells us the default threshold
518
525
  if 'detector_metadata' in results['info'] and \
519
526
  'typical_detection_threshold' in results['info']['detector_metadata']:
520
527
  default_threshold = results['info']['detector_metadata']['typical_detection_threshold']
528
+ # Worst case: we don't even know what detector this is
521
529
  elif ('detector' not in results['info']) or (results['info']['detector'] is None):
522
530
  print('Warning: detector version not available in results file, using MDv5 defaults')
523
531
  detector_metadata = get_detector_metadata_from_version_string('v5a.0.0')
524
532
  default_threshold = detector_metadata['typical_detection_threshold']
533
+ # We know what detector this is, but it doesn't have a default threshold
534
+ # in the .json file
525
535
  else:
526
536
  print('Warning: detector metadata not available in results file, inferring from MD version')
527
- detector_filename = results['info']['detector']
528
- detector_version = get_detector_version_from_filename(detector_filename)
529
- detector_metadata = get_detector_metadata_from_version_string(detector_version)
530
- default_threshold = detector_metadata['typical_detection_threshold']
537
+ try:
538
+ detector_filename = results['info']['detector']
539
+ detector_version = get_detector_version_from_filename(detector_filename)
540
+ detector_metadata = get_detector_metadata_from_version_string(detector_version)
541
+ if 'typical_detection_threshold' in detector_metadata:
542
+ default_threshold = detector_metadata['typical_detection_threshold']
543
+ except Exception:
544
+ pass
545
+
546
+ if default_threshold is None:
547
+ print('Could not determine threshold, using fallback threshold of {}'.format(
548
+ fallback_detection_threshold))
549
+ default_threshold = fallback_detection_threshold
531
550
 
532
551
  return default_threshold
533
552
 
@@ -546,7 +565,7 @@ def is_gpu_available(model_file):
546
565
  """
547
566
 
548
567
  if model_file.endswith('.pb'):
549
- import tensorflow.compat.v1 as tf
568
+ import tensorflow.compat.v1 as tf # type: ignore
550
569
  gpu_available = tf.test.is_gpu_available()
551
570
  print('TensorFlow version:', tf.__version__)
552
571
  print('tf.test.is_gpu_available:', gpu_available)
@@ -741,8 +760,9 @@ def load_and_run_detector(model_file,
741
760
  fn = '{}{}{}'.format(name, DETECTION_FILENAME_INSERT, '.jpg')
742
761
  if fn in output_filename_collision_counts:
743
762
  n_collisions = output_filename_collision_counts[fn]
763
+ fn_original = fn
744
764
  fn = '{:0>4d}'.format(n_collisions) + '_' + fn
745
- output_filename_collision_counts[fn] += 1
765
+ output_filename_collision_counts[fn_original] += 1
746
766
  else:
747
767
  output_filename_collision_counts[fn] = 0
748
768
  fn = os.path.join(output_dir, fn)
@@ -847,7 +867,11 @@ def _validate_zip_file(file_path, file_description='file'):
847
867
  """
848
868
  try:
849
869
  with zipfile.ZipFile(file_path, 'r') as zipf:
850
- zipf.testzip()
870
+ corrupt_file = zipf.testzip()
871
+ if corrupt_file is not None:
872
+ print('{} {} contains at least one corrupt file: {}'.format(
873
+ file_description.capitalize(), file_path, corrupt_file))
874
+ return False
851
875
  return True
852
876
  except (zipfile.BadZipFile, zipfile.LargeZipFile) as e:
853
877
  print('{} {} appears to be corrupted (bad zip): {}'.format(
@@ -887,7 +887,7 @@ def _process_images(im_files,
887
887
 
888
888
  if use_image_queue:
889
889
 
890
- _run_detector_with_image_queue(im_files,
890
+ results = _run_detector_with_image_queue(im_files,
891
891
  detector,
892
892
  confidence_threshold,
893
893
  quiet=quiet,
@@ -899,6 +899,7 @@ def _process_images(im_files,
899
899
  detector_options=detector_options,
900
900
  loader_workers=loader_workers,
901
901
  preprocess_on_image_queue=preprocess_on_image_queue)
902
+ return results
902
903
 
903
904
  else:
904
905
 
@@ -351,7 +351,9 @@ def run_inference_with_yolo_val(options):
351
351
  else:
352
352
  raise ValueError('Unknown output handling method {}'.format(options.overwrite_handling))
353
353
 
354
- os.makedirs(os.path.dirname(options.output_file),exist_ok=True)
354
+ output_dir = os.path.dirname(options.output_file)
355
+ if len(output_dir) > 0:
356
+ os.makedirs(output_dir, exist_ok=True)
355
357
 
356
358
  if options.input_folder is not None:
357
359
  options.input_folder = options.input_folder.replace('\\','/')
@@ -32,6 +32,7 @@ from megadetector.utils.ct_utils import write_json
32
32
  from megadetector.utils.ct_utils import make_temp_folder
33
33
  from megadetector.utils.ct_utils import is_list_sorted
34
34
  from megadetector.utils.ct_utils import is_sphinx_build
35
+ from megadetector.utils.ct_utils import args_to_object
35
36
  from megadetector.utils import path_utils
36
37
  from megadetector.visualization import visualization_utils as vis_utils
37
38
  from megadetector.postprocessing.validate_batch_results import \
@@ -62,18 +63,20 @@ DEFAULT_DETECTOR_BATCH_SIZE = 1
62
63
  DEFAULT_CLASSIFIER_BATCH_SIZE = 8
63
64
  DEFAULT_LOADER_WORKERS = 4
64
65
 
65
- # This determines the maximum number of images that can get read from disk
66
- # on each of the producer workers before blocking. The actual size of the queue
66
+ # This determines the maximum number of image filenames that can be assigned to
67
+ # each of the producer workers before blocking. The actual size of the queue
67
68
  # will be MAX_IMAGE_QUEUE_SIZE_PER_WORKER * n_workers. This is only used for
68
69
  # the classification step.
69
- MAX_IMAGE_QUEUE_SIZE_PER_WORKER = 10
70
+ MAX_IMAGE_QUEUE_SIZE_PER_WORKER = 30
70
71
 
71
72
  # This determines the maximum number of crops that can accumulate in the queue
72
73
  # used to communicate between the producers (which read and crop images) and the
73
74
  # consumer (which runs the classifier). This is only used for the classification step.
74
75
  MAX_BATCH_QUEUE_SIZE = 300
75
76
 
76
- DEAFULT_SECONDS_PER_VIDEO_FRAME = 1.0
77
+ # Default interval between frames we should process when processing video.
78
+ # This is only used for the detection step.
79
+ DEFAULT_SECONDS_PER_VIDEO_FRAME = 1.0
77
80
 
78
81
  # Max number of classification scores to include per detection
79
82
  DEFAULT_TOP_N_SCORES = 2
@@ -168,6 +171,12 @@ def _process_image_detections(file_path: str,
168
171
 
169
172
  detections = detection_results['detections']
170
173
 
174
+ # Don't bother loading images that have no above-threshold detections
175
+ detections_above_threshold = \
176
+ [d for d in detections if d['conf'] >= detection_confidence_threshold]
177
+ if len(detections_above_threshold) == 0:
178
+ return
179
+
171
180
  # Load the image
172
181
  try:
173
182
  image = vis_utils.load_image(absolute_file_path)
@@ -189,6 +198,11 @@ def _process_image_detections(file_path: str,
189
198
  return
190
199
 
191
200
  # Process each detection above threshold
201
+ #
202
+ # detection_index needs to index into the original list of detections
203
+ # (this is how classification results will be associated with detections
204
+ # later), so iterate over "detections" here, rather than
205
+ # "detections_above_threshold".
192
206
  for detection_index, detection in enumerate(detections):
193
207
 
194
208
  conf = detection['conf']
@@ -562,7 +576,7 @@ def _crop_consumer_func(batch_queue: Queue,
562
576
  taxonomy_map = {}
563
577
  geofence_map = {}
564
578
 
565
- if (enable_rollup is not None) or (country is not None):
579
+ if (enable_rollup) or (country is not None):
566
580
 
567
581
  # Note to self: there are a number of reasons loading the ensemble
568
582
  # could fail here; don't catch this exception, this should be a
@@ -1146,7 +1160,7 @@ def _run_classification_step(detector_results_file: str,
1146
1160
 
1147
1161
  # Add classifications to the detection
1148
1162
  detection['classifications'] = classification_pairs
1149
- detection['raw_classifications'] = raw_classification_pairs
1163
+ # detection['raw_classifications'] = raw_classification_pairs
1150
1164
 
1151
1165
  # ...if this classification contains a failure
1152
1166
 
@@ -1181,6 +1195,196 @@ def _run_classification_step(detector_results_file: str,
1181
1195
  # ...def _run_classification_step(...)
1182
1196
 
1183
1197
 
1198
+ #%% Options class
1199
+
1200
+ class RunMDSpeciesNetOptions:
1201
+ """
1202
+ Class controlling the behavior of run_md_and_speciesnet()
1203
+ """
1204
+
1205
+ def __init__(self):
1206
+
1207
+ #: Folder containing images and/or videos to process
1208
+ self.source = None
1209
+
1210
+ #: Output file for results (JSON format)
1211
+ self.output_file = None
1212
+
1213
+ #: MegaDetector model identifier (MDv5a, MDv5b, MDv1000-redwood, etc.)
1214
+ self.detector_model = DEFAULT_DETECTOR_MODEL
1215
+
1216
+ #: SpeciesNet classifier model identifier (e.g. kaggle:google/speciesnet/pyTorch/v4.0.1a)
1217
+ self.classification_model = DEFAULT_CLASSIFIER_MODEL
1218
+
1219
+ #: Batch size for MegaDetector inference
1220
+ self.detector_batch_size = DEFAULT_DETECTOR_BATCH_SIZE
1221
+
1222
+ #: Batch size for SpeciesNet classification
1223
+ self.classifier_batch_size = DEFAULT_CLASSIFIER_BATCH_SIZE
1224
+
1225
+ #: Number of worker threads for preprocessing
1226
+ self.loader_workers = DEFAULT_LOADER_WORKERS
1227
+
1228
+ #: Classify detections above this threshold
1229
+ self.detection_confidence_threshold_for_classification = \
1230
+ DEFAULT_DETECTION_CONFIDENCE_THRESHOLD_FOR_CLASSIFICATION
1231
+
1232
+ #: Include detections above this threshold in the output
1233
+ self.detection_confidence_threshold_for_output = \
1234
+ DEFAULT_DETECTION_CONFIDENCE_THRESHOLD_FOR_OUTPUT
1235
+
1236
+ #: Folder for intermediate files (default: system temp)
1237
+ self.intermediate_file_folder = None
1238
+
1239
+ #: Keep intermediate files (e.g. detection-only results file)
1240
+ self.keep_intermediate_files = False
1241
+
1242
+ #: Disable taxonomic rollup
1243
+ self.norollup = False
1244
+
1245
+ #: Country code (ISO 3166-1 alpha-3) for geofencing (default None, no geoferencing)
1246
+ self.country = None
1247
+
1248
+ #: Admin1 region/state code for geofencing
1249
+ self.admin1_region = None
1250
+
1251
+ #: Path to existing MegaDetector output file (skips detection step)
1252
+ self.detections_file = None
1253
+
1254
+ #: Ignore videos, only process images
1255
+ self.skip_video = False
1256
+
1257
+ #: Ignore images, only process videos
1258
+ self.skip_images = False
1259
+
1260
+ #: Sample every Nth frame from videos
1261
+ #:
1262
+ #: Mutually exclusive with time_sample
1263
+ self.frame_sample = None
1264
+
1265
+ #: Sample frames every N seconds from videos
1266
+ #:
1267
+ #: Mutually exclusive with frame_sample
1268
+ self.time_sample = DEFAULT_SECONDS_PER_VIDEO_FRAME
1269
+
1270
+ #: Enable additional debug output
1271
+ self.verbose = False
1272
+
1273
+ if self.time_sample is None and self.frame_sample is None:
1274
+ self.time_sample = DEFAULT_SECONDS_PER_VIDEO_FRAME
1275
+
1276
+ # ...class RunMDSpeciesNetOptions
1277
+
1278
+
1279
+ #%% Main function
1280
+
1281
+ def run_md_and_speciesnet(options):
1282
+ """
1283
+ Main entry point, runs MegaDetector and SpeciesNet on a folder. See
1284
+ RunMDSpeciesNetOptions for available arguments.
1285
+
1286
+ Args:
1287
+ options (RunMDSpeciesNetOptions): options controlling MD and SN inference
1288
+ """
1289
+
1290
+ # Set global verbose flag
1291
+ global verbose
1292
+ verbose = options.verbose
1293
+
1294
+ # Also set the run_detector_batch verbose flag
1295
+ run_detector_batch.verbose = verbose
1296
+
1297
+ # Validate arguments
1298
+ if not os.path.isdir(options.source):
1299
+ raise ValueError(
1300
+ 'Source folder does not exist: {}'.format(options.source))
1301
+
1302
+ if (options.admin1_region is not None) and (options.country is None):
1303
+ raise ValueError('--admin1_region requires --country to be specified')
1304
+
1305
+ if options.skip_images and options.skip_video:
1306
+ raise ValueError('Cannot skip both images and videos')
1307
+
1308
+ if (options.frame_sample is not None) and (options.time_sample is not None):
1309
+ raise ValueError('--frame_sample and --time_sample are mutually exclusive')
1310
+ if (options.frame_sample is None) and (options.time_sample is None):
1311
+ options.time_sample = DEFAULT_SECONDS_PER_VIDEO_FRAME
1312
+
1313
+ # Set up intermediate file folder
1314
+ if options.intermediate_file_folder:
1315
+ temp_folder = options.intermediate_file_folder
1316
+ os.makedirs(temp_folder, exist_ok=True)
1317
+ else:
1318
+ temp_folder = make_temp_folder(subfolder='run_md_and_speciesnet')
1319
+
1320
+ start_time = time.time()
1321
+
1322
+ print('Processing folder: {}'.format(options.source))
1323
+ print('Output file: {}'.format(options.output_file))
1324
+ print('Intermediate files: {}'.format(temp_folder))
1325
+
1326
+ # Determine detector output file path
1327
+ if options.detections_file is not None:
1328
+ detector_output_file = options.detections_file
1329
+ if VALIDATE_DETECTION_FILE:
1330
+ print('Using existing detections file: {}'.format(detector_output_file))
1331
+ validation_options = ValidateBatchResultsOptions()
1332
+ validation_options.check_image_existence = True
1333
+ validation_options.relative_path_base = options.source
1334
+ validation_options.raise_errors = True
1335
+ validate_batch_results(detector_output_file,options=validation_options)
1336
+ print('Validated detections file')
1337
+ else:
1338
+ print('Bypassing validation of {}'.format(options.detections_file))
1339
+ else:
1340
+ detector_output_file = os.path.join(temp_folder, 'detector_output.json')
1341
+
1342
+ # Run MegaDetector
1343
+ _run_detection_step(
1344
+ source_folder=options.source,
1345
+ detector_output_file=detector_output_file,
1346
+ detector_model=options.detector_model,
1347
+ detector_batch_size=options.detector_batch_size,
1348
+ detection_confidence_threshold=options.detection_confidence_threshold_for_output,
1349
+ detector_worker_threads=options.loader_workers,
1350
+ skip_images=options.skip_images,
1351
+ skip_video=options.skip_video,
1352
+ frame_sample=options.frame_sample,
1353
+ time_sample=options.time_sample
1354
+ )
1355
+
1356
+ # Run SpeciesNet
1357
+ _run_classification_step(
1358
+ detector_results_file=detector_output_file,
1359
+ merged_results_file=options.output_file,
1360
+ source_folder=options.source,
1361
+ classifier_model=options.classification_model,
1362
+ classifier_batch_size=options.classifier_batch_size,
1363
+ classifier_worker_threads=options.loader_workers,
1364
+ detection_confidence_threshold=options.detection_confidence_threshold_for_classification,
1365
+ enable_rollup=(not options.norollup),
1366
+ country=options.country,
1367
+ admin1_region=options.admin1_region,
1368
+ )
1369
+
1370
+ elapsed_time = time.time() - start_time
1371
+ print(
1372
+ 'Processing complete in {}'.format(humanfriendly.format_timespan(elapsed_time)))
1373
+ print('Results written to: {}'.format(options.output_file))
1374
+
1375
+ # Clean up intermediate files if requested
1376
+ if (not options.keep_intermediate_files) and \
1377
+ (not options.intermediate_file_folder) and \
1378
+ (not options.detections_file):
1379
+ try:
1380
+ os.remove(detector_output_file)
1381
+ except Exception as e:
1382
+ print('Warning: error removing temporary output file {}: {}'.format(
1383
+ detector_output_file, str(e)))
1384
+
1385
+ # ...def run_md_and_speciesnet(...)
1386
+
1387
+
1184
1388
  #%% Command-line driver
1185
1389
 
1186
1390
  def main():
@@ -1236,7 +1440,7 @@ def main():
1236
1440
  help='Folder for intermediate files (default: system temp)')
1237
1441
  parser.add_argument('--keep_intermediate_files',
1238
1442
  action='store_true',
1239
- help='Keep intermediate files for debugging')
1443
+ help='Keep intermediate files (e.g. detection-only results file)')
1240
1444
  parser.add_argument('--norollup',
1241
1445
  action='store_true',
1242
1446
  help='Disable taxonomic rollup')
@@ -1263,7 +1467,7 @@ def main():
1263
1467
  type=float,
1264
1468
  default=None,
1265
1469
  help='Sample frames every N seconds from videos (default {})'.\
1266
- format(DEAFULT_SECONDS_PER_VIDEO_FRAME) + \
1470
+ format(DEFAULT_SECONDS_PER_VIDEO_FRAME) + \
1267
1471
  ' (mutually exclusive with --frame_sample)')
1268
1472
  parser.add_argument('--verbose',
1269
1473
  action='store_true',
@@ -1275,100 +1479,10 @@ def main():
1275
1479
 
1276
1480
  args = parser.parse_args()
1277
1481
 
1278
- # Set global verbose flag
1279
- global verbose
1280
- verbose = args.verbose
1281
-
1282
- # Also set the run_detector_batch verbose flag
1283
- run_detector_batch.verbose = verbose
1284
-
1285
- # Validate arguments
1286
- if not os.path.isdir(args.source):
1287
- raise ValueError(
1288
- 'Source folder does not exist: {}'.format(args.source))
1289
-
1290
- if args.admin1_region and not args.country:
1291
- raise ValueError('--admin1_region requires --country to be specified')
1292
-
1293
- if args.skip_images and args.skip_video:
1294
- raise ValueError('Cannot skip both images and videos')
1295
-
1296
- if (args.frame_sample is not None) and (args.time_sample is not None):
1297
- raise ValueError('--frame_sample and --time_sample are mutually exclusive')
1298
- if (args.frame_sample is None) and (args.time_sample is None):
1299
- args.time_sample = DEAFULT_SECONDS_PER_VIDEO_FRAME
1300
-
1301
- # Set up intermediate file folder
1302
- if args.intermediate_file_folder:
1303
- temp_folder = args.intermediate_file_folder
1304
- os.makedirs(temp_folder, exist_ok=True)
1305
- else:
1306
- temp_folder = make_temp_folder(subfolder='run_md_and_speciesnet')
1307
-
1308
- start_time = time.time()
1309
-
1310
- print('Processing folder: {}'.format(args.source))
1311
- print('Output file: {}'.format(args.output_file))
1312
- print('Intermediate files: {}'.format(temp_folder))
1313
-
1314
- # Determine detector output file path
1315
- if args.detections_file is not None:
1316
- detector_output_file = args.detections_file
1317
- if VALIDATE_DETECTION_FILE:
1318
- print('Using existing detections file: {}'.format(detector_output_file))
1319
- validation_options = ValidateBatchResultsOptions()
1320
- validation_options.check_image_existence = True
1321
- validation_options.relative_path_base = args.source
1322
- validation_options.raise_errors = True
1323
- validate_batch_results(detector_output_file,options=validation_options)
1324
- print('Validated detections file')
1325
- else:
1326
- print('Bypassing validation of {}'.format(args.detections_file))
1327
- else:
1328
- detector_output_file = os.path.join(temp_folder, 'detector_output.json')
1329
-
1330
- # Run MegaDetector
1331
- _run_detection_step(
1332
- source_folder=args.source,
1333
- detector_output_file=detector_output_file,
1334
- detector_model=args.detector_model,
1335
- detector_batch_size=args.detector_batch_size,
1336
- detection_confidence_threshold=args.detection_confidence_threshold_for_output,
1337
- detector_worker_threads=args.loader_workers,
1338
- skip_images=args.skip_images,
1339
- skip_video=args.skip_video,
1340
- frame_sample=args.frame_sample,
1341
- time_sample=args.time_sample
1342
- )
1343
-
1344
- # Run SpeciesNet
1345
- _run_classification_step(
1346
- detector_results_file=detector_output_file,
1347
- merged_results_file=args.output_file,
1348
- source_folder=args.source,
1349
- classifier_model=args.classification_model,
1350
- classifier_batch_size=args.classifier_batch_size,
1351
- classifier_worker_threads=args.loader_workers,
1352
- detection_confidence_threshold=args.detection_confidence_threshold_for_classification,
1353
- enable_rollup=(not args.norollup),
1354
- country=args.country,
1355
- admin1_region=args.admin1_region,
1356
- )
1482
+ options = RunMDSpeciesNetOptions()
1483
+ args_to_object(args,options)
1357
1484
 
1358
- elapsed_time = time.time() - start_time
1359
- print(
1360
- 'Processing complete in {}'.format(humanfriendly.format_timespan(elapsed_time)))
1361
- print('Results written to: {}'.format(args.output_file))
1362
-
1363
- # Clean up intermediate files if requested
1364
- if (not args.keep_intermediate_files) and \
1365
- (not args.intermediate_file_folder) and \
1366
- (not args.detections_file):
1367
- try:
1368
- os.remove(detector_output_file)
1369
- except Exception as e:
1370
- print('Warning: error removing temporary output file {}: {}'.format(
1371
- detector_output_file, str(e)))
1485
+ run_md_and_speciesnet(options)
1372
1486
 
1373
1487
  # ...def main(...)
1374
1488
 
@@ -510,9 +510,9 @@ def run_tiled_inference(model_file,
510
510
  try:
511
511
  fn_relative = os.path.relpath(fn,image_folder)
512
512
  except ValueError:
513
- 'Illegal absolute path supplied to run_tiled_inference, {} is outside of {}'.format(
514
- fn,image_folder)
515
- raise
513
+ raise ValueError(
514
+ 'Illegal absolute path supplied to run_tiled_inference, {} is outside of {}'.format(
515
+ fn,image_folder))
516
516
  assert not fn_relative.startswith('..'), \
517
517
  'Illegal absolute path supplied to run_tiled_inference, {} is outside of {}'.format(
518
518
  fn,image_folder)
@@ -619,12 +619,12 @@ def run_tiled_inference(model_file,
619
619
  else:
620
620
 
621
621
  patch_file_names = []
622
- for im in all_image_patch_info:
622
+ for patch_info in all_image_patch_info:
623
623
  # If there was a patch generation error, don't run inference
624
624
  if patch_info['error'] is not None:
625
- assert im['image_fn'] in images_with_patch_errors
625
+ assert patch_info['image_fn'] in images_with_patch_errors
626
626
  continue
627
- for patch in im['patches']:
627
+ for patch in patch_info['patches']:
628
628
  patch_file_names.append(patch['patch_fn'])
629
629
 
630
630
  inference_results = load_and_run_detector_batch(model_file,
@@ -952,7 +952,7 @@ def main():
952
952
  parser.add_argument(
953
953
  '--tile_size_y',
954
954
  type=int,
955
- default=default_tile_size[0],
955
+ default=default_tile_size[1],
956
956
  help=('Tile height (defaults to {})'.format(default_tile_size[1])))
957
957
  parser.add_argument(
958
958
  '--tile_overlap',
@@ -42,7 +42,7 @@ class TFDetector:
42
42
  input and output tensor handles.
43
43
 
44
44
  Args:
45
- model_path (str): path to .pdb file
45
+ model_path (str): path to .pb file
46
46
  detector_options (dict, optional): key-value pairs that control detector
47
47
  options; currently not used by TFDetector
48
48
  """
@@ -221,11 +221,13 @@ def frames_to_video(images, fs, output_file_name, codec_spec=default_fourcc):
221
221
  print('Warning: no frames to render')
222
222
  return
223
223
 
224
- os.makedirs(os.path.dirname(output_file_name),exist_ok=True)
224
+ output_dir = os.path.dirname(output_file_name)
225
+ if len(output_dir) > 0:
226
+ os.makedirs(output_dir, exist_ok=True)
225
227
 
226
228
  # Determine the width and height from the first image
227
229
  frame = cv2.imread(images[0])
228
- cv2.imshow('video',frame)
230
+ # cv2.imshow('video',frame)
229
231
  height, width, channels = frame.shape
230
232
 
231
233
  # Define the codec and create VideoWriter object
@@ -297,7 +299,7 @@ def _filename_to_frame_number(filename):
297
299
  try:
298
300
  frame_number = int(frame_number)
299
301
  except Exception:
300
- raise ValueError('Filename {} does contain a valid frame number'.format(filename))
302
+ raise ValueError('Filename {} does not contain a valid frame number'.format(filename))
301
303
 
302
304
  return frame_number
303
305
 
@@ -1059,9 +1061,10 @@ def video_folder_to_frames(input_folder,
1059
1061
 
1060
1062
  finally:
1061
1063
 
1062
- pool.close()
1063
- pool.join()
1064
- print('Pool closed and joined for video processing')
1064
+ if pool is not None:
1065
+ pool.close()
1066
+ pool.join()
1067
+ print('Pool closed and joined for video processing')
1065
1068
 
1066
1069
  # ...try/finally
1067
1070
 
@@ -18,7 +18,8 @@ import json
18
18
  import sys
19
19
  import argparse
20
20
 
21
- from megadetector.utils import ct_utils
21
+ from megadetector.utils.ct_utils import get_max_conf
22
+ from megadetector.utils.ct_utils import write_json
22
23
 
23
24
 
24
25
  #%% Main function
@@ -39,15 +40,14 @@ def add_max_conf(input_file,output_file):
39
40
 
40
41
  for im in d['images']:
41
42
 
42
- max_conf = ct_utils.get_max_conf(im)
43
+ max_conf = get_max_conf(im)
43
44
 
44
45
  if 'max_detection_conf' in im:
45
46
  assert abs(max_conf - im['max_detection_conf']) < 0.00001
46
47
  else:
47
48
  im['max_detection_conf'] = max_conf
48
49
 
49
- with open(output_file,'w') as f:
50
- json.dump(d,f,indent=1)
50
+ write_json(output_file,d)
51
51
 
52
52
 
53
53
  #%% Driver
@@ -14,6 +14,8 @@ import json
14
14
  from collections import defaultdict
15
15
  from tqdm import tqdm
16
16
 
17
+ from megadetector.utils.ct_utils import write_json
18
+
17
19
 
18
20
  #%% Support classes
19
21
 
@@ -157,8 +159,7 @@ def categorize_detections_by_size(input_file,output_file=None,options=None):
157
159
  print('Found {} detections in category {}'.format(category_count,category_name))
158
160
 
159
161
  if output_file is not None:
160
- with open(output_file,'w') as f:
161
- json.dump(data,f,indent=1)
162
+ write_json(output_file,data)
162
163
 
163
164
  return data
164
165