megadetector 5.0.5__py3-none-any.whl → 5.0.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of megadetector might be problematic. Click here for more details.

Files changed (132) hide show
  1. api/batch_processing/data_preparation/manage_local_batch.py +302 -263
  2. api/batch_processing/data_preparation/manage_video_batch.py +81 -2
  3. api/batch_processing/postprocessing/add_max_conf.py +1 -0
  4. api/batch_processing/postprocessing/categorize_detections_by_size.py +50 -19
  5. api/batch_processing/postprocessing/compare_batch_results.py +110 -60
  6. api/batch_processing/postprocessing/load_api_results.py +56 -70
  7. api/batch_processing/postprocessing/md_to_coco.py +1 -1
  8. api/batch_processing/postprocessing/md_to_labelme.py +2 -1
  9. api/batch_processing/postprocessing/postprocess_batch_results.py +240 -81
  10. api/batch_processing/postprocessing/render_detection_confusion_matrix.py +625 -0
  11. api/batch_processing/postprocessing/repeat_detection_elimination/find_repeat_detections.py +71 -23
  12. api/batch_processing/postprocessing/repeat_detection_elimination/remove_repeat_detections.py +1 -1
  13. api/batch_processing/postprocessing/repeat_detection_elimination/repeat_detections_core.py +227 -75
  14. api/batch_processing/postprocessing/subset_json_detector_output.py +132 -5
  15. api/batch_processing/postprocessing/top_folders_to_bottom.py +1 -1
  16. api/synchronous/api_core/animal_detection_api/detection/run_detector_batch.py +2 -2
  17. classification/prepare_classification_script.py +191 -191
  18. data_management/coco_to_yolo.py +68 -45
  19. data_management/databases/integrity_check_json_db.py +7 -5
  20. data_management/generate_crops_from_cct.py +3 -3
  21. data_management/get_image_sizes.py +8 -6
  22. data_management/importers/add_timestamps_to_icct.py +79 -0
  23. data_management/importers/animl_results_to_md_results.py +160 -0
  24. data_management/importers/auckland_doc_test_to_json.py +4 -4
  25. data_management/importers/auckland_doc_to_json.py +1 -1
  26. data_management/importers/awc_to_json.py +5 -5
  27. data_management/importers/bellevue_to_json.py +5 -5
  28. data_management/importers/carrizo_shrubfree_2018.py +5 -5
  29. data_management/importers/carrizo_trail_cam_2017.py +5 -5
  30. data_management/importers/cct_field_adjustments.py +2 -3
  31. data_management/importers/channel_islands_to_cct.py +4 -4
  32. data_management/importers/ena24_to_json.py +5 -5
  33. data_management/importers/helena_to_cct.py +10 -10
  34. data_management/importers/idaho-camera-traps.py +12 -12
  35. data_management/importers/idfg_iwildcam_lila_prep.py +8 -8
  36. data_management/importers/jb_csv_to_json.py +4 -4
  37. data_management/importers/missouri_to_json.py +1 -1
  38. data_management/importers/noaa_seals_2019.py +1 -1
  39. data_management/importers/pc_to_json.py +5 -5
  40. data_management/importers/prepare-noaa-fish-data-for-lila.py +4 -4
  41. data_management/importers/prepare_zsl_imerit.py +5 -5
  42. data_management/importers/rspb_to_json.py +4 -4
  43. data_management/importers/save_the_elephants_survey_A.py +5 -5
  44. data_management/importers/save_the_elephants_survey_B.py +6 -6
  45. data_management/importers/snapshot_safari_importer.py +9 -9
  46. data_management/importers/snapshot_serengeti_lila.py +9 -9
  47. data_management/importers/timelapse_csv_set_to_json.py +5 -7
  48. data_management/importers/ubc_to_json.py +4 -4
  49. data_management/importers/umn_to_json.py +4 -4
  50. data_management/importers/wellington_to_json.py +1 -1
  51. data_management/importers/wi_to_json.py +2 -2
  52. data_management/importers/zamba_results_to_md_results.py +181 -0
  53. data_management/labelme_to_coco.py +35 -7
  54. data_management/labelme_to_yolo.py +229 -0
  55. data_management/lila/add_locations_to_island_camera_traps.py +1 -1
  56. data_management/lila/add_locations_to_nacti.py +147 -0
  57. data_management/lila/create_lila_blank_set.py +474 -0
  58. data_management/lila/create_lila_test_set.py +2 -1
  59. data_management/lila/create_links_to_md_results_files.py +106 -0
  60. data_management/lila/download_lila_subset.py +46 -21
  61. data_management/lila/generate_lila_per_image_labels.py +23 -14
  62. data_management/lila/get_lila_annotation_counts.py +17 -11
  63. data_management/lila/lila_common.py +14 -11
  64. data_management/lila/test_lila_metadata_urls.py +116 -0
  65. data_management/ocr_tools.py +829 -0
  66. data_management/resize_coco_dataset.py +13 -11
  67. data_management/yolo_output_to_md_output.py +84 -12
  68. data_management/yolo_to_coco.py +38 -20
  69. detection/process_video.py +36 -14
  70. detection/pytorch_detector.py +23 -8
  71. detection/run_detector.py +76 -19
  72. detection/run_detector_batch.py +178 -63
  73. detection/run_inference_with_yolov5_val.py +326 -57
  74. detection/run_tiled_inference.py +153 -43
  75. detection/video_utils.py +34 -8
  76. md_utils/ct_utils.py +172 -1
  77. md_utils/md_tests.py +372 -51
  78. md_utils/path_utils.py +167 -39
  79. md_utils/process_utils.py +26 -7
  80. md_utils/split_locations_into_train_val.py +215 -0
  81. md_utils/string_utils.py +10 -0
  82. md_utils/url_utils.py +0 -2
  83. md_utils/write_html_image_list.py +9 -26
  84. md_visualization/plot_utils.py +12 -8
  85. md_visualization/visualization_utils.py +106 -7
  86. md_visualization/visualize_db.py +16 -8
  87. md_visualization/visualize_detector_output.py +208 -97
  88. {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/METADATA +3 -6
  89. {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/RECORD +98 -121
  90. {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/WHEEL +1 -1
  91. taxonomy_mapping/map_lila_taxonomy_to_wi_taxonomy.py +1 -1
  92. taxonomy_mapping/map_new_lila_datasets.py +43 -39
  93. taxonomy_mapping/prepare_lila_taxonomy_release.py +5 -2
  94. taxonomy_mapping/preview_lila_taxonomy.py +27 -27
  95. taxonomy_mapping/species_lookup.py +33 -13
  96. taxonomy_mapping/taxonomy_csv_checker.py +7 -5
  97. api/synchronous/api_core/yolov5/detect.py +0 -252
  98. api/synchronous/api_core/yolov5/export.py +0 -607
  99. api/synchronous/api_core/yolov5/hubconf.py +0 -146
  100. api/synchronous/api_core/yolov5/models/__init__.py +0 -0
  101. api/synchronous/api_core/yolov5/models/common.py +0 -738
  102. api/synchronous/api_core/yolov5/models/experimental.py +0 -104
  103. api/synchronous/api_core/yolov5/models/tf.py +0 -574
  104. api/synchronous/api_core/yolov5/models/yolo.py +0 -338
  105. api/synchronous/api_core/yolov5/train.py +0 -670
  106. api/synchronous/api_core/yolov5/utils/__init__.py +0 -36
  107. api/synchronous/api_core/yolov5/utils/activations.py +0 -103
  108. api/synchronous/api_core/yolov5/utils/augmentations.py +0 -284
  109. api/synchronous/api_core/yolov5/utils/autoanchor.py +0 -170
  110. api/synchronous/api_core/yolov5/utils/autobatch.py +0 -66
  111. api/synchronous/api_core/yolov5/utils/aws/__init__.py +0 -0
  112. api/synchronous/api_core/yolov5/utils/aws/resume.py +0 -40
  113. api/synchronous/api_core/yolov5/utils/benchmarks.py +0 -148
  114. api/synchronous/api_core/yolov5/utils/callbacks.py +0 -71
  115. api/synchronous/api_core/yolov5/utils/dataloaders.py +0 -1087
  116. api/synchronous/api_core/yolov5/utils/downloads.py +0 -178
  117. api/synchronous/api_core/yolov5/utils/flask_rest_api/example_request.py +0 -19
  118. api/synchronous/api_core/yolov5/utils/flask_rest_api/restapi.py +0 -46
  119. api/synchronous/api_core/yolov5/utils/general.py +0 -1018
  120. api/synchronous/api_core/yolov5/utils/loggers/__init__.py +0 -187
  121. api/synchronous/api_core/yolov5/utils/loggers/wandb/__init__.py +0 -0
  122. api/synchronous/api_core/yolov5/utils/loggers/wandb/log_dataset.py +0 -27
  123. api/synchronous/api_core/yolov5/utils/loggers/wandb/sweep.py +0 -41
  124. api/synchronous/api_core/yolov5/utils/loggers/wandb/wandb_utils.py +0 -577
  125. api/synchronous/api_core/yolov5/utils/loss.py +0 -234
  126. api/synchronous/api_core/yolov5/utils/metrics.py +0 -355
  127. api/synchronous/api_core/yolov5/utils/plots.py +0 -489
  128. api/synchronous/api_core/yolov5/utils/torch_utils.py +0 -314
  129. api/synchronous/api_core/yolov5/val.py +0 -394
  130. md_utils/matlab_porting_tools.py +0 -97
  131. {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/LICENSE +0 -0
  132. {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/top_level.txt +0 -0
@@ -1,40 +0,0 @@
1
- # Resume all interrupted trainings in yolov5/ dir including DDP trainings
2
- # Usage: $ python utils/aws/resume.py
3
-
4
- import os
5
- import sys
6
- from pathlib import Path
7
-
8
- import torch
9
- import yaml
10
-
11
- FILE = Path(__file__).resolve()
12
- ROOT = FILE.parents[2] # YOLOv5 root directory
13
- if str(ROOT) not in sys.path:
14
- sys.path.append(str(ROOT)) # add ROOT to PATH
15
-
16
- port = 0 # --master_port
17
- path = Path('').resolve()
18
- for last in path.rglob('*/**/last.pt'):
19
- ckpt = torch.load(last)
20
- if ckpt['optimizer'] is None:
21
- continue
22
-
23
- # Load opt.yaml
24
- with open(last.parent.parent / 'opt.yaml', errors='ignore') as f:
25
- opt = yaml.safe_load(f)
26
-
27
- # Get device count
28
- d = opt['device'].split(',') # devices
29
- nd = len(d) # number of devices
30
- ddp = nd > 1 or (nd == 0 and torch.cuda.device_count() > 1) # distributed data parallel
31
-
32
- if ddp: # multi-GPU
33
- port += 1
34
- cmd = f'python -m torch.distributed.run --nproc_per_node {nd} --master_port {port} train.py --resume {last}'
35
- else: # single-GPU
36
- cmd = f'python train.py --resume {last}'
37
-
38
- cmd += ' > /dev/null 2>&1 &' # redirect output to dev/null and run in daemon thread
39
- print(cmd)
40
- os.system(cmd)
@@ -1,148 +0,0 @@
1
- # YOLOv5 🚀 by Ultralytics, GPL-3.0 license
2
- """
3
- Run YOLOv5 benchmarks on all supported export formats
4
-
5
- Format | `export.py --include` | Model
6
- --- | --- | ---
7
- PyTorch | - | yolov5s.pt
8
- TorchScript | `torchscript` | yolov5s.torchscript
9
- ONNX | `onnx` | yolov5s.onnx
10
- OpenVINO | `openvino` | yolov5s_openvino_model/
11
- TensorRT | `engine` | yolov5s.engine
12
- CoreML | `coreml` | yolov5s.mlmodel
13
- TensorFlow SavedModel | `saved_model` | yolov5s_saved_model/
14
- TensorFlow GraphDef | `pb` | yolov5s.pb
15
- TensorFlow Lite | `tflite` | yolov5s.tflite
16
- TensorFlow Edge TPU | `edgetpu` | yolov5s_edgetpu.tflite
17
- TensorFlow.js | `tfjs` | yolov5s_web_model/
18
-
19
- Requirements:
20
- $ pip install -r requirements.txt coremltools onnx onnx-simplifier onnxruntime openvino-dev tensorflow-cpu # CPU
21
- $ pip install -r requirements.txt coremltools onnx onnx-simplifier onnxruntime-gpu openvino-dev tensorflow # GPU
22
- $ pip install -U nvidia-tensorrt --index-url https://pypi.ngc.nvidia.com # TensorRT
23
-
24
- Usage:
25
- $ python utils/benchmarks.py --weights yolov5s.pt --img 640
26
- """
27
-
28
- import argparse
29
- import sys
30
- import time
31
- from pathlib import Path
32
-
33
- import pandas as pd
34
-
35
- FILE = Path(__file__).resolve()
36
- ROOT = FILE.parents[1] # YOLOv5 root directory
37
- if str(ROOT) not in sys.path:
38
- sys.path.append(str(ROOT)) # add ROOT to PATH
39
- # ROOT = ROOT.relative_to(Path.cwd()) # relative
40
-
41
- import export
42
- import val
43
- from utils import notebook_init
44
- from utils.general import LOGGER, check_yaml, print_args
45
- from utils.torch_utils import select_device
46
-
47
-
48
- def run(
49
- weights=ROOT / 'yolov5s.pt', # weights path
50
- imgsz=640, # inference size (pixels)
51
- batch_size=1, # batch size
52
- data=ROOT / 'data/coco128.yaml', # dataset.yaml path
53
- device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu
54
- half=False, # use FP16 half-precision inference
55
- test=False, # test exports only
56
- pt_only=False, # test PyTorch only
57
- ):
58
- y, t = [], time.time()
59
- device = select_device(device)
60
- for i, (name, f, suffix, gpu) in export.export_formats().iterrows(): # index, (name, file, suffix, gpu-capable)
61
- try:
62
- assert i != 9, 'Edge TPU not supported'
63
- assert i != 10, 'TF.js not supported'
64
- if device.type != 'cpu':
65
- assert gpu, f'{name} inference not supported on GPU'
66
-
67
- # Export
68
- if f == '-':
69
- w = weights # PyTorch format
70
- else:
71
- w = export.run(weights=weights, imgsz=[imgsz], include=[f], device=device, half=half)[-1] # all others
72
- assert suffix in str(w), 'export failed'
73
-
74
- # Validate
75
- result = val.run(data, w, batch_size, imgsz, plots=False, device=device, task='benchmark', half=half)
76
- metrics = result[0] # metrics (mp, mr, map50, map, *losses(box, obj, cls))
77
- speeds = result[2] # times (preprocess, inference, postprocess)
78
- y.append([name, round(metrics[3], 4), round(speeds[1], 2)]) # mAP, t_inference
79
- except Exception as e:
80
- LOGGER.warning(f'WARNING: Benchmark failure for {name}: {e}')
81
- y.append([name, None, None]) # mAP, t_inference
82
- if pt_only and i == 0:
83
- break # break after PyTorch
84
-
85
- # Print results
86
- LOGGER.info('\n')
87
- parse_opt()
88
- notebook_init() # print system info
89
- py = pd.DataFrame(y, columns=['Format', 'mAP@0.5:0.95', 'Inference time (ms)'] if map else ['Format', 'Export', ''])
90
- LOGGER.info(f'\nBenchmarks complete ({time.time() - t:.2f}s)')
91
- LOGGER.info(str(py if map else py.iloc[:, :2]))
92
- return py
93
-
94
-
95
- def test(
96
- weights=ROOT / 'yolov5s.pt', # weights path
97
- imgsz=640, # inference size (pixels)
98
- batch_size=1, # batch size
99
- data=ROOT / 'data/coco128.yaml', # dataset.yaml path
100
- device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu
101
- half=False, # use FP16 half-precision inference
102
- test=False, # test exports only
103
- pt_only=False, # test PyTorch only
104
- ):
105
- y, t = [], time.time()
106
- device = select_device(device)
107
- for i, (name, f, suffix, gpu) in export.export_formats().iterrows(): # index, (name, file, suffix, gpu-capable)
108
- try:
109
- w = weights if f == '-' else \
110
- export.run(weights=weights, imgsz=[imgsz], include=[f], device=device, half=half)[-1] # weights
111
- assert suffix in str(w), 'export failed'
112
- y.append([name, True])
113
- except Exception:
114
- y.append([name, False]) # mAP, t_inference
115
-
116
- # Print results
117
- LOGGER.info('\n')
118
- parse_opt()
119
- notebook_init() # print system info
120
- py = pd.DataFrame(y, columns=['Format', 'Export'])
121
- LOGGER.info(f'\nExports complete ({time.time() - t:.2f}s)')
122
- LOGGER.info(str(py))
123
- return py
124
-
125
-
126
- def parse_opt():
127
- parser = argparse.ArgumentParser()
128
- parser.add_argument('--weights', type=str, default=ROOT / 'yolov5s.pt', help='weights path')
129
- parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=640, help='inference size (pixels)')
130
- parser.add_argument('--batch-size', type=int, default=1, help='batch size')
131
- parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='dataset.yaml path')
132
- parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')
133
- parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference')
134
- parser.add_argument('--test', action='store_true', help='test exports only')
135
- parser.add_argument('--pt-only', action='store_true', help='test PyTorch only')
136
- opt = parser.parse_args()
137
- opt.data = check_yaml(opt.data) # check YAML
138
- print_args(vars(opt))
139
- return opt
140
-
141
-
142
- def main(opt):
143
- test(**vars(opt)) if opt.test else run(**vars(opt))
144
-
145
-
146
- if __name__ == "__main__":
147
- opt = parse_opt()
148
- main(opt)
@@ -1,71 +0,0 @@
1
- # YOLOv5 🚀 by Ultralytics, GPL-3.0 license
2
- """
3
- Callback utils
4
- """
5
-
6
-
7
- class Callbacks:
8
- """"
9
- Handles all registered callbacks for YOLOv5 Hooks
10
- """
11
-
12
- def __init__(self):
13
- # Define the available callbacks
14
- self._callbacks = {
15
- 'on_pretrain_routine_start': [],
16
- 'on_pretrain_routine_end': [],
17
- 'on_train_start': [],
18
- 'on_train_epoch_start': [],
19
- 'on_train_batch_start': [],
20
- 'optimizer_step': [],
21
- 'on_before_zero_grad': [],
22
- 'on_train_batch_end': [],
23
- 'on_train_epoch_end': [],
24
- 'on_val_start': [],
25
- 'on_val_batch_start': [],
26
- 'on_val_image_end': [],
27
- 'on_val_batch_end': [],
28
- 'on_val_end': [],
29
- 'on_fit_epoch_end': [], # fit = train + val
30
- 'on_model_save': [],
31
- 'on_train_end': [],
32
- 'on_params_update': [],
33
- 'teardown': [],}
34
- self.stop_training = False # set True to interrupt training
35
-
36
- def register_action(self, hook, name='', callback=None):
37
- """
38
- Register a new action to a callback hook
39
-
40
- Args:
41
- hook: The callback hook name to register the action to
42
- name: The name of the action for later reference
43
- callback: The callback to fire
44
- """
45
- assert hook in self._callbacks, f"hook '{hook}' not found in callbacks {self._callbacks}"
46
- assert callable(callback), f"callback '{callback}' is not callable"
47
- self._callbacks[hook].append({'name': name, 'callback': callback})
48
-
49
- def get_registered_actions(self, hook=None):
50
- """"
51
- Returns all the registered actions by callback hook
52
-
53
- Args:
54
- hook: The name of the hook to check, defaults to all
55
- """
56
- return self._callbacks[hook] if hook else self._callbacks
57
-
58
- def run(self, hook, *args, **kwargs):
59
- """
60
- Loop through the registered actions and fire all callbacks
61
-
62
- Args:
63
- hook: The name of the hook to check, defaults to all
64
- args: Arguments to receive from YOLOv5
65
- kwargs: Keyword Arguments to receive from YOLOv5
66
- """
67
-
68
- assert hook in self._callbacks, f"hook '{hook}' not found in callbacks {self._callbacks}"
69
-
70
- for logger in self._callbacks[hook]:
71
- logger['callback'](*args, **kwargs)