megadetector 5.0.5__py3-none-any.whl → 5.0.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of megadetector might be problematic. Click here for more details.

Files changed (132) hide show
  1. api/batch_processing/data_preparation/manage_local_batch.py +302 -263
  2. api/batch_processing/data_preparation/manage_video_batch.py +81 -2
  3. api/batch_processing/postprocessing/add_max_conf.py +1 -0
  4. api/batch_processing/postprocessing/categorize_detections_by_size.py +50 -19
  5. api/batch_processing/postprocessing/compare_batch_results.py +110 -60
  6. api/batch_processing/postprocessing/load_api_results.py +56 -70
  7. api/batch_processing/postprocessing/md_to_coco.py +1 -1
  8. api/batch_processing/postprocessing/md_to_labelme.py +2 -1
  9. api/batch_processing/postprocessing/postprocess_batch_results.py +240 -81
  10. api/batch_processing/postprocessing/render_detection_confusion_matrix.py +625 -0
  11. api/batch_processing/postprocessing/repeat_detection_elimination/find_repeat_detections.py +71 -23
  12. api/batch_processing/postprocessing/repeat_detection_elimination/remove_repeat_detections.py +1 -1
  13. api/batch_processing/postprocessing/repeat_detection_elimination/repeat_detections_core.py +227 -75
  14. api/batch_processing/postprocessing/subset_json_detector_output.py +132 -5
  15. api/batch_processing/postprocessing/top_folders_to_bottom.py +1 -1
  16. api/synchronous/api_core/animal_detection_api/detection/run_detector_batch.py +2 -2
  17. classification/prepare_classification_script.py +191 -191
  18. data_management/coco_to_yolo.py +68 -45
  19. data_management/databases/integrity_check_json_db.py +7 -5
  20. data_management/generate_crops_from_cct.py +3 -3
  21. data_management/get_image_sizes.py +8 -6
  22. data_management/importers/add_timestamps_to_icct.py +79 -0
  23. data_management/importers/animl_results_to_md_results.py +160 -0
  24. data_management/importers/auckland_doc_test_to_json.py +4 -4
  25. data_management/importers/auckland_doc_to_json.py +1 -1
  26. data_management/importers/awc_to_json.py +5 -5
  27. data_management/importers/bellevue_to_json.py +5 -5
  28. data_management/importers/carrizo_shrubfree_2018.py +5 -5
  29. data_management/importers/carrizo_trail_cam_2017.py +5 -5
  30. data_management/importers/cct_field_adjustments.py +2 -3
  31. data_management/importers/channel_islands_to_cct.py +4 -4
  32. data_management/importers/ena24_to_json.py +5 -5
  33. data_management/importers/helena_to_cct.py +10 -10
  34. data_management/importers/idaho-camera-traps.py +12 -12
  35. data_management/importers/idfg_iwildcam_lila_prep.py +8 -8
  36. data_management/importers/jb_csv_to_json.py +4 -4
  37. data_management/importers/missouri_to_json.py +1 -1
  38. data_management/importers/noaa_seals_2019.py +1 -1
  39. data_management/importers/pc_to_json.py +5 -5
  40. data_management/importers/prepare-noaa-fish-data-for-lila.py +4 -4
  41. data_management/importers/prepare_zsl_imerit.py +5 -5
  42. data_management/importers/rspb_to_json.py +4 -4
  43. data_management/importers/save_the_elephants_survey_A.py +5 -5
  44. data_management/importers/save_the_elephants_survey_B.py +6 -6
  45. data_management/importers/snapshot_safari_importer.py +9 -9
  46. data_management/importers/snapshot_serengeti_lila.py +9 -9
  47. data_management/importers/timelapse_csv_set_to_json.py +5 -7
  48. data_management/importers/ubc_to_json.py +4 -4
  49. data_management/importers/umn_to_json.py +4 -4
  50. data_management/importers/wellington_to_json.py +1 -1
  51. data_management/importers/wi_to_json.py +2 -2
  52. data_management/importers/zamba_results_to_md_results.py +181 -0
  53. data_management/labelme_to_coco.py +35 -7
  54. data_management/labelme_to_yolo.py +229 -0
  55. data_management/lila/add_locations_to_island_camera_traps.py +1 -1
  56. data_management/lila/add_locations_to_nacti.py +147 -0
  57. data_management/lila/create_lila_blank_set.py +474 -0
  58. data_management/lila/create_lila_test_set.py +2 -1
  59. data_management/lila/create_links_to_md_results_files.py +106 -0
  60. data_management/lila/download_lila_subset.py +46 -21
  61. data_management/lila/generate_lila_per_image_labels.py +23 -14
  62. data_management/lila/get_lila_annotation_counts.py +17 -11
  63. data_management/lila/lila_common.py +14 -11
  64. data_management/lila/test_lila_metadata_urls.py +116 -0
  65. data_management/ocr_tools.py +829 -0
  66. data_management/resize_coco_dataset.py +13 -11
  67. data_management/yolo_output_to_md_output.py +84 -12
  68. data_management/yolo_to_coco.py +38 -20
  69. detection/process_video.py +36 -14
  70. detection/pytorch_detector.py +23 -8
  71. detection/run_detector.py +76 -19
  72. detection/run_detector_batch.py +178 -63
  73. detection/run_inference_with_yolov5_val.py +326 -57
  74. detection/run_tiled_inference.py +153 -43
  75. detection/video_utils.py +34 -8
  76. md_utils/ct_utils.py +172 -1
  77. md_utils/md_tests.py +372 -51
  78. md_utils/path_utils.py +167 -39
  79. md_utils/process_utils.py +26 -7
  80. md_utils/split_locations_into_train_val.py +215 -0
  81. md_utils/string_utils.py +10 -0
  82. md_utils/url_utils.py +0 -2
  83. md_utils/write_html_image_list.py +9 -26
  84. md_visualization/plot_utils.py +12 -8
  85. md_visualization/visualization_utils.py +106 -7
  86. md_visualization/visualize_db.py +16 -8
  87. md_visualization/visualize_detector_output.py +208 -97
  88. {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/METADATA +3 -6
  89. {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/RECORD +98 -121
  90. {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/WHEEL +1 -1
  91. taxonomy_mapping/map_lila_taxonomy_to_wi_taxonomy.py +1 -1
  92. taxonomy_mapping/map_new_lila_datasets.py +43 -39
  93. taxonomy_mapping/prepare_lila_taxonomy_release.py +5 -2
  94. taxonomy_mapping/preview_lila_taxonomy.py +27 -27
  95. taxonomy_mapping/species_lookup.py +33 -13
  96. taxonomy_mapping/taxonomy_csv_checker.py +7 -5
  97. api/synchronous/api_core/yolov5/detect.py +0 -252
  98. api/synchronous/api_core/yolov5/export.py +0 -607
  99. api/synchronous/api_core/yolov5/hubconf.py +0 -146
  100. api/synchronous/api_core/yolov5/models/__init__.py +0 -0
  101. api/synchronous/api_core/yolov5/models/common.py +0 -738
  102. api/synchronous/api_core/yolov5/models/experimental.py +0 -104
  103. api/synchronous/api_core/yolov5/models/tf.py +0 -574
  104. api/synchronous/api_core/yolov5/models/yolo.py +0 -338
  105. api/synchronous/api_core/yolov5/train.py +0 -670
  106. api/synchronous/api_core/yolov5/utils/__init__.py +0 -36
  107. api/synchronous/api_core/yolov5/utils/activations.py +0 -103
  108. api/synchronous/api_core/yolov5/utils/augmentations.py +0 -284
  109. api/synchronous/api_core/yolov5/utils/autoanchor.py +0 -170
  110. api/synchronous/api_core/yolov5/utils/autobatch.py +0 -66
  111. api/synchronous/api_core/yolov5/utils/aws/__init__.py +0 -0
  112. api/synchronous/api_core/yolov5/utils/aws/resume.py +0 -40
  113. api/synchronous/api_core/yolov5/utils/benchmarks.py +0 -148
  114. api/synchronous/api_core/yolov5/utils/callbacks.py +0 -71
  115. api/synchronous/api_core/yolov5/utils/dataloaders.py +0 -1087
  116. api/synchronous/api_core/yolov5/utils/downloads.py +0 -178
  117. api/synchronous/api_core/yolov5/utils/flask_rest_api/example_request.py +0 -19
  118. api/synchronous/api_core/yolov5/utils/flask_rest_api/restapi.py +0 -46
  119. api/synchronous/api_core/yolov5/utils/general.py +0 -1018
  120. api/synchronous/api_core/yolov5/utils/loggers/__init__.py +0 -187
  121. api/synchronous/api_core/yolov5/utils/loggers/wandb/__init__.py +0 -0
  122. api/synchronous/api_core/yolov5/utils/loggers/wandb/log_dataset.py +0 -27
  123. api/synchronous/api_core/yolov5/utils/loggers/wandb/sweep.py +0 -41
  124. api/synchronous/api_core/yolov5/utils/loggers/wandb/wandb_utils.py +0 -577
  125. api/synchronous/api_core/yolov5/utils/loss.py +0 -234
  126. api/synchronous/api_core/yolov5/utils/metrics.py +0 -355
  127. api/synchronous/api_core/yolov5/utils/plots.py +0 -489
  128. api/synchronous/api_core/yolov5/utils/torch_utils.py +0 -314
  129. api/synchronous/api_core/yolov5/val.py +0 -394
  130. md_utils/matlab_porting_tools.py +0 -97
  131. {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/LICENSE +0 -0
  132. {megadetector-5.0.5.dist-info → megadetector-5.0.7.dist-info}/top_level.txt +0 -0
@@ -1,187 +0,0 @@
1
- # YOLOv5 🚀 by Ultralytics, GPL-3.0 license
2
- """
3
- Logging utils
4
- """
5
-
6
- import os
7
- import warnings
8
-
9
- import pkg_resources as pkg
10
- import torch
11
- from torch.utils.tensorboard import SummaryWriter
12
-
13
- from utils.general import colorstr, cv2, emojis
14
- from utils.loggers.wandb.wandb_utils import WandbLogger
15
- from utils.plots import plot_images, plot_results
16
- from utils.torch_utils import de_parallel
17
-
18
- LOGGERS = ('csv', 'tb', 'wandb') # text-file, TensorBoard, Weights & Biases
19
- RANK = int(os.getenv('RANK', -1))
20
-
21
- try:
22
- import wandb
23
-
24
- assert hasattr(wandb, '__version__') # verify package import not local dir
25
- if pkg.parse_version(wandb.__version__) >= pkg.parse_version('0.12.2') and RANK in {0, -1}:
26
- try:
27
- wandb_login_success = wandb.login(timeout=30)
28
- except wandb.errors.UsageError: # known non-TTY terminal issue
29
- wandb_login_success = False
30
- if not wandb_login_success:
31
- wandb = None
32
- except (ImportError, AssertionError):
33
- wandb = None
34
-
35
-
36
- class Loggers():
37
- # YOLOv5 Loggers class
38
- def __init__(self, save_dir=None, weights=None, opt=None, hyp=None, logger=None, include=LOGGERS):
39
- self.save_dir = save_dir
40
- self.weights = weights
41
- self.opt = opt
42
- self.hyp = hyp
43
- self.logger = logger # for printing results to console
44
- self.include = include
45
- self.keys = [
46
- 'train/box_loss',
47
- 'train/obj_loss',
48
- 'train/cls_loss', # train loss
49
- 'metrics/precision',
50
- 'metrics/recall',
51
- 'metrics/mAP_0.5',
52
- 'metrics/mAP_0.5:0.95', # metrics
53
- 'val/box_loss',
54
- 'val/obj_loss',
55
- 'val/cls_loss', # val loss
56
- 'x/lr0',
57
- 'x/lr1',
58
- 'x/lr2'] # params
59
- self.best_keys = ['best/epoch', 'best/precision', 'best/recall', 'best/mAP_0.5', 'best/mAP_0.5:0.95']
60
- for k in LOGGERS:
61
- setattr(self, k, None) # init empty logger dictionary
62
- self.csv = True # always log to csv
63
-
64
- # Message
65
- if not wandb:
66
- prefix = colorstr('Weights & Biases: ')
67
- s = f"{prefix}run 'pip install wandb' to automatically track and visualize YOLOv5 🚀 runs (RECOMMENDED)"
68
- self.logger.info(emojis(s))
69
-
70
- # TensorBoard
71
- s = self.save_dir
72
- if 'tb' in self.include and not self.opt.evolve:
73
- prefix = colorstr('TensorBoard: ')
74
- self.logger.info(f"{prefix}Start with 'tensorboard --logdir {s.parent}', view at http://localhost:6006/")
75
- self.tb = SummaryWriter(str(s))
76
-
77
- # W&B
78
- if wandb and 'wandb' in self.include:
79
- wandb_artifact_resume = isinstance(self.opt.resume, str) and self.opt.resume.startswith('wandb-artifact://')
80
- run_id = torch.load(self.weights).get('wandb_id') if self.opt.resume and not wandb_artifact_resume else None
81
- self.opt.hyp = self.hyp # add hyperparameters
82
- self.wandb = WandbLogger(self.opt, run_id)
83
- # temp warn. because nested artifacts not supported after 0.12.10
84
- if pkg.parse_version(wandb.__version__) >= pkg.parse_version('0.12.11'):
85
- self.logger.warning(
86
- "YOLOv5 temporarily requires wandb version 0.12.10 or below. Some features may not work as expected."
87
- )
88
- else:
89
- self.wandb = None
90
-
91
- def on_train_start(self):
92
- # Callback runs on train start
93
- pass
94
-
95
- def on_pretrain_routine_end(self):
96
- # Callback runs on pre-train routine end
97
- paths = self.save_dir.glob('*labels*.jpg') # training labels
98
- if self.wandb:
99
- self.wandb.log({"Labels": [wandb.Image(str(x), caption=x.name) for x in paths]})
100
-
101
- def on_train_batch_end(self, ni, model, imgs, targets, paths, plots):
102
- # Callback runs on train batch end
103
- if plots:
104
- if ni == 0:
105
- if not self.opt.sync_bn: # --sync known issue https://github.com/ultralytics/yolov5/issues/3754
106
- with warnings.catch_warnings():
107
- warnings.simplefilter('ignore') # suppress jit trace warning
108
- self.tb.add_graph(torch.jit.trace(de_parallel(model), imgs[0:1], strict=False), [])
109
- if ni < 3:
110
- f = self.save_dir / f'train_batch{ni}.jpg' # filename
111
- plot_images(imgs, targets, paths, f)
112
- if self.wandb and ni == 10:
113
- files = sorted(self.save_dir.glob('train*.jpg'))
114
- self.wandb.log({'Mosaics': [wandb.Image(str(f), caption=f.name) for f in files if f.exists()]})
115
-
116
- def on_train_epoch_end(self, epoch):
117
- # Callback runs on train epoch end
118
- if self.wandb:
119
- self.wandb.current_epoch = epoch + 1
120
-
121
- def on_val_image_end(self, pred, predn, path, names, im):
122
- # Callback runs on val image end
123
- if self.wandb:
124
- self.wandb.val_one_image(pred, predn, path, names, im)
125
-
126
- def on_val_end(self):
127
- # Callback runs on val end
128
- if self.wandb:
129
- files = sorted(self.save_dir.glob('val*.jpg'))
130
- self.wandb.log({"Validation": [wandb.Image(str(f), caption=f.name) for f in files]})
131
-
132
- def on_fit_epoch_end(self, vals, epoch, best_fitness, fi):
133
- # Callback runs at the end of each fit (train+val) epoch
134
- x = dict(zip(self.keys, vals))
135
- if self.csv:
136
- file = self.save_dir / 'results.csv'
137
- n = len(x) + 1 # number of cols
138
- s = '' if file.exists() else (('%20s,' * n % tuple(['epoch'] + self.keys)).rstrip(',') + '\n') # add header
139
- with open(file, 'a') as f:
140
- f.write(s + ('%20.5g,' * n % tuple([epoch] + vals)).rstrip(',') + '\n')
141
-
142
- if self.tb:
143
- for k, v in x.items():
144
- self.tb.add_scalar(k, v, epoch)
145
-
146
- if self.wandb:
147
- if best_fitness == fi:
148
- best_results = [epoch] + vals[3:7]
149
- for i, name in enumerate(self.best_keys):
150
- self.wandb.wandb_run.summary[name] = best_results[i] # log best results in the summary
151
- self.wandb.log(x)
152
- self.wandb.end_epoch(best_result=best_fitness == fi)
153
-
154
- def on_model_save(self, last, epoch, final_epoch, best_fitness, fi):
155
- # Callback runs on model save event
156
- if self.wandb:
157
- if ((epoch + 1) % self.opt.save_period == 0 and not final_epoch) and self.opt.save_period != -1:
158
- self.wandb.log_model(last.parent, self.opt, epoch, fi, best_model=best_fitness == fi)
159
-
160
- def on_train_end(self, last, best, plots, epoch, results):
161
- # Callback runs on training end
162
- if plots:
163
- plot_results(file=self.save_dir / 'results.csv') # save results.png
164
- files = ['results.png', 'confusion_matrix.png', *(f'{x}_curve.png' for x in ('F1', 'PR', 'P', 'R'))]
165
- files = [(self.save_dir / f) for f in files if (self.save_dir / f).exists()] # filter
166
- self.logger.info(f"Results saved to {colorstr('bold', self.save_dir)}")
167
-
168
- if self.tb:
169
- for f in files:
170
- self.tb.add_image(f.stem, cv2.imread(str(f))[..., ::-1], epoch, dataformats='HWC')
171
-
172
- if self.wandb:
173
- self.wandb.log(dict(zip(self.keys[3:10], results)))
174
- self.wandb.log({"Results": [wandb.Image(str(f), caption=f.name) for f in files]})
175
- # Calling wandb.log. TODO: Refactor this into WandbLogger.log_model
176
- if not self.opt.evolve:
177
- wandb.log_artifact(str(best if best.exists() else last),
178
- type='model',
179
- name=f'run_{self.wandb.wandb_run.id}_model',
180
- aliases=['latest', 'best', 'stripped'])
181
- self.wandb.finish_run()
182
-
183
- def on_params_update(self, params):
184
- # Update hyperparams or configs of the experiment
185
- # params: A dict containing {param: value} pairs
186
- if self.wandb:
187
- self.wandb.wandb_run.config.update(params, allow_val_change=True)
@@ -1,27 +0,0 @@
1
- import argparse
2
-
3
- from wandb_utils import WandbLogger
4
-
5
- from utils.general import LOGGER
6
-
7
- WANDB_ARTIFACT_PREFIX = 'wandb-artifact://'
8
-
9
-
10
- def create_dataset_artifact(opt):
11
- logger = WandbLogger(opt, None, job_type='Dataset Creation') # TODO: return value unused
12
- if not logger.wandb:
13
- LOGGER.info("install wandb using `pip install wandb` to log the dataset")
14
-
15
-
16
- if __name__ == '__main__':
17
- parser = argparse.ArgumentParser()
18
- parser.add_argument('--data', type=str, default='data/coco128.yaml', help='data.yaml path')
19
- parser.add_argument('--single-cls', action='store_true', help='train as single-class dataset')
20
- parser.add_argument('--project', type=str, default='YOLOv5', help='name of W&B Project')
21
- parser.add_argument('--entity', default=None, help='W&B entity')
22
- parser.add_argument('--name', type=str, default='log dataset', help='name of W&B run')
23
-
24
- opt = parser.parse_args()
25
- opt.resume = False # Explicitly disallow resume check for dataset upload job
26
-
27
- create_dataset_artifact(opt)
@@ -1,41 +0,0 @@
1
- import sys
2
- from pathlib import Path
3
-
4
- import wandb
5
-
6
- FILE = Path(__file__).resolve()
7
- ROOT = FILE.parents[3] # YOLOv5 root directory
8
- if str(ROOT) not in sys.path:
9
- sys.path.append(str(ROOT)) # add ROOT to PATH
10
-
11
- from train import parse_opt, train
12
- from utils.callbacks import Callbacks
13
- from utils.general import increment_path
14
- from utils.torch_utils import select_device
15
-
16
-
17
- def sweep():
18
- wandb.init()
19
- # Get hyp dict from sweep agent. Copy because train() modifies parameters which confused wandb.
20
- hyp_dict = vars(wandb.config).get("_items").copy()
21
-
22
- # Workaround: get necessary opt args
23
- opt = parse_opt(known=True)
24
- opt.batch_size = hyp_dict.get("batch_size")
25
- opt.save_dir = str(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok or opt.evolve))
26
- opt.epochs = hyp_dict.get("epochs")
27
- opt.nosave = True
28
- opt.data = hyp_dict.get("data")
29
- opt.weights = str(opt.weights)
30
- opt.cfg = str(opt.cfg)
31
- opt.data = str(opt.data)
32
- opt.hyp = str(opt.hyp)
33
- opt.project = str(opt.project)
34
- device = select_device(opt.device, batch_size=opt.batch_size)
35
-
36
- # train
37
- train(hyp_dict, opt, device, callbacks=Callbacks())
38
-
39
-
40
- if __name__ == "__main__":
41
- sweep()