megadetector 5.0.8__py3-none-any.whl → 5.0.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of megadetector might be problematic. Click here for more details.

Files changed (190) hide show
  1. api/__init__.py +0 -0
  2. api/batch_processing/__init__.py +0 -0
  3. api/batch_processing/api_core/__init__.py +0 -0
  4. api/batch_processing/api_core/batch_service/__init__.py +0 -0
  5. api/batch_processing/api_core/batch_service/score.py +0 -1
  6. api/batch_processing/api_core/server_job_status_table.py +0 -1
  7. api/batch_processing/api_core_support/__init__.py +0 -0
  8. api/batch_processing/api_core_support/aggregate_results_manually.py +0 -1
  9. api/batch_processing/api_support/__init__.py +0 -0
  10. api/batch_processing/api_support/summarize_daily_activity.py +0 -1
  11. api/batch_processing/data_preparation/__init__.py +0 -0
  12. api/batch_processing/data_preparation/manage_local_batch.py +65 -65
  13. api/batch_processing/data_preparation/manage_video_batch.py +8 -8
  14. api/batch_processing/integration/digiKam/xmp_integration.py +0 -1
  15. api/batch_processing/integration/eMammal/test_scripts/push_annotations_to_emammal.py +0 -1
  16. api/batch_processing/postprocessing/__init__.py +0 -0
  17. api/batch_processing/postprocessing/add_max_conf.py +12 -12
  18. api/batch_processing/postprocessing/categorize_detections_by_size.py +32 -14
  19. api/batch_processing/postprocessing/combine_api_outputs.py +68 -54
  20. api/batch_processing/postprocessing/compare_batch_results.py +113 -43
  21. api/batch_processing/postprocessing/convert_output_format.py +41 -16
  22. api/batch_processing/postprocessing/load_api_results.py +16 -17
  23. api/batch_processing/postprocessing/md_to_coco.py +31 -21
  24. api/batch_processing/postprocessing/md_to_labelme.py +52 -22
  25. api/batch_processing/postprocessing/merge_detections.py +14 -14
  26. api/batch_processing/postprocessing/postprocess_batch_results.py +246 -174
  27. api/batch_processing/postprocessing/remap_detection_categories.py +32 -25
  28. api/batch_processing/postprocessing/render_detection_confusion_matrix.py +60 -27
  29. api/batch_processing/postprocessing/repeat_detection_elimination/find_repeat_detections.py +53 -44
  30. api/batch_processing/postprocessing/repeat_detection_elimination/remove_repeat_detections.py +25 -14
  31. api/batch_processing/postprocessing/repeat_detection_elimination/repeat_detections_core.py +242 -158
  32. api/batch_processing/postprocessing/separate_detections_into_folders.py +159 -114
  33. api/batch_processing/postprocessing/subset_json_detector_output.py +146 -169
  34. api/batch_processing/postprocessing/top_folders_to_bottom.py +77 -43
  35. api/synchronous/__init__.py +0 -0
  36. api/synchronous/api_core/animal_detection_api/__init__.py +0 -0
  37. api/synchronous/api_core/animal_detection_api/api_backend.py +0 -2
  38. api/synchronous/api_core/animal_detection_api/api_frontend.py +266 -268
  39. api/synchronous/api_core/animal_detection_api/config.py +35 -35
  40. api/synchronous/api_core/tests/__init__.py +0 -0
  41. api/synchronous/api_core/tests/load_test.py +109 -109
  42. classification/__init__.py +0 -0
  43. classification/aggregate_classifier_probs.py +21 -24
  44. classification/analyze_failed_images.py +11 -13
  45. classification/cache_batchapi_outputs.py +51 -51
  46. classification/create_classification_dataset.py +69 -68
  47. classification/crop_detections.py +54 -53
  48. classification/csv_to_json.py +97 -100
  49. classification/detect_and_crop.py +105 -105
  50. classification/evaluate_model.py +43 -42
  51. classification/identify_mislabeled_candidates.py +47 -46
  52. classification/json_to_azcopy_list.py +10 -10
  53. classification/json_validator.py +72 -71
  54. classification/map_classification_categories.py +44 -43
  55. classification/merge_classification_detection_output.py +68 -68
  56. classification/prepare_classification_script.py +157 -154
  57. classification/prepare_classification_script_mc.py +228 -228
  58. classification/run_classifier.py +27 -26
  59. classification/save_mislabeled.py +30 -30
  60. classification/train_classifier.py +20 -20
  61. classification/train_classifier_tf.py +21 -22
  62. classification/train_utils.py +10 -10
  63. data_management/__init__.py +0 -0
  64. data_management/annotations/__init__.py +0 -0
  65. data_management/annotations/annotation_constants.py +18 -31
  66. data_management/camtrap_dp_to_coco.py +238 -0
  67. data_management/cct_json_utils.py +102 -59
  68. data_management/cct_to_md.py +176 -158
  69. data_management/cct_to_wi.py +247 -219
  70. data_management/coco_to_labelme.py +272 -263
  71. data_management/coco_to_yolo.py +79 -58
  72. data_management/databases/__init__.py +0 -0
  73. data_management/databases/add_width_and_height_to_db.py +20 -16
  74. data_management/databases/combine_coco_camera_traps_files.py +35 -31
  75. data_management/databases/integrity_check_json_db.py +62 -24
  76. data_management/databases/subset_json_db.py +24 -15
  77. data_management/generate_crops_from_cct.py +27 -45
  78. data_management/get_image_sizes.py +188 -162
  79. data_management/importers/add_nacti_sizes.py +8 -8
  80. data_management/importers/add_timestamps_to_icct.py +78 -78
  81. data_management/importers/animl_results_to_md_results.py +158 -158
  82. data_management/importers/auckland_doc_test_to_json.py +9 -9
  83. data_management/importers/auckland_doc_to_json.py +8 -8
  84. data_management/importers/awc_to_json.py +7 -7
  85. data_management/importers/bellevue_to_json.py +15 -15
  86. data_management/importers/cacophony-thermal-importer.py +13 -13
  87. data_management/importers/carrizo_shrubfree_2018.py +8 -8
  88. data_management/importers/carrizo_trail_cam_2017.py +8 -8
  89. data_management/importers/cct_field_adjustments.py +9 -9
  90. data_management/importers/channel_islands_to_cct.py +10 -10
  91. data_management/importers/eMammal/copy_and_unzip_emammal.py +1 -0
  92. data_management/importers/ena24_to_json.py +7 -7
  93. data_management/importers/filenames_to_json.py +8 -8
  94. data_management/importers/helena_to_cct.py +7 -7
  95. data_management/importers/idaho-camera-traps.py +7 -7
  96. data_management/importers/idfg_iwildcam_lila_prep.py +10 -10
  97. data_management/importers/jb_csv_to_json.py +9 -9
  98. data_management/importers/mcgill_to_json.py +8 -8
  99. data_management/importers/missouri_to_json.py +18 -18
  100. data_management/importers/nacti_fieldname_adjustments.py +10 -10
  101. data_management/importers/noaa_seals_2019.py +7 -7
  102. data_management/importers/pc_to_json.py +7 -7
  103. data_management/importers/plot_wni_giraffes.py +7 -7
  104. data_management/importers/prepare-noaa-fish-data-for-lila.py +359 -359
  105. data_management/importers/prepare_zsl_imerit.py +7 -7
  106. data_management/importers/rspb_to_json.py +8 -8
  107. data_management/importers/save_the_elephants_survey_A.py +8 -8
  108. data_management/importers/save_the_elephants_survey_B.py +9 -9
  109. data_management/importers/snapshot_safari_importer.py +26 -26
  110. data_management/importers/snapshot_safari_importer_reprise.py +665 -665
  111. data_management/importers/snapshot_serengeti_lila.py +14 -14
  112. data_management/importers/sulross_get_exif.py +8 -9
  113. data_management/importers/timelapse_csv_set_to_json.py +11 -11
  114. data_management/importers/ubc_to_json.py +13 -13
  115. data_management/importers/umn_to_json.py +7 -7
  116. data_management/importers/wellington_to_json.py +8 -8
  117. data_management/importers/wi_to_json.py +9 -9
  118. data_management/importers/zamba_results_to_md_results.py +181 -181
  119. data_management/labelme_to_coco.py +65 -24
  120. data_management/labelme_to_yolo.py +8 -8
  121. data_management/lila/__init__.py +0 -0
  122. data_management/lila/add_locations_to_island_camera_traps.py +9 -9
  123. data_management/lila/add_locations_to_nacti.py +147 -147
  124. data_management/lila/create_lila_blank_set.py +13 -13
  125. data_management/lila/create_lila_test_set.py +8 -8
  126. data_management/lila/create_links_to_md_results_files.py +106 -106
  127. data_management/lila/download_lila_subset.py +44 -110
  128. data_management/lila/generate_lila_per_image_labels.py +55 -42
  129. data_management/lila/get_lila_annotation_counts.py +18 -15
  130. data_management/lila/get_lila_image_counts.py +11 -11
  131. data_management/lila/lila_common.py +96 -33
  132. data_management/lila/test_lila_metadata_urls.py +132 -116
  133. data_management/ocr_tools.py +173 -128
  134. data_management/read_exif.py +110 -97
  135. data_management/remap_coco_categories.py +83 -83
  136. data_management/remove_exif.py +58 -62
  137. data_management/resize_coco_dataset.py +30 -23
  138. data_management/wi_download_csv_to_coco.py +246 -239
  139. data_management/yolo_output_to_md_output.py +86 -73
  140. data_management/yolo_to_coco.py +300 -60
  141. detection/__init__.py +0 -0
  142. detection/detector_training/__init__.py +0 -0
  143. detection/process_video.py +85 -33
  144. detection/pytorch_detector.py +43 -25
  145. detection/run_detector.py +157 -72
  146. detection/run_detector_batch.py +179 -113
  147. detection/run_inference_with_yolov5_val.py +108 -48
  148. detection/run_tiled_inference.py +111 -40
  149. detection/tf_detector.py +51 -29
  150. detection/video_utils.py +606 -521
  151. docs/source/conf.py +43 -0
  152. md_utils/__init__.py +0 -0
  153. md_utils/azure_utils.py +9 -9
  154. md_utils/ct_utils.py +228 -68
  155. md_utils/directory_listing.py +59 -64
  156. md_utils/md_tests.py +968 -871
  157. md_utils/path_utils.py +460 -134
  158. md_utils/process_utils.py +157 -133
  159. md_utils/sas_blob_utils.py +20 -20
  160. md_utils/split_locations_into_train_val.py +45 -32
  161. md_utils/string_utils.py +33 -10
  162. md_utils/url_utils.py +176 -60
  163. md_utils/write_html_image_list.py +40 -33
  164. md_visualization/__init__.py +0 -0
  165. md_visualization/plot_utils.py +102 -109
  166. md_visualization/render_images_with_thumbnails.py +34 -34
  167. md_visualization/visualization_utils.py +597 -291
  168. md_visualization/visualize_db.py +76 -48
  169. md_visualization/visualize_detector_output.py +61 -42
  170. {megadetector-5.0.8.dist-info → megadetector-5.0.10.dist-info}/METADATA +13 -7
  171. megadetector-5.0.10.dist-info/RECORD +224 -0
  172. {megadetector-5.0.8.dist-info → megadetector-5.0.10.dist-info}/top_level.txt +1 -0
  173. taxonomy_mapping/__init__.py +0 -0
  174. taxonomy_mapping/map_lila_taxonomy_to_wi_taxonomy.py +342 -335
  175. taxonomy_mapping/map_new_lila_datasets.py +154 -154
  176. taxonomy_mapping/prepare_lila_taxonomy_release.py +142 -134
  177. taxonomy_mapping/preview_lila_taxonomy.py +591 -591
  178. taxonomy_mapping/retrieve_sample_image.py +12 -12
  179. taxonomy_mapping/simple_image_download.py +11 -11
  180. taxonomy_mapping/species_lookup.py +10 -10
  181. taxonomy_mapping/taxonomy_csv_checker.py +18 -18
  182. taxonomy_mapping/taxonomy_graph.py +47 -47
  183. taxonomy_mapping/validate_lila_category_mappings.py +83 -76
  184. data_management/cct_json_to_filename_json.py +0 -89
  185. data_management/cct_to_csv.py +0 -140
  186. data_management/databases/remove_corrupted_images_from_db.py +0 -191
  187. detection/detector_training/copy_checkpoints.py +0 -43
  188. megadetector-5.0.8.dist-info/RECORD +0 -205
  189. {megadetector-5.0.8.dist-info → megadetector-5.0.10.dist-info}/LICENSE +0 -0
  190. {megadetector-5.0.8.dist-info → megadetector-5.0.10.dist-info}/WHEEL +0 -0
@@ -1,158 +1,176 @@
1
- ########
2
- #
3
- # cct_to_md.py
4
- #
5
- # "Converts" a COCO Camera Traps file to a MD results file. Currently ignores
6
- # non-bounding-box annotations, and gives all annotations a confidence of 1.0.
7
- #
8
- # The only reason to do this is if you are going to add information to an existing
9
- # CCT-formatted dataset, and want to do that in Timelapse.
10
- #
11
- # Currently assumes that width and height are present in the input data, does not
12
- # read them from images.
13
- #
14
- ########
15
-
16
- #%% Constants and imports
17
-
18
- import os
19
- import json
20
-
21
- from collections import defaultdict
22
- from tqdm import tqdm
23
-
24
-
25
- #%% Functions
26
-
27
- def cct_to_md(input_filename,output_filename=None):
28
-
29
- ## Validate input
30
-
31
- assert os.path.isfile(input_filename)
32
-
33
- if (output_filename is None):
34
-
35
- tokens = os.path.splitext(input_filename)
36
- assert len(tokens) == 2
37
- output_filename = tokens[0] + '_md-format' + tokens[1]
38
-
39
-
40
- ## Read input
41
-
42
- with open(input_filename,'r') as f:
43
- d = json.load(f)
44
-
45
- for s in ['annotations','images','categories']:
46
- assert s in d.keys(), 'Cannot find category {} in input file, is this a CCT file?'.format(s)
47
-
48
-
49
- ## Prepare metadata
50
-
51
- image_id_to_annotations = defaultdict(list)
52
-
53
- # ann = d['annotations'][0]
54
- for ann in tqdm(d['annotations']):
55
- image_id_to_annotations[ann['image_id']].append(ann)
56
-
57
- category_id_to_name = {}
58
- for cat in d['categories']:
59
- category_id_to_name[str(cat['id'])] = cat['name']
60
-
61
- results = {}
62
-
63
- info = {}
64
- info['format_version'] = "1.3"
65
- info['detector'] = 'cct_to_md'
66
- results['info'] = info
67
- results['detection_categories'] = category_id_to_name
68
-
69
-
70
- ## Process images
71
-
72
- images_out = []
73
-
74
- # im = d['images'][0]
75
- for im in tqdm(d['images']):
76
-
77
- im_out = {}
78
- im_out['file'] = im['file_name']
79
- im_out['location'] = im['location']
80
- im_out['id'] = im['id']
81
-
82
- image_h = im['height']
83
- image_w = im['width']
84
-
85
- detections = []
86
-
87
- annotations_this_image = image_id_to_annotations[im['id']]
88
-
89
- # This field is no longer included in MD output files by default
90
- # max_detection_conf = 0
91
-
92
- for ann in annotations_this_image:
93
-
94
- if 'bbox' in ann:
95
-
96
- det = {}
97
- det['category'] = str(ann['category_id'])
98
- det['conf'] = 1.0
99
- # max_detection_conf = 1.0
100
-
101
- # MegaDetector: [x,y,width,height] (normalized, origin upper-left)
102
- # CCT: [x,y,width,height] (absolute, origin upper-left)
103
- bbox_in = ann['bbox']
104
- bbox_out = [bbox_in[0]/image_w,bbox_in[1]/image_h,
105
- bbox_in[2]/image_w,bbox_in[3]/image_h]
106
- det['bbox'] = bbox_out
107
- detections.append(det)
108
-
109
- # ...if there's a bounding box
110
-
111
- # ...for each annotation
112
-
113
- im_out['detections'] = detections
114
-
115
- # This field is no longer included in MD output files by default
116
- # im_out['max_detection_conf'] = max_detection_conf
117
-
118
- images_out.append(im_out)
119
-
120
- # ...for each image
121
-
122
-
123
- ## Write output
124
-
125
- results['images'] = images_out
126
-
127
- with open(output_filename,'w') as f:
128
- json.dump(results, f, indent=1)
129
-
130
- return output_filename
131
-
132
- # ...cct_to_md()
133
-
134
-
135
- #%% Interactive driver
136
-
137
- if False:
138
-
139
- pass
140
-
141
- #%%
142
-
143
- input_filename = r"G:\temp\noaa_estuary_fish.json"
144
- output_filename = None
145
- output_filename = cct_to_md(input_filename,output_filename)
146
-
147
- #%%
148
-
149
- from md_visualization import visualize_detector_output
150
-
151
- visualize_detector_output.visualize_detector_output(
152
- detector_output_path=output_filename,
153
- out_dir=r'g:\temp\fish_output',
154
- images_dir=r'G:\temp\noaa_estuary_fish-images\JPEGImages',
155
- output_image_width=-1,
156
- sample=100,
157
- render_detections_only=True)
158
-
1
+ """
2
+
3
+ cct_to_md.py
4
+
5
+ "Converts" a COCO Camera Traps file to a MD results file. Currently ignores
6
+ non-bounding-box annotations, and gives all annotations a confidence of 1.0.
7
+
8
+ The only reason to do this is if you are going to add information to an existing
9
+ CCT-formatted dataset, and you want to do that in Timelapse.
10
+
11
+ Currently assumes that width and height are present in the input data, does not
12
+ read them from images.
13
+
14
+ """
15
+
16
+ #%% Constants and imports
17
+
18
+ import os
19
+ import json
20
+
21
+ from collections import defaultdict
22
+ from tqdm import tqdm
23
+
24
+
25
+ #%% Functions
26
+
27
+ def cct_to_md(input_filename,output_filename=None):
28
+ """
29
+ "Converts" a COCO Camera Traps file to a MD results file. Currently ignores
30
+ non-bounding-box annotations, and gives all annotations a confidence of 1.0.
31
+
32
+ The only reason to do this is if you are going to add information to an existing
33
+ CCT-formatted dataset, and you want to do that in Timelapse.
34
+
35
+ Currently assumes that width and height are present in the input data, does not
36
+ read them from images.
37
+
38
+ Args:
39
+ input_filename (str): the COCO Camera Traps .json file to read
40
+ output_filename (str, optional): the .json file to write in MD results format
41
+
42
+ Returns:
43
+ dict: MD-formatted results, identical to the content of [output_filename] if
44
+ [output_filename] is not None
45
+ """
46
+
47
+ ## Validate input
48
+
49
+ assert os.path.isfile(input_filename)
50
+
51
+ if (output_filename is None):
52
+
53
+ tokens = os.path.splitext(input_filename)
54
+ assert len(tokens) == 2
55
+ output_filename = tokens[0] + '_md-format' + tokens[1]
56
+
57
+
58
+ ## Read input
59
+
60
+ with open(input_filename,'r') as f:
61
+ d = json.load(f)
62
+
63
+ for s in ['annotations','images','categories']:
64
+ assert s in d.keys(), 'Cannot find category {} in input file, is this a CCT file?'.format(s)
65
+
66
+
67
+ ## Prepare metadata
68
+
69
+ image_id_to_annotations = defaultdict(list)
70
+
71
+ # ann = d['annotations'][0]
72
+ for ann in tqdm(d['annotations']):
73
+ image_id_to_annotations[ann['image_id']].append(ann)
74
+
75
+ category_id_to_name = {}
76
+ for cat in d['categories']:
77
+ category_id_to_name[str(cat['id'])] = cat['name']
78
+
79
+ results = {}
80
+
81
+ info = {}
82
+ info['format_version'] = "1.3"
83
+ info['detector'] = 'cct_to_md'
84
+ results['info'] = info
85
+ results['detection_categories'] = category_id_to_name
86
+
87
+
88
+ ## Process images
89
+
90
+ images_out = []
91
+
92
+ # im = d['images'][0]
93
+ for im in tqdm(d['images']):
94
+
95
+ im_out = {}
96
+ im_out['file'] = im['file_name']
97
+ im_out['location'] = im['location']
98
+ im_out['id'] = im['id']
99
+
100
+ image_h = im['height']
101
+ image_w = im['width']
102
+
103
+ detections = []
104
+
105
+ annotations_this_image = image_id_to_annotations[im['id']]
106
+
107
+ # This field is no longer included in MD output files by default
108
+ # max_detection_conf = 0
109
+
110
+ for ann in annotations_this_image:
111
+
112
+ if 'bbox' in ann:
113
+
114
+ det = {}
115
+ det['category'] = str(ann['category_id'])
116
+ det['conf'] = 1.0
117
+ # max_detection_conf = 1.0
118
+
119
+ # MegaDetector: [x,y,width,height] (normalized, origin upper-left)
120
+ # CCT: [x,y,width,height] (absolute, origin upper-left)
121
+ bbox_in = ann['bbox']
122
+ bbox_out = [bbox_in[0]/image_w,bbox_in[1]/image_h,
123
+ bbox_in[2]/image_w,bbox_in[3]/image_h]
124
+ det['bbox'] = bbox_out
125
+ detections.append(det)
126
+
127
+ # ...if there's a bounding box
128
+
129
+ # ...for each annotation
130
+
131
+ im_out['detections'] = detections
132
+
133
+ # This field is no longer included in MD output files by default
134
+ # im_out['max_detection_conf'] = max_detection_conf
135
+
136
+ images_out.append(im_out)
137
+
138
+ # ...for each image
139
+
140
+
141
+ ## Write output
142
+
143
+ results['images'] = images_out
144
+
145
+ with open(output_filename,'w') as f:
146
+ json.dump(results, f, indent=1)
147
+
148
+ return output_filename
149
+
150
+ # ...cct_to_md()
151
+
152
+
153
+ #%% Interactive driver
154
+
155
+ if False:
156
+
157
+ pass
158
+
159
+ #%%
160
+
161
+ input_filename = r"G:\temp\noaa_estuary_fish.json"
162
+ output_filename = None
163
+ output_filename = cct_to_md(input_filename,output_filename)
164
+
165
+ #%%
166
+
167
+ from md_visualization import visualize_detector_output
168
+
169
+ visualize_detector_output.visualize_detector_output(
170
+ detector_output_path=output_filename,
171
+ out_dir=r'g:\temp\fish_output',
172
+ images_dir=r'g:\temp\noaa_estuary_fish-images\JPEGImages',
173
+ output_image_width=-1,
174
+ sample=100,
175
+ render_detections_only=True)
176
+