megadetector 5.0.7__py3-none-any.whl → 5.0.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of megadetector might be problematic. Click here for more details.
- api/__init__.py +0 -0
- api/batch_processing/__init__.py +0 -0
- api/batch_processing/api_core/__init__.py +0 -0
- api/batch_processing/api_core/batch_service/__init__.py +0 -0
- api/batch_processing/api_core/batch_service/score.py +0 -1
- api/batch_processing/api_core/server_job_status_table.py +0 -1
- api/batch_processing/api_core_support/__init__.py +0 -0
- api/batch_processing/api_core_support/aggregate_results_manually.py +0 -1
- api/batch_processing/api_support/__init__.py +0 -0
- api/batch_processing/api_support/summarize_daily_activity.py +0 -1
- api/batch_processing/data_preparation/__init__.py +0 -0
- api/batch_processing/data_preparation/manage_local_batch.py +93 -79
- api/batch_processing/data_preparation/manage_video_batch.py +8 -8
- api/batch_processing/integration/digiKam/xmp_integration.py +0 -1
- api/batch_processing/integration/eMammal/test_scripts/push_annotations_to_emammal.py +0 -1
- api/batch_processing/postprocessing/__init__.py +0 -0
- api/batch_processing/postprocessing/add_max_conf.py +12 -12
- api/batch_processing/postprocessing/categorize_detections_by_size.py +32 -14
- api/batch_processing/postprocessing/combine_api_outputs.py +69 -55
- api/batch_processing/postprocessing/compare_batch_results.py +114 -44
- api/batch_processing/postprocessing/convert_output_format.py +62 -19
- api/batch_processing/postprocessing/load_api_results.py +17 -20
- api/batch_processing/postprocessing/md_to_coco.py +31 -21
- api/batch_processing/postprocessing/md_to_labelme.py +165 -68
- api/batch_processing/postprocessing/merge_detections.py +40 -15
- api/batch_processing/postprocessing/postprocess_batch_results.py +270 -186
- api/batch_processing/postprocessing/remap_detection_categories.py +170 -0
- api/batch_processing/postprocessing/render_detection_confusion_matrix.py +75 -39
- api/batch_processing/postprocessing/repeat_detection_elimination/find_repeat_detections.py +53 -44
- api/batch_processing/postprocessing/repeat_detection_elimination/remove_repeat_detections.py +25 -14
- api/batch_processing/postprocessing/repeat_detection_elimination/repeat_detections_core.py +244 -160
- api/batch_processing/postprocessing/separate_detections_into_folders.py +159 -114
- api/batch_processing/postprocessing/subset_json_detector_output.py +146 -169
- api/batch_processing/postprocessing/top_folders_to_bottom.py +77 -43
- api/synchronous/__init__.py +0 -0
- api/synchronous/api_core/animal_detection_api/__init__.py +0 -0
- api/synchronous/api_core/animal_detection_api/api_backend.py +0 -2
- api/synchronous/api_core/animal_detection_api/api_frontend.py +266 -268
- api/synchronous/api_core/animal_detection_api/config.py +35 -35
- api/synchronous/api_core/tests/__init__.py +0 -0
- api/synchronous/api_core/tests/load_test.py +109 -109
- classification/__init__.py +0 -0
- classification/aggregate_classifier_probs.py +21 -24
- classification/analyze_failed_images.py +11 -13
- classification/cache_batchapi_outputs.py +51 -51
- classification/create_classification_dataset.py +69 -68
- classification/crop_detections.py +54 -53
- classification/csv_to_json.py +97 -100
- classification/detect_and_crop.py +105 -105
- classification/evaluate_model.py +43 -42
- classification/identify_mislabeled_candidates.py +47 -46
- classification/json_to_azcopy_list.py +10 -10
- classification/json_validator.py +72 -71
- classification/map_classification_categories.py +44 -43
- classification/merge_classification_detection_output.py +68 -68
- classification/prepare_classification_script.py +157 -154
- classification/prepare_classification_script_mc.py +228 -228
- classification/run_classifier.py +27 -26
- classification/save_mislabeled.py +30 -30
- classification/train_classifier.py +20 -20
- classification/train_classifier_tf.py +21 -22
- classification/train_utils.py +10 -10
- data_management/__init__.py +0 -0
- data_management/annotations/__init__.py +0 -0
- data_management/annotations/annotation_constants.py +18 -31
- data_management/camtrap_dp_to_coco.py +238 -0
- data_management/cct_json_utils.py +107 -59
- data_management/cct_to_md.py +176 -158
- data_management/cct_to_wi.py +247 -219
- data_management/coco_to_labelme.py +272 -0
- data_management/coco_to_yolo.py +86 -62
- data_management/databases/__init__.py +0 -0
- data_management/databases/add_width_and_height_to_db.py +20 -16
- data_management/databases/combine_coco_camera_traps_files.py +35 -31
- data_management/databases/integrity_check_json_db.py +130 -83
- data_management/databases/subset_json_db.py +25 -16
- data_management/generate_crops_from_cct.py +27 -45
- data_management/get_image_sizes.py +188 -144
- data_management/importers/add_nacti_sizes.py +8 -8
- data_management/importers/add_timestamps_to_icct.py +78 -78
- data_management/importers/animl_results_to_md_results.py +158 -160
- data_management/importers/auckland_doc_test_to_json.py +9 -9
- data_management/importers/auckland_doc_to_json.py +8 -8
- data_management/importers/awc_to_json.py +7 -7
- data_management/importers/bellevue_to_json.py +15 -15
- data_management/importers/cacophony-thermal-importer.py +13 -13
- data_management/importers/carrizo_shrubfree_2018.py +8 -8
- data_management/importers/carrizo_trail_cam_2017.py +8 -8
- data_management/importers/cct_field_adjustments.py +9 -9
- data_management/importers/channel_islands_to_cct.py +10 -10
- data_management/importers/eMammal/copy_and_unzip_emammal.py +1 -0
- data_management/importers/ena24_to_json.py +7 -7
- data_management/importers/filenames_to_json.py +8 -8
- data_management/importers/helena_to_cct.py +7 -7
- data_management/importers/idaho-camera-traps.py +7 -7
- data_management/importers/idfg_iwildcam_lila_prep.py +10 -10
- data_management/importers/jb_csv_to_json.py +9 -9
- data_management/importers/mcgill_to_json.py +8 -8
- data_management/importers/missouri_to_json.py +18 -18
- data_management/importers/nacti_fieldname_adjustments.py +10 -10
- data_management/importers/noaa_seals_2019.py +8 -8
- data_management/importers/pc_to_json.py +7 -7
- data_management/importers/plot_wni_giraffes.py +7 -7
- data_management/importers/prepare-noaa-fish-data-for-lila.py +359 -359
- data_management/importers/prepare_zsl_imerit.py +7 -7
- data_management/importers/rspb_to_json.py +8 -8
- data_management/importers/save_the_elephants_survey_A.py +8 -8
- data_management/importers/save_the_elephants_survey_B.py +9 -9
- data_management/importers/snapshot_safari_importer.py +26 -26
- data_management/importers/snapshot_safari_importer_reprise.py +665 -665
- data_management/importers/snapshot_serengeti_lila.py +14 -14
- data_management/importers/sulross_get_exif.py +8 -9
- data_management/importers/timelapse_csv_set_to_json.py +11 -11
- data_management/importers/ubc_to_json.py +13 -13
- data_management/importers/umn_to_json.py +7 -7
- data_management/importers/wellington_to_json.py +8 -8
- data_management/importers/wi_to_json.py +9 -9
- data_management/importers/zamba_results_to_md_results.py +181 -181
- data_management/labelme_to_coco.py +309 -159
- data_management/labelme_to_yolo.py +103 -60
- data_management/lila/__init__.py +0 -0
- data_management/lila/add_locations_to_island_camera_traps.py +9 -9
- data_management/lila/add_locations_to_nacti.py +147 -147
- data_management/lila/create_lila_blank_set.py +114 -31
- data_management/lila/create_lila_test_set.py +8 -8
- data_management/lila/create_links_to_md_results_files.py +106 -106
- data_management/lila/download_lila_subset.py +92 -90
- data_management/lila/generate_lila_per_image_labels.py +56 -43
- data_management/lila/get_lila_annotation_counts.py +18 -15
- data_management/lila/get_lila_image_counts.py +11 -11
- data_management/lila/lila_common.py +103 -70
- data_management/lila/test_lila_metadata_urls.py +132 -116
- data_management/ocr_tools.py +173 -128
- data_management/read_exif.py +161 -99
- data_management/remap_coco_categories.py +84 -0
- data_management/remove_exif.py +58 -62
- data_management/resize_coco_dataset.py +32 -44
- data_management/wi_download_csv_to_coco.py +246 -0
- data_management/yolo_output_to_md_output.py +86 -73
- data_management/yolo_to_coco.py +535 -95
- detection/__init__.py +0 -0
- detection/detector_training/__init__.py +0 -0
- detection/process_video.py +85 -33
- detection/pytorch_detector.py +43 -25
- detection/run_detector.py +157 -72
- detection/run_detector_batch.py +189 -114
- detection/run_inference_with_yolov5_val.py +118 -51
- detection/run_tiled_inference.py +113 -42
- detection/tf_detector.py +51 -28
- detection/video_utils.py +606 -521
- docs/source/conf.py +43 -0
- md_utils/__init__.py +0 -0
- md_utils/azure_utils.py +9 -9
- md_utils/ct_utils.py +249 -70
- md_utils/directory_listing.py +59 -64
- md_utils/md_tests.py +968 -862
- md_utils/path_utils.py +655 -155
- md_utils/process_utils.py +157 -133
- md_utils/sas_blob_utils.py +20 -20
- md_utils/split_locations_into_train_val.py +45 -32
- md_utils/string_utils.py +33 -10
- md_utils/url_utils.py +208 -27
- md_utils/write_html_image_list.py +51 -35
- md_visualization/__init__.py +0 -0
- md_visualization/plot_utils.py +102 -109
- md_visualization/render_images_with_thumbnails.py +34 -34
- md_visualization/visualization_utils.py +908 -311
- md_visualization/visualize_db.py +109 -58
- md_visualization/visualize_detector_output.py +61 -42
- {megadetector-5.0.7.dist-info → megadetector-5.0.9.dist-info}/METADATA +21 -17
- megadetector-5.0.9.dist-info/RECORD +224 -0
- {megadetector-5.0.7.dist-info → megadetector-5.0.9.dist-info}/WHEEL +1 -1
- {megadetector-5.0.7.dist-info → megadetector-5.0.9.dist-info}/top_level.txt +1 -0
- taxonomy_mapping/__init__.py +0 -0
- taxonomy_mapping/map_lila_taxonomy_to_wi_taxonomy.py +342 -335
- taxonomy_mapping/map_new_lila_datasets.py +154 -154
- taxonomy_mapping/prepare_lila_taxonomy_release.py +142 -134
- taxonomy_mapping/preview_lila_taxonomy.py +591 -591
- taxonomy_mapping/retrieve_sample_image.py +12 -12
- taxonomy_mapping/simple_image_download.py +11 -11
- taxonomy_mapping/species_lookup.py +10 -10
- taxonomy_mapping/taxonomy_csv_checker.py +18 -18
- taxonomy_mapping/taxonomy_graph.py +47 -47
- taxonomy_mapping/validate_lila_category_mappings.py +83 -76
- data_management/cct_json_to_filename_json.py +0 -89
- data_management/cct_to_csv.py +0 -140
- data_management/databases/remove_corrupted_images_from_db.py +0 -191
- detection/detector_training/copy_checkpoints.py +0 -43
- md_visualization/visualize_megadb.py +0 -183
- megadetector-5.0.7.dist-info/RECORD +0 -202
- {megadetector-5.0.7.dist-info → megadetector-5.0.9.dist-info}/LICENSE +0 -0
data_management/cct_to_md.py
CHANGED
|
@@ -1,158 +1,176 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
#%% Constants and imports
|
|
17
|
-
|
|
18
|
-
import os
|
|
19
|
-
import json
|
|
20
|
-
|
|
21
|
-
from collections import defaultdict
|
|
22
|
-
from tqdm import tqdm
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
#%% Functions
|
|
26
|
-
|
|
27
|
-
def cct_to_md(input_filename,output_filename=None):
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
1
|
+
"""
|
|
2
|
+
|
|
3
|
+
cct_to_md.py
|
|
4
|
+
|
|
5
|
+
"Converts" a COCO Camera Traps file to a MD results file. Currently ignores
|
|
6
|
+
non-bounding-box annotations, and gives all annotations a confidence of 1.0.
|
|
7
|
+
|
|
8
|
+
The only reason to do this is if you are going to add information to an existing
|
|
9
|
+
CCT-formatted dataset, and you want to do that in Timelapse.
|
|
10
|
+
|
|
11
|
+
Currently assumes that width and height are present in the input data, does not
|
|
12
|
+
read them from images.
|
|
13
|
+
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
#%% Constants and imports
|
|
17
|
+
|
|
18
|
+
import os
|
|
19
|
+
import json
|
|
20
|
+
|
|
21
|
+
from collections import defaultdict
|
|
22
|
+
from tqdm import tqdm
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
#%% Functions
|
|
26
|
+
|
|
27
|
+
def cct_to_md(input_filename,output_filename=None):
|
|
28
|
+
"""
|
|
29
|
+
"Converts" a COCO Camera Traps file to a MD results file. Currently ignores
|
|
30
|
+
non-bounding-box annotations, and gives all annotations a confidence of 1.0.
|
|
31
|
+
|
|
32
|
+
The only reason to do this is if you are going to add information to an existing
|
|
33
|
+
CCT-formatted dataset, and you want to do that in Timelapse.
|
|
34
|
+
|
|
35
|
+
Currently assumes that width and height are present in the input data, does not
|
|
36
|
+
read them from images.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
input_filename (str): the COCO Camera Traps .json file to read
|
|
40
|
+
output_filename (str, optional): the .json file to write in MD results format
|
|
41
|
+
|
|
42
|
+
Returns:
|
|
43
|
+
dict: MD-formatted results, identical to the content of [output_filename] if
|
|
44
|
+
[output_filename] is not None
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
## Validate input
|
|
48
|
+
|
|
49
|
+
assert os.path.isfile(input_filename)
|
|
50
|
+
|
|
51
|
+
if (output_filename is None):
|
|
52
|
+
|
|
53
|
+
tokens = os.path.splitext(input_filename)
|
|
54
|
+
assert len(tokens) == 2
|
|
55
|
+
output_filename = tokens[0] + '_md-format' + tokens[1]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
## Read input
|
|
59
|
+
|
|
60
|
+
with open(input_filename,'r') as f:
|
|
61
|
+
d = json.load(f)
|
|
62
|
+
|
|
63
|
+
for s in ['annotations','images','categories']:
|
|
64
|
+
assert s in d.keys(), 'Cannot find category {} in input file, is this a CCT file?'.format(s)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
## Prepare metadata
|
|
68
|
+
|
|
69
|
+
image_id_to_annotations = defaultdict(list)
|
|
70
|
+
|
|
71
|
+
# ann = d['annotations'][0]
|
|
72
|
+
for ann in tqdm(d['annotations']):
|
|
73
|
+
image_id_to_annotations[ann['image_id']].append(ann)
|
|
74
|
+
|
|
75
|
+
category_id_to_name = {}
|
|
76
|
+
for cat in d['categories']:
|
|
77
|
+
category_id_to_name[str(cat['id'])] = cat['name']
|
|
78
|
+
|
|
79
|
+
results = {}
|
|
80
|
+
|
|
81
|
+
info = {}
|
|
82
|
+
info['format_version'] = "1.3"
|
|
83
|
+
info['detector'] = 'cct_to_md'
|
|
84
|
+
results['info'] = info
|
|
85
|
+
results['detection_categories'] = category_id_to_name
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
## Process images
|
|
89
|
+
|
|
90
|
+
images_out = []
|
|
91
|
+
|
|
92
|
+
# im = d['images'][0]
|
|
93
|
+
for im in tqdm(d['images']):
|
|
94
|
+
|
|
95
|
+
im_out = {}
|
|
96
|
+
im_out['file'] = im['file_name']
|
|
97
|
+
im_out['location'] = im['location']
|
|
98
|
+
im_out['id'] = im['id']
|
|
99
|
+
|
|
100
|
+
image_h = im['height']
|
|
101
|
+
image_w = im['width']
|
|
102
|
+
|
|
103
|
+
detections = []
|
|
104
|
+
|
|
105
|
+
annotations_this_image = image_id_to_annotations[im['id']]
|
|
106
|
+
|
|
107
|
+
# This field is no longer included in MD output files by default
|
|
108
|
+
# max_detection_conf = 0
|
|
109
|
+
|
|
110
|
+
for ann in annotations_this_image:
|
|
111
|
+
|
|
112
|
+
if 'bbox' in ann:
|
|
113
|
+
|
|
114
|
+
det = {}
|
|
115
|
+
det['category'] = str(ann['category_id'])
|
|
116
|
+
det['conf'] = 1.0
|
|
117
|
+
# max_detection_conf = 1.0
|
|
118
|
+
|
|
119
|
+
# MegaDetector: [x,y,width,height] (normalized, origin upper-left)
|
|
120
|
+
# CCT: [x,y,width,height] (absolute, origin upper-left)
|
|
121
|
+
bbox_in = ann['bbox']
|
|
122
|
+
bbox_out = [bbox_in[0]/image_w,bbox_in[1]/image_h,
|
|
123
|
+
bbox_in[2]/image_w,bbox_in[3]/image_h]
|
|
124
|
+
det['bbox'] = bbox_out
|
|
125
|
+
detections.append(det)
|
|
126
|
+
|
|
127
|
+
# ...if there's a bounding box
|
|
128
|
+
|
|
129
|
+
# ...for each annotation
|
|
130
|
+
|
|
131
|
+
im_out['detections'] = detections
|
|
132
|
+
|
|
133
|
+
# This field is no longer included in MD output files by default
|
|
134
|
+
# im_out['max_detection_conf'] = max_detection_conf
|
|
135
|
+
|
|
136
|
+
images_out.append(im_out)
|
|
137
|
+
|
|
138
|
+
# ...for each image
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
## Write output
|
|
142
|
+
|
|
143
|
+
results['images'] = images_out
|
|
144
|
+
|
|
145
|
+
with open(output_filename,'w') as f:
|
|
146
|
+
json.dump(results, f, indent=1)
|
|
147
|
+
|
|
148
|
+
return output_filename
|
|
149
|
+
|
|
150
|
+
# ...cct_to_md()
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
#%% Interactive driver
|
|
154
|
+
|
|
155
|
+
if False:
|
|
156
|
+
|
|
157
|
+
pass
|
|
158
|
+
|
|
159
|
+
#%%
|
|
160
|
+
|
|
161
|
+
input_filename = r"G:\temp\noaa_estuary_fish.json"
|
|
162
|
+
output_filename = None
|
|
163
|
+
output_filename = cct_to_md(input_filename,output_filename)
|
|
164
|
+
|
|
165
|
+
#%%
|
|
166
|
+
|
|
167
|
+
from md_visualization import visualize_detector_output
|
|
168
|
+
|
|
169
|
+
visualize_detector_output.visualize_detector_output(
|
|
170
|
+
detector_output_path=output_filename,
|
|
171
|
+
out_dir=r'g:\temp\fish_output',
|
|
172
|
+
images_dir=r'g:\temp\noaa_estuary_fish-images\JPEGImages',
|
|
173
|
+
output_image_width=-1,
|
|
174
|
+
sample=100,
|
|
175
|
+
render_detections_only=True)
|
|
176
|
+
|