megadetector 10.0.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- megadetector/__init__.py +0 -0
- megadetector/api/__init__.py +0 -0
- megadetector/api/batch_processing/integration/digiKam/setup.py +6 -0
- megadetector/api/batch_processing/integration/digiKam/xmp_integration.py +465 -0
- megadetector/api/batch_processing/integration/eMammal/test_scripts/config_template.py +5 -0
- megadetector/api/batch_processing/integration/eMammal/test_scripts/push_annotations_to_emammal.py +125 -0
- megadetector/api/batch_processing/integration/eMammal/test_scripts/select_images_for_testing.py +55 -0
- megadetector/classification/__init__.py +0 -0
- megadetector/classification/aggregate_classifier_probs.py +108 -0
- megadetector/classification/analyze_failed_images.py +227 -0
- megadetector/classification/cache_batchapi_outputs.py +198 -0
- megadetector/classification/create_classification_dataset.py +626 -0
- megadetector/classification/crop_detections.py +516 -0
- megadetector/classification/csv_to_json.py +226 -0
- megadetector/classification/detect_and_crop.py +853 -0
- megadetector/classification/efficientnet/__init__.py +9 -0
- megadetector/classification/efficientnet/model.py +415 -0
- megadetector/classification/efficientnet/utils.py +608 -0
- megadetector/classification/evaluate_model.py +520 -0
- megadetector/classification/identify_mislabeled_candidates.py +152 -0
- megadetector/classification/json_to_azcopy_list.py +63 -0
- megadetector/classification/json_validator.py +696 -0
- megadetector/classification/map_classification_categories.py +276 -0
- megadetector/classification/merge_classification_detection_output.py +509 -0
- megadetector/classification/prepare_classification_script.py +194 -0
- megadetector/classification/prepare_classification_script_mc.py +228 -0
- megadetector/classification/run_classifier.py +287 -0
- megadetector/classification/save_mislabeled.py +110 -0
- megadetector/classification/train_classifier.py +827 -0
- megadetector/classification/train_classifier_tf.py +725 -0
- megadetector/classification/train_utils.py +323 -0
- megadetector/data_management/__init__.py +0 -0
- megadetector/data_management/animl_to_md.py +161 -0
- megadetector/data_management/annotations/__init__.py +0 -0
- megadetector/data_management/annotations/annotation_constants.py +33 -0
- megadetector/data_management/camtrap_dp_to_coco.py +270 -0
- megadetector/data_management/cct_json_utils.py +566 -0
- megadetector/data_management/cct_to_md.py +184 -0
- megadetector/data_management/cct_to_wi.py +293 -0
- megadetector/data_management/coco_to_labelme.py +284 -0
- megadetector/data_management/coco_to_yolo.py +701 -0
- megadetector/data_management/databases/__init__.py +0 -0
- megadetector/data_management/databases/add_width_and_height_to_db.py +107 -0
- megadetector/data_management/databases/combine_coco_camera_traps_files.py +210 -0
- megadetector/data_management/databases/integrity_check_json_db.py +563 -0
- megadetector/data_management/databases/subset_json_db.py +195 -0
- megadetector/data_management/generate_crops_from_cct.py +200 -0
- megadetector/data_management/get_image_sizes.py +164 -0
- megadetector/data_management/labelme_to_coco.py +559 -0
- megadetector/data_management/labelme_to_yolo.py +349 -0
- megadetector/data_management/lila/__init__.py +0 -0
- megadetector/data_management/lila/create_lila_blank_set.py +556 -0
- megadetector/data_management/lila/create_lila_test_set.py +192 -0
- megadetector/data_management/lila/create_links_to_md_results_files.py +106 -0
- megadetector/data_management/lila/download_lila_subset.py +182 -0
- megadetector/data_management/lila/generate_lila_per_image_labels.py +777 -0
- megadetector/data_management/lila/get_lila_annotation_counts.py +174 -0
- megadetector/data_management/lila/get_lila_image_counts.py +112 -0
- megadetector/data_management/lila/lila_common.py +319 -0
- megadetector/data_management/lila/test_lila_metadata_urls.py +164 -0
- megadetector/data_management/mewc_to_md.py +344 -0
- megadetector/data_management/ocr_tools.py +873 -0
- megadetector/data_management/read_exif.py +964 -0
- megadetector/data_management/remap_coco_categories.py +195 -0
- megadetector/data_management/remove_exif.py +156 -0
- megadetector/data_management/rename_images.py +194 -0
- megadetector/data_management/resize_coco_dataset.py +665 -0
- megadetector/data_management/speciesnet_to_md.py +41 -0
- megadetector/data_management/wi_download_csv_to_coco.py +247 -0
- megadetector/data_management/yolo_output_to_md_output.py +594 -0
- megadetector/data_management/yolo_to_coco.py +984 -0
- megadetector/data_management/zamba_to_md.py +188 -0
- megadetector/detection/__init__.py +0 -0
- megadetector/detection/change_detection.py +840 -0
- megadetector/detection/process_video.py +479 -0
- megadetector/detection/pytorch_detector.py +1451 -0
- megadetector/detection/run_detector.py +1267 -0
- megadetector/detection/run_detector_batch.py +2172 -0
- megadetector/detection/run_inference_with_yolov5_val.py +1314 -0
- megadetector/detection/run_md_and_speciesnet.py +1604 -0
- megadetector/detection/run_tiled_inference.py +1044 -0
- megadetector/detection/tf_detector.py +209 -0
- megadetector/detection/video_utils.py +1379 -0
- megadetector/postprocessing/__init__.py +0 -0
- megadetector/postprocessing/add_max_conf.py +72 -0
- megadetector/postprocessing/categorize_detections_by_size.py +166 -0
- megadetector/postprocessing/classification_postprocessing.py +1943 -0
- megadetector/postprocessing/combine_batch_outputs.py +249 -0
- megadetector/postprocessing/compare_batch_results.py +2110 -0
- megadetector/postprocessing/convert_output_format.py +403 -0
- megadetector/postprocessing/create_crop_folder.py +629 -0
- megadetector/postprocessing/detector_calibration.py +570 -0
- megadetector/postprocessing/generate_csv_report.py +522 -0
- megadetector/postprocessing/load_api_results.py +223 -0
- megadetector/postprocessing/md_to_coco.py +428 -0
- megadetector/postprocessing/md_to_labelme.py +351 -0
- megadetector/postprocessing/md_to_wi.py +41 -0
- megadetector/postprocessing/merge_detections.py +392 -0
- megadetector/postprocessing/postprocess_batch_results.py +2140 -0
- megadetector/postprocessing/remap_detection_categories.py +226 -0
- megadetector/postprocessing/render_detection_confusion_matrix.py +677 -0
- megadetector/postprocessing/repeat_detection_elimination/find_repeat_detections.py +206 -0
- megadetector/postprocessing/repeat_detection_elimination/remove_repeat_detections.py +82 -0
- megadetector/postprocessing/repeat_detection_elimination/repeat_detections_core.py +1665 -0
- megadetector/postprocessing/separate_detections_into_folders.py +795 -0
- megadetector/postprocessing/subset_json_detector_output.py +964 -0
- megadetector/postprocessing/top_folders_to_bottom.py +238 -0
- megadetector/postprocessing/validate_batch_results.py +332 -0
- megadetector/taxonomy_mapping/__init__.py +0 -0
- megadetector/taxonomy_mapping/map_lila_taxonomy_to_wi_taxonomy.py +491 -0
- megadetector/taxonomy_mapping/map_new_lila_datasets.py +211 -0
- megadetector/taxonomy_mapping/prepare_lila_taxonomy_release.py +165 -0
- megadetector/taxonomy_mapping/preview_lila_taxonomy.py +543 -0
- megadetector/taxonomy_mapping/retrieve_sample_image.py +71 -0
- megadetector/taxonomy_mapping/simple_image_download.py +231 -0
- megadetector/taxonomy_mapping/species_lookup.py +1008 -0
- megadetector/taxonomy_mapping/taxonomy_csv_checker.py +159 -0
- megadetector/taxonomy_mapping/taxonomy_graph.py +346 -0
- megadetector/taxonomy_mapping/validate_lila_category_mappings.py +83 -0
- megadetector/tests/__init__.py +0 -0
- megadetector/tests/test_nms_synthetic.py +335 -0
- megadetector/utils/__init__.py +0 -0
- megadetector/utils/ct_utils.py +1857 -0
- megadetector/utils/directory_listing.py +199 -0
- megadetector/utils/extract_frames_from_video.py +307 -0
- megadetector/utils/gpu_test.py +125 -0
- megadetector/utils/md_tests.py +2072 -0
- megadetector/utils/path_utils.py +2872 -0
- megadetector/utils/process_utils.py +172 -0
- megadetector/utils/split_locations_into_train_val.py +237 -0
- megadetector/utils/string_utils.py +234 -0
- megadetector/utils/url_utils.py +825 -0
- megadetector/utils/wi_platform_utils.py +968 -0
- megadetector/utils/wi_taxonomy_utils.py +1766 -0
- megadetector/utils/write_html_image_list.py +239 -0
- megadetector/visualization/__init__.py +0 -0
- megadetector/visualization/plot_utils.py +309 -0
- megadetector/visualization/render_images_with_thumbnails.py +243 -0
- megadetector/visualization/visualization_utils.py +1973 -0
- megadetector/visualization/visualize_db.py +630 -0
- megadetector/visualization/visualize_detector_output.py +498 -0
- megadetector/visualization/visualize_video_output.py +705 -0
- megadetector-10.0.15.dist-info/METADATA +115 -0
- megadetector-10.0.15.dist-info/RECORD +147 -0
- megadetector-10.0.15.dist-info/WHEEL +5 -0
- megadetector-10.0.15.dist-info/licenses/LICENSE +19 -0
- megadetector-10.0.15.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,701 @@
|
|
|
1
|
+
"""
|
|
2
|
+
|
|
3
|
+
coco_to_yolo.py
|
|
4
|
+
|
|
5
|
+
Converts a COCO-formatted dataset to a YOLO-formatted dataset, flattening
|
|
6
|
+
the dataset (to a single folder) in the process.
|
|
7
|
+
|
|
8
|
+
If the input and output folders are the same, writes .txt files to the input folder,
|
|
9
|
+
and neither moves nor modifies images.
|
|
10
|
+
|
|
11
|
+
Currently ignores segmentation masks, and errors if an annotation has a
|
|
12
|
+
segmentation polygon but no bbox.
|
|
13
|
+
|
|
14
|
+
Has only been tested on a handful of COCO Camera Traps data sets; if you
|
|
15
|
+
use it for more general COCO conversion, YMMV.
|
|
16
|
+
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
#%% Imports and constants
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import shutil
|
|
24
|
+
import sys
|
|
25
|
+
import argparse
|
|
26
|
+
|
|
27
|
+
from collections import defaultdict
|
|
28
|
+
from tqdm import tqdm
|
|
29
|
+
|
|
30
|
+
from megadetector.utils.path_utils import safe_create_link,find_images
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
#%% Support functions
|
|
34
|
+
|
|
35
|
+
def write_yolo_dataset_file(yolo_dataset_file,
|
|
36
|
+
dataset_base_dir,
|
|
37
|
+
class_list,
|
|
38
|
+
train_folder_relative=None,
|
|
39
|
+
val_folder_relative=None,
|
|
40
|
+
test_folder_relative=None):
|
|
41
|
+
"""
|
|
42
|
+
Write a YOLOv5 dataset.yaml file to the absolute path [yolo_dataset_file] (should
|
|
43
|
+
have a .yaml extension, though it's only a warning if it doesn't).
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
yolo_dataset_file (str): the file, typically ending in .yaml or .yml, to write.
|
|
47
|
+
Does not have to be within dataset_base_dir.
|
|
48
|
+
dataset_base_dir (str): the absolute base path of the YOLO dataset
|
|
49
|
+
class_list (list or str): an ordered list of class names (the first item will be class 0,
|
|
50
|
+
etc.), or the name of a text file containing an ordered list of class names (one per
|
|
51
|
+
line, starting from class zero).
|
|
52
|
+
train_folder_relative (str, optional): train folder name, used only to
|
|
53
|
+
populate dataset.yaml. Can also be a filename (e.g. a .txt file with image
|
|
54
|
+
files).
|
|
55
|
+
val_folder_relative (str, optional): val folder name, used only to
|
|
56
|
+
populate dataset.yaml. Can also be a filename (e.g. a .txt file with image
|
|
57
|
+
files).
|
|
58
|
+
test_folder_relative (str, optional): test folder name, used only to
|
|
59
|
+
populate dataset.yaml. Can also be a filename (e.g. a .txt file with image
|
|
60
|
+
files).
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
# Read class names
|
|
64
|
+
if isinstance(class_list,str):
|
|
65
|
+
with open(class_list,'r') as f:
|
|
66
|
+
class_lines = f.readlines()
|
|
67
|
+
class_lines = [s.strip() for s in class_lines]
|
|
68
|
+
class_list = [s for s in class_lines if len(s) > 0]
|
|
69
|
+
|
|
70
|
+
if not (yolo_dataset_file.endswith('.yml') or yolo_dataset_file.endswith('.yaml')):
|
|
71
|
+
print('Warning: writing dataset file to a non-yml/yaml extension:\n{}'.format(
|
|
72
|
+
yolo_dataset_file))
|
|
73
|
+
|
|
74
|
+
# Write dataset.yaml
|
|
75
|
+
with open(yolo_dataset_file,'w') as f:
|
|
76
|
+
|
|
77
|
+
f.write('# Train/val sets\n')
|
|
78
|
+
f.write('path: {}\n'.format(dataset_base_dir))
|
|
79
|
+
if train_folder_relative is not None:
|
|
80
|
+
f.write('train: {}\n'.format(train_folder_relative))
|
|
81
|
+
if val_folder_relative is not None:
|
|
82
|
+
f.write('val: {}\n'.format(val_folder_relative))
|
|
83
|
+
if test_folder_relative is not None:
|
|
84
|
+
f.write('test: {}\n'.format(test_folder_relative))
|
|
85
|
+
|
|
86
|
+
f.write('\n')
|
|
87
|
+
|
|
88
|
+
f.write('# Classes\n')
|
|
89
|
+
f.write('names:\n')
|
|
90
|
+
for i_class,class_name in enumerate(class_list):
|
|
91
|
+
f.write(' {}: {}\n'.format(i_class,class_name))
|
|
92
|
+
|
|
93
|
+
# ...def write_yolo_dataset_file(...)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def coco_to_yolo(input_image_folder,
|
|
97
|
+
output_folder,
|
|
98
|
+
input_file,
|
|
99
|
+
source_format='coco',
|
|
100
|
+
overwrite_images=False,
|
|
101
|
+
create_image_and_label_folders=False,
|
|
102
|
+
class_file_name='classes.txt',
|
|
103
|
+
allow_empty_annotations=False,
|
|
104
|
+
clip_boxes=False,
|
|
105
|
+
image_id_to_output_image_json_file=None,
|
|
106
|
+
images_to_exclude=None,
|
|
107
|
+
path_replacement_char='#',
|
|
108
|
+
category_names_to_exclude=None,
|
|
109
|
+
category_names_to_include=None,
|
|
110
|
+
write_output=True,
|
|
111
|
+
flatten_paths=False,
|
|
112
|
+
empty_image_handling='write_empty'):
|
|
113
|
+
"""
|
|
114
|
+
Converts a COCO-formatted dataset to a YOLO-formatted dataset, optionally flattening the
|
|
115
|
+
dataset to a single folder in the process.
|
|
116
|
+
|
|
117
|
+
If the input and output folders are the same, writes .txt files to the input folder,
|
|
118
|
+
and neither moves nor modifies images.
|
|
119
|
+
|
|
120
|
+
Currently ignores segmentation masks, and errors if an annotation has a
|
|
121
|
+
segmentation polygon but no bbox.
|
|
122
|
+
|
|
123
|
+
Args:
|
|
124
|
+
input_image_folder (str): the folder where images live; filenames in the COCO .json
|
|
125
|
+
file [input_file] should be relative to this folder
|
|
126
|
+
output_folder (str): the base folder for the YOLO dataset
|
|
127
|
+
input_file (str): a .json file in COCO format; can be the same as [input_image_folder], in which case
|
|
128
|
+
images are left alone.
|
|
129
|
+
source_format (str, optional): can be 'coco' (default) or 'coco_camera_traps'. The only difference
|
|
130
|
+
is that when source_format is 'coco_camera_traps', we treat an image with a non-bbox
|
|
131
|
+
annotation as a special case, i.e. that's how an empty image is indicated. The original
|
|
132
|
+
COCO standard is a little ambiguous on this issue. If source_format is 'coco', we
|
|
133
|
+
either treat images as empty or error, depending on the value of [allow_empty_annotations].
|
|
134
|
+
[allow_empty_annotations] has no effect if source_format is 'coco_camera_traps'.
|
|
135
|
+
overwrite_images (bool, optional): over-write images in the output folder if they exist
|
|
136
|
+
create_image_and_label_folders (bool, optional): whether to create separate folders called 'images' and
|
|
137
|
+
'labels' in the YOLO output folder. If create_image_and_label_folders is False,
|
|
138
|
+
a/b/c/image001.jpg will become a#b#c#image001.jpg, and the corresponding text file will
|
|
139
|
+
be a#b#c#image001.txt. If create_image_and_label_folders is True, a/b/c/image001.jpg will become
|
|
140
|
+
images/a#b#c#image001.jpg, and the corresponding text file will be
|
|
141
|
+
labels/a#b#c#image001.txt.
|
|
142
|
+
class_file_name (str, optional): .txt file (relative to the output folder) that we should
|
|
143
|
+
populate with a list of classes (or None to omit)
|
|
144
|
+
allow_empty_annotations (bool, optional): if this is False and [source_format] is 'coco',
|
|
145
|
+
we'll error on annotations that have no 'bbox' field
|
|
146
|
+
clip_boxes (bool, optional): whether to clip bounding box coordinates to the range [0,1] before
|
|
147
|
+
converting to YOLO xywh format
|
|
148
|
+
image_id_to_output_image_json_file (str, optional): an optional *output* file, to which we will write
|
|
149
|
+
a mapping from image IDs to output file names
|
|
150
|
+
images_to_exclude (list, optional): a list of image files (relative paths in the input folder) that we
|
|
151
|
+
should ignore
|
|
152
|
+
path_replacement_char (str, optional): only relevant if [flatten_paths] is True; this is used to replace
|
|
153
|
+
path separators, e.g. if [path_replacement_char] is '#' and [flatten_paths] is True, a/b/c/d.jpg
|
|
154
|
+
becomes a#b#c#d.jpg
|
|
155
|
+
category_names_to_exclude (str, optional): category names that should not be represented in the
|
|
156
|
+
YOLO output; only impacts annotations, does not prevent copying images. There's almost no reason
|
|
157
|
+
you would want to specify this and [category_names_to_include].
|
|
158
|
+
category_names_to_include (str, optional): allow-list of category names that should be represented
|
|
159
|
+
in the YOLO output; only impacts annotations, does not prevent copying images. There's almost
|
|
160
|
+
no reason you would want to specify this and [category_names_to_exclude].
|
|
161
|
+
write_output (bool, optional): determines whether we actually copy images and write annotations;
|
|
162
|
+
setting this to False mostly puts this function in "dry run" "mode. The class list
|
|
163
|
+
file is written regardless of the value of write_output.
|
|
164
|
+
flatten_paths (bool, optional): replace /'s in image filenames with [path_replacement_char],
|
|
165
|
+
which ensures that the output folder is a single flat folder.
|
|
166
|
+
empty_image_handling (str, optional): whether to omit .txt files for images with no
|
|
167
|
+
annotations ('omit') or write empty .txt files ('write_empty'). Both are generally considered
|
|
168
|
+
valid YOLO.
|
|
169
|
+
|
|
170
|
+
Returns:
|
|
171
|
+
dict: information about the coco --> yolo mapping, containing at least the fields:
|
|
172
|
+
|
|
173
|
+
- class_list_filename: the filename to which we wrote the flat list of class names required
|
|
174
|
+
by the YOLO format.
|
|
175
|
+
- source_image_to_dest_image: a dict mapping source images to destination images
|
|
176
|
+
- coco_id_to_yolo_id: a dict mapping COCO category IDs to YOLO category IDs
|
|
177
|
+
"""
|
|
178
|
+
|
|
179
|
+
## Validate input
|
|
180
|
+
|
|
181
|
+
assert empty_image_handling in ('omit','write_empty'), \
|
|
182
|
+
'Unrecognized value for empty_image_handling: {}'.format(empty_image_handling)
|
|
183
|
+
|
|
184
|
+
if category_names_to_include is not None and category_names_to_exclude is not None:
|
|
185
|
+
raise ValueError('category_names_to_include and category_names_to_exclude are mutually exclusive')
|
|
186
|
+
|
|
187
|
+
if output_folder is None:
|
|
188
|
+
output_folder = input_image_folder
|
|
189
|
+
|
|
190
|
+
if images_to_exclude is not None:
|
|
191
|
+
images_to_exclude = set(images_to_exclude)
|
|
192
|
+
|
|
193
|
+
if category_names_to_exclude is None:
|
|
194
|
+
category_names_to_exclude = {}
|
|
195
|
+
|
|
196
|
+
assert os.path.isdir(input_image_folder)
|
|
197
|
+
assert os.path.isfile(input_file)
|
|
198
|
+
os.makedirs(output_folder,exist_ok=True)
|
|
199
|
+
|
|
200
|
+
if (output_folder == input_image_folder) and (overwrite_images) and \
|
|
201
|
+
(not create_image_and_label_folders) and (not flatten_paths):
|
|
202
|
+
print('Warning: output folder and input folder are the same, disabling overwrite_images')
|
|
203
|
+
overwrite_images = False
|
|
204
|
+
|
|
205
|
+
## Read input data
|
|
206
|
+
|
|
207
|
+
with open(input_file,'r') as f:
|
|
208
|
+
data = json.load(f)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
## Parse annotations
|
|
212
|
+
|
|
213
|
+
image_id_to_annotations = defaultdict(list)
|
|
214
|
+
|
|
215
|
+
# i_ann = 0; ann = data['annotations'][0]
|
|
216
|
+
for i_ann,ann in enumerate(data['annotations']):
|
|
217
|
+
|
|
218
|
+
# Make sure no annotations have *only* segmentation data
|
|
219
|
+
if ( \
|
|
220
|
+
('segmentation' in ann.keys()) and \
|
|
221
|
+
(ann['segmentation'] is not None) and \
|
|
222
|
+
(len(ann['segmentation']) > 0) ) \
|
|
223
|
+
and \
|
|
224
|
+
(('bbox' not in ann.keys()) or (ann['bbox'] is None) or (len(ann['bbox'])==0)):
|
|
225
|
+
raise ValueError('Oops: segmentation data present without bbox information, ' + \
|
|
226
|
+
'this script isn\'t ready for this dataset')
|
|
227
|
+
|
|
228
|
+
image_id_to_annotations[ann['image_id']].append(ann)
|
|
229
|
+
|
|
230
|
+
print('Parsed annotations for {} images'.format(len(image_id_to_annotations)))
|
|
231
|
+
|
|
232
|
+
# Re-map class IDs to make sure they run from 0...n-classes-1
|
|
233
|
+
#
|
|
234
|
+
# Note: this allows unused categories in the output data set. This is OK for
|
|
235
|
+
# some training pipelines, not for others.
|
|
236
|
+
next_category_id = 0
|
|
237
|
+
coco_id_to_yolo_id = {}
|
|
238
|
+
coco_id_to_name = {}
|
|
239
|
+
yolo_id_to_name = {}
|
|
240
|
+
coco_category_ids_to_exclude = set()
|
|
241
|
+
|
|
242
|
+
for category in data['categories']:
|
|
243
|
+
coco_id_to_name[category['id']] = category['name']
|
|
244
|
+
if (category_names_to_include is not None) and \
|
|
245
|
+
(category['name'] not in category_names_to_include):
|
|
246
|
+
coco_category_ids_to_exclude.add(category['id'])
|
|
247
|
+
continue
|
|
248
|
+
elif (category['name'] in category_names_to_exclude):
|
|
249
|
+
coco_category_ids_to_exclude.add(category['id'])
|
|
250
|
+
continue
|
|
251
|
+
assert category['id'] not in coco_id_to_yolo_id
|
|
252
|
+
coco_id_to_yolo_id[category['id']] = next_category_id
|
|
253
|
+
yolo_id_to_name[next_category_id] = category['name']
|
|
254
|
+
next_category_id += 1
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
## Process images (everything but I/O)
|
|
258
|
+
|
|
259
|
+
# List of dictionaries with keys 'source_image','dest_image','bboxes','dest_txt'
|
|
260
|
+
images_to_copy = []
|
|
261
|
+
|
|
262
|
+
missing_images = []
|
|
263
|
+
excluded_images = []
|
|
264
|
+
|
|
265
|
+
image_names = set()
|
|
266
|
+
|
|
267
|
+
typical_image_extensions = set(['.jpg','.jpeg','.png','.gif','.tif','.bmp'])
|
|
268
|
+
|
|
269
|
+
printed_empty_annotation_warning = False
|
|
270
|
+
|
|
271
|
+
image_id_to_output_image_name = {}
|
|
272
|
+
|
|
273
|
+
print('Processing annotations')
|
|
274
|
+
|
|
275
|
+
n_clipped_boxes = 0
|
|
276
|
+
n_total_boxes = 0
|
|
277
|
+
|
|
278
|
+
# i_image = 0; im = data['images'][i_image]
|
|
279
|
+
for i_image,im in tqdm(enumerate(data['images']),total=len(data['images'])):
|
|
280
|
+
|
|
281
|
+
output_info = {}
|
|
282
|
+
source_image = os.path.join(input_image_folder,im['file_name'])
|
|
283
|
+
output_info['source_image'] = source_image
|
|
284
|
+
|
|
285
|
+
if images_to_exclude is not None and im['file_name'] in images_to_exclude:
|
|
286
|
+
excluded_images.append(im['file_name'])
|
|
287
|
+
continue
|
|
288
|
+
|
|
289
|
+
tokens = os.path.splitext(im['file_name'])
|
|
290
|
+
if tokens[1].lower() not in typical_image_extensions:
|
|
291
|
+
print('Warning: unusual image file name {}'.format(im['file_name']))
|
|
292
|
+
|
|
293
|
+
if flatten_paths:
|
|
294
|
+
image_name = tokens[0].replace('\\','/').replace('/',path_replacement_char) + \
|
|
295
|
+
'_' + str(i_image).zfill(6)
|
|
296
|
+
else:
|
|
297
|
+
image_name = tokens[0]
|
|
298
|
+
|
|
299
|
+
assert image_name not in image_names, 'Image name collision for {}'.format(image_name)
|
|
300
|
+
image_names.add(image_name)
|
|
301
|
+
|
|
302
|
+
assert im['id'] not in image_id_to_output_image_name
|
|
303
|
+
image_id_to_output_image_name[im['id']] = image_name
|
|
304
|
+
|
|
305
|
+
dest_image_relative = image_name + tokens[1]
|
|
306
|
+
output_info['dest_image_relative'] = dest_image_relative
|
|
307
|
+
dest_txt_relative = image_name + '.txt'
|
|
308
|
+
output_info['dest_txt_relative'] = dest_txt_relative
|
|
309
|
+
output_info['bboxes'] = []
|
|
310
|
+
|
|
311
|
+
# assert os.path.isfile(source_image), 'Could not find image {}'.format(source_image)
|
|
312
|
+
if not os.path.isfile(source_image):
|
|
313
|
+
print('Warning: could not find image {}'.format(source_image))
|
|
314
|
+
missing_images.append(im['file_name'])
|
|
315
|
+
continue
|
|
316
|
+
|
|
317
|
+
image_id = im['id']
|
|
318
|
+
|
|
319
|
+
image_bboxes = []
|
|
320
|
+
|
|
321
|
+
if image_id in image_id_to_annotations:
|
|
322
|
+
|
|
323
|
+
for ann in image_id_to_annotations[image_id]:
|
|
324
|
+
|
|
325
|
+
# If this annotation has no bounding boxes...
|
|
326
|
+
if 'bbox' not in ann or ann['bbox'] is None or len(ann['bbox']) == 0:
|
|
327
|
+
|
|
328
|
+
if source_format == 'coco':
|
|
329
|
+
|
|
330
|
+
if not allow_empty_annotations:
|
|
331
|
+
# This is not entirely clear from the COCO spec, but it seems to be consensus
|
|
332
|
+
# that if you want to specify an image with no objects, you don't include any
|
|
333
|
+
# annotations for that image.
|
|
334
|
+
raise ValueError('If an annotation exists, it should have content')
|
|
335
|
+
else:
|
|
336
|
+
continue
|
|
337
|
+
|
|
338
|
+
elif source_format == 'coco_camera_traps':
|
|
339
|
+
|
|
340
|
+
# We allow empty bbox lists in COCO camera traps files; this is typically a
|
|
341
|
+
# negative example in a dataset that has bounding boxes, and 0 is typically
|
|
342
|
+
# the empty category, which is typically 0.
|
|
343
|
+
if ann['category_id'] != 0:
|
|
344
|
+
if not printed_empty_annotation_warning:
|
|
345
|
+
printed_empty_annotation_warning = True
|
|
346
|
+
print('Warning: non-bbox annotation found with category {}'.format(
|
|
347
|
+
ann['category_id']))
|
|
348
|
+
continue
|
|
349
|
+
|
|
350
|
+
else:
|
|
351
|
+
|
|
352
|
+
raise ValueError('Unrecognized COCO variant: {}'.format(source_format))
|
|
353
|
+
|
|
354
|
+
# ...if this is an empty annotation
|
|
355
|
+
|
|
356
|
+
coco_bbox = ann['bbox']
|
|
357
|
+
|
|
358
|
+
# This category isn't in our category list. This typically corresponds to whole sets
|
|
359
|
+
# of images that were excluded from the YOLO set.
|
|
360
|
+
if ann['category_id'] in coco_category_ids_to_exclude:
|
|
361
|
+
continue
|
|
362
|
+
|
|
363
|
+
yolo_category_id = coco_id_to_yolo_id[ann['category_id']]
|
|
364
|
+
|
|
365
|
+
# COCO: [x_min, y_min, width, height] in absolute coordinates
|
|
366
|
+
# YOLO: [class, x_center, y_center, width, height] in normalized coordinates
|
|
367
|
+
|
|
368
|
+
# Convert from COCO coordinates to YOLO coordinates
|
|
369
|
+
img_w = im['width']
|
|
370
|
+
img_h = im['height']
|
|
371
|
+
|
|
372
|
+
if source_format in ('coco','coco_camera_traps'):
|
|
373
|
+
|
|
374
|
+
x_min_absolute = coco_bbox[0]
|
|
375
|
+
y_min_absolute = coco_bbox[1]
|
|
376
|
+
box_w_absolute = coco_bbox[2]
|
|
377
|
+
box_h_absolute = coco_bbox[3]
|
|
378
|
+
|
|
379
|
+
x_center_absolute = (x_min_absolute + (x_min_absolute + box_w_absolute)) / 2
|
|
380
|
+
y_center_absolute = (y_min_absolute + (y_min_absolute + box_h_absolute)) / 2
|
|
381
|
+
|
|
382
|
+
x_center_relative = x_center_absolute / img_w
|
|
383
|
+
y_center_relative = y_center_absolute / img_h
|
|
384
|
+
|
|
385
|
+
box_w_relative = box_w_absolute / img_w
|
|
386
|
+
box_h_relative = box_h_absolute / img_h
|
|
387
|
+
|
|
388
|
+
else:
|
|
389
|
+
|
|
390
|
+
raise ValueError('Unrecognized source format {}'.format(source_format))
|
|
391
|
+
|
|
392
|
+
if clip_boxes:
|
|
393
|
+
|
|
394
|
+
clipped_box = False
|
|
395
|
+
|
|
396
|
+
box_right = x_center_relative + (box_w_relative / 2.0)
|
|
397
|
+
if box_right > 1.0:
|
|
398
|
+
clipped_box = True
|
|
399
|
+
overhang = box_right - 1.0
|
|
400
|
+
box_w_relative -= overhang
|
|
401
|
+
x_center_relative -= (overhang / 2.0)
|
|
402
|
+
|
|
403
|
+
box_bottom = y_center_relative + (box_h_relative / 2.0)
|
|
404
|
+
if box_bottom > 1.0:
|
|
405
|
+
clipped_box = True
|
|
406
|
+
overhang = box_bottom - 1.0
|
|
407
|
+
box_h_relative -= overhang
|
|
408
|
+
y_center_relative -= (overhang / 2.0)
|
|
409
|
+
|
|
410
|
+
box_left = x_center_relative - (box_w_relative / 2.0)
|
|
411
|
+
if box_left < 0.0:
|
|
412
|
+
clipped_box = True
|
|
413
|
+
overhang = abs(box_left)
|
|
414
|
+
box_w_relative -= overhang
|
|
415
|
+
x_center_relative += (overhang / 2.0)
|
|
416
|
+
|
|
417
|
+
box_top = y_center_relative - (box_h_relative / 2.0)
|
|
418
|
+
if box_top < 0.0:
|
|
419
|
+
clipped_box = True
|
|
420
|
+
overhang = abs(box_top)
|
|
421
|
+
box_h_relative -= overhang
|
|
422
|
+
y_center_relative += (overhang / 2.0)
|
|
423
|
+
|
|
424
|
+
if clipped_box:
|
|
425
|
+
n_clipped_boxes += 1
|
|
426
|
+
|
|
427
|
+
yolo_box = [yolo_category_id,
|
|
428
|
+
x_center_relative, y_center_relative,
|
|
429
|
+
box_w_relative, box_h_relative]
|
|
430
|
+
|
|
431
|
+
image_bboxes.append(yolo_box)
|
|
432
|
+
n_total_boxes += 1
|
|
433
|
+
|
|
434
|
+
# ...for each annotation
|
|
435
|
+
|
|
436
|
+
# ...if this image has annotations
|
|
437
|
+
|
|
438
|
+
output_info['bboxes'] = image_bboxes
|
|
439
|
+
|
|
440
|
+
images_to_copy.append(output_info)
|
|
441
|
+
|
|
442
|
+
# ...for each image
|
|
443
|
+
|
|
444
|
+
print('\nWriting {} boxes ({} clipped) for {} images'.format(n_total_boxes,
|
|
445
|
+
n_clipped_boxes,len(images_to_copy)))
|
|
446
|
+
print('{} missing images (of {})'.format(len(missing_images),len(data['images'])))
|
|
447
|
+
|
|
448
|
+
if images_to_exclude is not None:
|
|
449
|
+
print('{} excluded images (of {})'.format(len(excluded_images),len(data['images'])))
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
## Write output
|
|
453
|
+
|
|
454
|
+
print('Generating class list')
|
|
455
|
+
|
|
456
|
+
if class_file_name is not None:
|
|
457
|
+
class_list_filename = os.path.join(output_folder,class_file_name)
|
|
458
|
+
with open(class_list_filename, 'w') as f:
|
|
459
|
+
print('Writing class list to {}'.format(class_list_filename))
|
|
460
|
+
for i_class in range(0,len(yolo_id_to_name)):
|
|
461
|
+
# Category IDs should range from 0..N-1
|
|
462
|
+
assert i_class in yolo_id_to_name
|
|
463
|
+
f.write(yolo_id_to_name[i_class] + '\n')
|
|
464
|
+
else:
|
|
465
|
+
class_list_filename = None
|
|
466
|
+
|
|
467
|
+
if image_id_to_output_image_json_file is not None:
|
|
468
|
+
print('Writing image ID mapping to {}'.format(image_id_to_output_image_json_file))
|
|
469
|
+
with open(image_id_to_output_image_json_file,'w') as f:
|
|
470
|
+
json.dump(image_id_to_output_image_name,f,indent=1)
|
|
471
|
+
|
|
472
|
+
if (output_folder == input_image_folder) and (not create_image_and_label_folders):
|
|
473
|
+
print('Creating annotation files (not copying images, input and output folder are the same)')
|
|
474
|
+
else:
|
|
475
|
+
print('Copying images and creating annotation files')
|
|
476
|
+
|
|
477
|
+
if create_image_and_label_folders:
|
|
478
|
+
dest_image_folder = os.path.join(output_folder,'images')
|
|
479
|
+
dest_txt_folder = os.path.join(output_folder,'labels')
|
|
480
|
+
else:
|
|
481
|
+
dest_image_folder = output_folder
|
|
482
|
+
dest_txt_folder = output_folder
|
|
483
|
+
|
|
484
|
+
source_image_to_dest_image = {}
|
|
485
|
+
|
|
486
|
+
label_files_written = []
|
|
487
|
+
n_boxes_written = 0
|
|
488
|
+
|
|
489
|
+
# TODO: parallelize this loop
|
|
490
|
+
#
|
|
491
|
+
# output_info = images_to_copy[0]
|
|
492
|
+
for output_info in tqdm(images_to_copy):
|
|
493
|
+
|
|
494
|
+
source_image = output_info['source_image']
|
|
495
|
+
dest_image_relative = output_info['dest_image_relative']
|
|
496
|
+
dest_txt_relative = output_info['dest_txt_relative']
|
|
497
|
+
|
|
498
|
+
dest_image = os.path.join(dest_image_folder,dest_image_relative)
|
|
499
|
+
dest_txt = os.path.join(dest_txt_folder,dest_txt_relative)
|
|
500
|
+
|
|
501
|
+
source_image_to_dest_image[source_image] = dest_image
|
|
502
|
+
|
|
503
|
+
# Copy the image if necessary
|
|
504
|
+
if write_output:
|
|
505
|
+
|
|
506
|
+
os.makedirs(os.path.dirname(dest_image),exist_ok=True)
|
|
507
|
+
os.makedirs(os.path.dirname(dest_txt),exist_ok=True)
|
|
508
|
+
|
|
509
|
+
if not create_image_and_label_folders:
|
|
510
|
+
assert os.path.dirname(dest_image) == os.path.dirname(dest_txt)
|
|
511
|
+
|
|
512
|
+
if (not os.path.isfile(dest_image)) or (overwrite_images):
|
|
513
|
+
shutil.copyfile(source_image,dest_image)
|
|
514
|
+
|
|
515
|
+
bboxes = output_info['bboxes']
|
|
516
|
+
|
|
517
|
+
# Write the annotation file if necessary
|
|
518
|
+
if (len(bboxes) > 0) or (empty_image_handling == 'write_empty'):
|
|
519
|
+
|
|
520
|
+
n_boxes_written += len(bboxes)
|
|
521
|
+
label_files_written.append(dest_txt)
|
|
522
|
+
|
|
523
|
+
if write_output:
|
|
524
|
+
|
|
525
|
+
with open(dest_txt,'w') as f:
|
|
526
|
+
|
|
527
|
+
# bbox = bboxes[0]
|
|
528
|
+
for bbox in bboxes:
|
|
529
|
+
assert len(bbox) == 5
|
|
530
|
+
s = '{} {} {} {} {}'.format(bbox[0],bbox[1],bbox[2],bbox[3],bbox[4])
|
|
531
|
+
f.write(s + '\n')
|
|
532
|
+
|
|
533
|
+
# ...if there are boxes for this image
|
|
534
|
+
|
|
535
|
+
# ...for each image
|
|
536
|
+
|
|
537
|
+
coco_to_yolo_info = {}
|
|
538
|
+
coco_to_yolo_info['class_list_filename'] = class_list_filename
|
|
539
|
+
coco_to_yolo_info['source_image_to_dest_image'] = source_image_to_dest_image
|
|
540
|
+
coco_to_yolo_info['coco_id_to_yolo_id'] = coco_id_to_yolo_id
|
|
541
|
+
coco_to_yolo_info['label_files_written'] = label_files_written
|
|
542
|
+
coco_to_yolo_info['n_boxes_written'] = n_boxes_written
|
|
543
|
+
|
|
544
|
+
return coco_to_yolo_info
|
|
545
|
+
|
|
546
|
+
# ...def coco_to_yolo(...)
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def create_yolo_symlinks(source_folder,
|
|
550
|
+
images_folder,
|
|
551
|
+
labels_folder,
|
|
552
|
+
class_list_file=None,
|
|
553
|
+
class_list_output_name='object.data',
|
|
554
|
+
force_lowercase_image_extension=False):
|
|
555
|
+
"""
|
|
556
|
+
Given a YOLO-formatted folder of images and .txt files, creates a folder
|
|
557
|
+
of symlinks to all the images, and a folder of symlinks to all the labels.
|
|
558
|
+
Used to support preview/editing tools that assume images and labels are in separate
|
|
559
|
+
folders.
|
|
560
|
+
|
|
561
|
+
Args:
|
|
562
|
+
source_folder (str): input folder
|
|
563
|
+
images_folder (str): output folder with links to images
|
|
564
|
+
labels_folder (str): output folder with links to labels
|
|
565
|
+
class_list_file (str, optional): list to classes.txt file
|
|
566
|
+
class_list_output_name (str, optional): output file to write with class information
|
|
567
|
+
force_lowercase_image_extension (bool, False): create symlinks with, e.g., .jpg, even
|
|
568
|
+
if the input image is, e.g., .JPG
|
|
569
|
+
|
|
570
|
+
:meta private:
|
|
571
|
+
"""
|
|
572
|
+
|
|
573
|
+
assert source_folder != images_folder and source_folder != labels_folder
|
|
574
|
+
|
|
575
|
+
os.makedirs(images_folder,exist_ok=True)
|
|
576
|
+
os.makedirs(labels_folder,exist_ok=True)
|
|
577
|
+
|
|
578
|
+
image_files_relative = find_images(source_folder,recursive=True,return_relative_paths=True)
|
|
579
|
+
|
|
580
|
+
# image_fn_relative = image_files_relative[0]=
|
|
581
|
+
for image_fn_relative in tqdm(image_files_relative):
|
|
582
|
+
|
|
583
|
+
source_file_abs = os.path.join(source_folder,image_fn_relative)
|
|
584
|
+
target_file_abs = os.path.join(images_folder,image_fn_relative)
|
|
585
|
+
|
|
586
|
+
if force_lowercase_image_extension:
|
|
587
|
+
tokens = os.path.splitext(target_file_abs)
|
|
588
|
+
target_file_abs = tokens[0] + tokens[1].lower()
|
|
589
|
+
|
|
590
|
+
os.makedirs(os.path.dirname(target_file_abs),exist_ok=True)
|
|
591
|
+
safe_create_link(source_file_abs,target_file_abs)
|
|
592
|
+
source_annotation_file_abs = os.path.splitext(source_file_abs)[0] + '.txt'
|
|
593
|
+
if os.path.isfile(source_annotation_file_abs):
|
|
594
|
+
target_annotation_file_abs = \
|
|
595
|
+
os.path.splitext(os.path.join(labels_folder,image_fn_relative))[0] + '.txt'
|
|
596
|
+
os.makedirs(os.path.dirname(target_annotation_file_abs),exist_ok=True)
|
|
597
|
+
safe_create_link(source_annotation_file_abs,target_annotation_file_abs)
|
|
598
|
+
|
|
599
|
+
# ...for each image
|
|
600
|
+
|
|
601
|
+
if class_list_file is not None:
|
|
602
|
+
target_class_list_file = os.path.join(labels_folder,class_list_output_name)
|
|
603
|
+
safe_create_link(class_list_file,target_class_list_file)
|
|
604
|
+
|
|
605
|
+
# ...def create_yolo_symlinks(...)
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
#%% Interactive driver
|
|
609
|
+
|
|
610
|
+
if False:
|
|
611
|
+
|
|
612
|
+
pass
|
|
613
|
+
|
|
614
|
+
#%% Options
|
|
615
|
+
|
|
616
|
+
input_file = os.path.expanduser('~/data/md-test-coco.json')
|
|
617
|
+
image_folder = os.path.expanduser('~/data/md-test')
|
|
618
|
+
output_folder = os.path.expanduser('~/data/md-test-yolo')
|
|
619
|
+
create_image_and_label_folders=False
|
|
620
|
+
class_file_name='classes.txt'
|
|
621
|
+
allow_empty_annotations=False
|
|
622
|
+
clip_boxes=False
|
|
623
|
+
image_id_to_output_image_json_file=None
|
|
624
|
+
images_to_exclude=None
|
|
625
|
+
path_replacement_char='#'
|
|
626
|
+
category_names_to_exclude=None
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
#%% Programmatic execution
|
|
630
|
+
|
|
631
|
+
coco_to_yolo_results = coco_to_yolo(image_folder,output_folder,input_file,
|
|
632
|
+
source_format='coco',
|
|
633
|
+
overwrite_images=False,
|
|
634
|
+
create_image_and_label_folders=create_image_and_label_folders,
|
|
635
|
+
class_file_name=class_file_name,
|
|
636
|
+
allow_empty_annotations=allow_empty_annotations,
|
|
637
|
+
clip_boxes=clip_boxes)
|
|
638
|
+
|
|
639
|
+
create_yolo_symlinks(source_folder=output_folder,
|
|
640
|
+
images_folder=output_folder + '/images',
|
|
641
|
+
labels_folder=output_folder + '/labels',
|
|
642
|
+
class_list_file=coco_to_yolo_results['class_list_filename'],
|
|
643
|
+
class_list_output_name='object.data',
|
|
644
|
+
force_lowercase_image_extension=True)
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
#%% Prepare command-line example
|
|
648
|
+
|
|
649
|
+
s = 'python coco_to_yolo.py {} {} {} --create_bounding_box_editor_symlinks'.format(
|
|
650
|
+
image_folder,output_folder,input_file)
|
|
651
|
+
print(s)
|
|
652
|
+
import clipboard; clipboard.copy(s)
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
#%% Command-line driver
|
|
656
|
+
|
|
657
|
+
def main(): # noqa
|
|
658
|
+
|
|
659
|
+
parser = argparse.ArgumentParser(
|
|
660
|
+
description='Convert COCO-formatted data to YOLO format, flattening the image structure')
|
|
661
|
+
|
|
662
|
+
# input_image_folder,output_folder,input_file
|
|
663
|
+
|
|
664
|
+
parser.add_argument(
|
|
665
|
+
'input_folder',
|
|
666
|
+
type=str,
|
|
667
|
+
help='Path to input images')
|
|
668
|
+
|
|
669
|
+
parser.add_argument(
|
|
670
|
+
'output_folder',
|
|
671
|
+
type=str,
|
|
672
|
+
help='Path to flat, YOLO-formatted dataset')
|
|
673
|
+
|
|
674
|
+
parser.add_argument(
|
|
675
|
+
'input_file',
|
|
676
|
+
type=str,
|
|
677
|
+
help='Path to COCO dataset file (.json)')
|
|
678
|
+
|
|
679
|
+
parser.add_argument(
|
|
680
|
+
'--create_bounding_box_editor_symlinks',
|
|
681
|
+
action='store_true',
|
|
682
|
+
help='Prepare symlinks so the whole folder appears to contain "images" and "labels" folderss')
|
|
683
|
+
|
|
684
|
+
if len(sys.argv[1:]) == 0:
|
|
685
|
+
parser.print_help()
|
|
686
|
+
parser.exit()
|
|
687
|
+
|
|
688
|
+
args = parser.parse_args()
|
|
689
|
+
|
|
690
|
+
coco_to_yolo_results = coco_to_yolo(args.input_folder,args.output_folder,args.input_file)
|
|
691
|
+
|
|
692
|
+
if args.create_bounding_box_editor_symlinks:
|
|
693
|
+
create_yolo_symlinks(source_folder=args.output_folder,
|
|
694
|
+
images_folder=args.output_folder + '/images',
|
|
695
|
+
labels_folder=args.output_folder + '/labels',
|
|
696
|
+
class_list_file=coco_to_yolo_results['class_list_filename'],
|
|
697
|
+
class_list_output_name='object.data',
|
|
698
|
+
force_lowercase_image_extension=True)
|
|
699
|
+
|
|
700
|
+
if __name__ == '__main__':
|
|
701
|
+
main()
|