megadetector 5.0.28__py3-none-any.whl → 10.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of megadetector might be problematic. Click here for more details.
- megadetector/api/batch_processing/integration/digiKam/xmp_integration.py +2 -2
- megadetector/api/batch_processing/integration/eMammal/test_scripts/push_annotations_to_emammal.py +1 -1
- megadetector/api/batch_processing/integration/eMammal/test_scripts/select_images_for_testing.py +1 -1
- megadetector/classification/aggregate_classifier_probs.py +3 -3
- megadetector/classification/analyze_failed_images.py +5 -5
- megadetector/classification/cache_batchapi_outputs.py +5 -5
- megadetector/classification/create_classification_dataset.py +11 -12
- megadetector/classification/crop_detections.py +10 -10
- megadetector/classification/csv_to_json.py +8 -8
- megadetector/classification/detect_and_crop.py +13 -15
- megadetector/classification/efficientnet/model.py +8 -8
- megadetector/classification/efficientnet/utils.py +6 -5
- megadetector/classification/evaluate_model.py +7 -7
- megadetector/classification/identify_mislabeled_candidates.py +6 -6
- megadetector/classification/json_to_azcopy_list.py +1 -1
- megadetector/classification/json_validator.py +29 -32
- megadetector/classification/map_classification_categories.py +9 -9
- megadetector/classification/merge_classification_detection_output.py +12 -9
- megadetector/classification/prepare_classification_script.py +19 -19
- megadetector/classification/prepare_classification_script_mc.py +26 -26
- megadetector/classification/run_classifier.py +4 -4
- megadetector/classification/save_mislabeled.py +6 -6
- megadetector/classification/train_classifier.py +1 -1
- megadetector/classification/train_classifier_tf.py +9 -9
- megadetector/classification/train_utils.py +10 -10
- megadetector/data_management/annotations/annotation_constants.py +1 -2
- megadetector/data_management/camtrap_dp_to_coco.py +79 -46
- megadetector/data_management/cct_json_utils.py +103 -103
- megadetector/data_management/cct_to_md.py +49 -49
- megadetector/data_management/cct_to_wi.py +33 -33
- megadetector/data_management/coco_to_labelme.py +75 -75
- megadetector/data_management/coco_to_yolo.py +210 -193
- megadetector/data_management/databases/add_width_and_height_to_db.py +86 -12
- megadetector/data_management/databases/combine_coco_camera_traps_files.py +40 -40
- megadetector/data_management/databases/integrity_check_json_db.py +228 -200
- megadetector/data_management/databases/subset_json_db.py +33 -33
- megadetector/data_management/generate_crops_from_cct.py +88 -39
- megadetector/data_management/get_image_sizes.py +54 -49
- megadetector/data_management/labelme_to_coco.py +133 -125
- megadetector/data_management/labelme_to_yolo.py +159 -73
- megadetector/data_management/lila/create_lila_blank_set.py +81 -83
- megadetector/data_management/lila/create_lila_test_set.py +32 -31
- megadetector/data_management/lila/create_links_to_md_results_files.py +18 -18
- megadetector/data_management/lila/download_lila_subset.py +21 -24
- megadetector/data_management/lila/generate_lila_per_image_labels.py +365 -107
- megadetector/data_management/lila/get_lila_annotation_counts.py +35 -33
- megadetector/data_management/lila/get_lila_image_counts.py +22 -22
- megadetector/data_management/lila/lila_common.py +73 -70
- megadetector/data_management/lila/test_lila_metadata_urls.py +28 -19
- megadetector/data_management/mewc_to_md.py +344 -340
- megadetector/data_management/ocr_tools.py +262 -255
- megadetector/data_management/read_exif.py +249 -227
- megadetector/data_management/remap_coco_categories.py +90 -28
- megadetector/data_management/remove_exif.py +81 -21
- megadetector/data_management/rename_images.py +187 -187
- megadetector/data_management/resize_coco_dataset.py +588 -120
- megadetector/data_management/speciesnet_to_md.py +41 -41
- megadetector/data_management/wi_download_csv_to_coco.py +55 -55
- megadetector/data_management/yolo_output_to_md_output.py +248 -122
- megadetector/data_management/yolo_to_coco.py +333 -191
- megadetector/detection/change_detection.py +832 -0
- megadetector/detection/process_video.py +340 -337
- megadetector/detection/pytorch_detector.py +358 -278
- megadetector/detection/run_detector.py +399 -186
- megadetector/detection/run_detector_batch.py +404 -377
- megadetector/detection/run_inference_with_yolov5_val.py +340 -327
- megadetector/detection/run_tiled_inference.py +257 -249
- megadetector/detection/tf_detector.py +24 -24
- megadetector/detection/video_utils.py +332 -295
- megadetector/postprocessing/add_max_conf.py +19 -11
- megadetector/postprocessing/categorize_detections_by_size.py +45 -45
- megadetector/postprocessing/classification_postprocessing.py +468 -433
- megadetector/postprocessing/combine_batch_outputs.py +23 -23
- megadetector/postprocessing/compare_batch_results.py +590 -525
- megadetector/postprocessing/convert_output_format.py +106 -102
- megadetector/postprocessing/create_crop_folder.py +347 -147
- megadetector/postprocessing/detector_calibration.py +173 -168
- megadetector/postprocessing/generate_csv_report.py +508 -499
- megadetector/postprocessing/load_api_results.py +48 -27
- megadetector/postprocessing/md_to_coco.py +133 -102
- megadetector/postprocessing/md_to_labelme.py +107 -90
- megadetector/postprocessing/md_to_wi.py +40 -40
- megadetector/postprocessing/merge_detections.py +92 -114
- megadetector/postprocessing/postprocess_batch_results.py +319 -301
- megadetector/postprocessing/remap_detection_categories.py +91 -38
- megadetector/postprocessing/render_detection_confusion_matrix.py +214 -205
- megadetector/postprocessing/repeat_detection_elimination/find_repeat_detections.py +57 -57
- megadetector/postprocessing/repeat_detection_elimination/remove_repeat_detections.py +27 -28
- megadetector/postprocessing/repeat_detection_elimination/repeat_detections_core.py +704 -679
- megadetector/postprocessing/separate_detections_into_folders.py +226 -211
- megadetector/postprocessing/subset_json_detector_output.py +265 -262
- megadetector/postprocessing/top_folders_to_bottom.py +45 -45
- megadetector/postprocessing/validate_batch_results.py +70 -70
- megadetector/taxonomy_mapping/map_lila_taxonomy_to_wi_taxonomy.py +52 -52
- megadetector/taxonomy_mapping/map_new_lila_datasets.py +18 -19
- megadetector/taxonomy_mapping/prepare_lila_taxonomy_release.py +54 -33
- megadetector/taxonomy_mapping/preview_lila_taxonomy.py +67 -67
- megadetector/taxonomy_mapping/retrieve_sample_image.py +16 -16
- megadetector/taxonomy_mapping/simple_image_download.py +8 -8
- megadetector/taxonomy_mapping/species_lookup.py +156 -74
- megadetector/taxonomy_mapping/taxonomy_csv_checker.py +14 -14
- megadetector/taxonomy_mapping/taxonomy_graph.py +10 -10
- megadetector/taxonomy_mapping/validate_lila_category_mappings.py +13 -13
- megadetector/utils/ct_utils.py +1049 -211
- megadetector/utils/directory_listing.py +21 -77
- megadetector/utils/gpu_test.py +22 -22
- megadetector/utils/md_tests.py +632 -529
- megadetector/utils/path_utils.py +1520 -431
- megadetector/utils/process_utils.py +41 -41
- megadetector/utils/split_locations_into_train_val.py +62 -62
- megadetector/utils/string_utils.py +148 -27
- megadetector/utils/url_utils.py +489 -176
- megadetector/utils/wi_utils.py +2658 -2526
- megadetector/utils/write_html_image_list.py +137 -137
- megadetector/visualization/plot_utils.py +34 -30
- megadetector/visualization/render_images_with_thumbnails.py +39 -74
- megadetector/visualization/visualization_utils.py +487 -435
- megadetector/visualization/visualize_db.py +232 -198
- megadetector/visualization/visualize_detector_output.py +82 -76
- {megadetector-5.0.28.dist-info → megadetector-10.0.0.dist-info}/METADATA +5 -2
- megadetector-10.0.0.dist-info/RECORD +139 -0
- {megadetector-5.0.28.dist-info → megadetector-10.0.0.dist-info}/WHEEL +1 -1
- megadetector/api/batch_processing/api_core/__init__.py +0 -0
- megadetector/api/batch_processing/api_core/batch_service/__init__.py +0 -0
- megadetector/api/batch_processing/api_core/batch_service/score.py +0 -439
- megadetector/api/batch_processing/api_core/server.py +0 -294
- megadetector/api/batch_processing/api_core/server_api_config.py +0 -97
- megadetector/api/batch_processing/api_core/server_app_config.py +0 -55
- megadetector/api/batch_processing/api_core/server_batch_job_manager.py +0 -220
- megadetector/api/batch_processing/api_core/server_job_status_table.py +0 -149
- megadetector/api/batch_processing/api_core/server_orchestration.py +0 -360
- megadetector/api/batch_processing/api_core/server_utils.py +0 -88
- megadetector/api/batch_processing/api_core_support/__init__.py +0 -0
- megadetector/api/batch_processing/api_core_support/aggregate_results_manually.py +0 -46
- megadetector/api/batch_processing/api_support/__init__.py +0 -0
- megadetector/api/batch_processing/api_support/summarize_daily_activity.py +0 -152
- megadetector/api/batch_processing/data_preparation/__init__.py +0 -0
- megadetector/api/synchronous/__init__.py +0 -0
- megadetector/api/synchronous/api_core/animal_detection_api/__init__.py +0 -0
- megadetector/api/synchronous/api_core/animal_detection_api/api_backend.py +0 -151
- megadetector/api/synchronous/api_core/animal_detection_api/api_frontend.py +0 -263
- megadetector/api/synchronous/api_core/animal_detection_api/config.py +0 -35
- megadetector/api/synchronous/api_core/tests/__init__.py +0 -0
- megadetector/api/synchronous/api_core/tests/load_test.py +0 -110
- megadetector/data_management/importers/add_nacti_sizes.py +0 -52
- megadetector/data_management/importers/add_timestamps_to_icct.py +0 -79
- megadetector/data_management/importers/animl_results_to_md_results.py +0 -158
- megadetector/data_management/importers/auckland_doc_test_to_json.py +0 -373
- megadetector/data_management/importers/auckland_doc_to_json.py +0 -201
- megadetector/data_management/importers/awc_to_json.py +0 -191
- megadetector/data_management/importers/bellevue_to_json.py +0 -272
- megadetector/data_management/importers/cacophony-thermal-importer.py +0 -793
- megadetector/data_management/importers/carrizo_shrubfree_2018.py +0 -269
- megadetector/data_management/importers/carrizo_trail_cam_2017.py +0 -289
- megadetector/data_management/importers/cct_field_adjustments.py +0 -58
- megadetector/data_management/importers/channel_islands_to_cct.py +0 -913
- megadetector/data_management/importers/eMammal/copy_and_unzip_emammal.py +0 -180
- megadetector/data_management/importers/eMammal/eMammal_helpers.py +0 -249
- megadetector/data_management/importers/eMammal/make_eMammal_json.py +0 -223
- megadetector/data_management/importers/ena24_to_json.py +0 -276
- megadetector/data_management/importers/filenames_to_json.py +0 -386
- megadetector/data_management/importers/helena_to_cct.py +0 -283
- megadetector/data_management/importers/idaho-camera-traps.py +0 -1407
- megadetector/data_management/importers/idfg_iwildcam_lila_prep.py +0 -294
- megadetector/data_management/importers/import_desert_lion_conservation_camera_traps.py +0 -387
- megadetector/data_management/importers/jb_csv_to_json.py +0 -150
- megadetector/data_management/importers/mcgill_to_json.py +0 -250
- megadetector/data_management/importers/missouri_to_json.py +0 -490
- megadetector/data_management/importers/nacti_fieldname_adjustments.py +0 -79
- megadetector/data_management/importers/noaa_seals_2019.py +0 -181
- megadetector/data_management/importers/osu-small-animals-to-json.py +0 -364
- megadetector/data_management/importers/pc_to_json.py +0 -365
- megadetector/data_management/importers/plot_wni_giraffes.py +0 -123
- megadetector/data_management/importers/prepare_zsl_imerit.py +0 -131
- megadetector/data_management/importers/raic_csv_to_md_results.py +0 -416
- megadetector/data_management/importers/rspb_to_json.py +0 -356
- megadetector/data_management/importers/save_the_elephants_survey_A.py +0 -320
- megadetector/data_management/importers/save_the_elephants_survey_B.py +0 -329
- megadetector/data_management/importers/snapshot_safari_importer.py +0 -758
- megadetector/data_management/importers/snapshot_serengeti_lila.py +0 -1067
- megadetector/data_management/importers/snapshotserengeti/make_full_SS_json.py +0 -150
- megadetector/data_management/importers/snapshotserengeti/make_per_season_SS_json.py +0 -153
- megadetector/data_management/importers/sulross_get_exif.py +0 -65
- megadetector/data_management/importers/timelapse_csv_set_to_json.py +0 -490
- megadetector/data_management/importers/ubc_to_json.py +0 -399
- megadetector/data_management/importers/umn_to_json.py +0 -507
- megadetector/data_management/importers/wellington_to_json.py +0 -263
- megadetector/data_management/importers/wi_to_json.py +0 -442
- megadetector/data_management/importers/zamba_results_to_md_results.py +0 -180
- megadetector/data_management/lila/add_locations_to_island_camera_traps.py +0 -101
- megadetector/data_management/lila/add_locations_to_nacti.py +0 -151
- megadetector/utils/azure_utils.py +0 -178
- megadetector/utils/sas_blob_utils.py +0 -509
- megadetector-5.0.28.dist-info/RECORD +0 -209
- /megadetector/{api/batch_processing/__init__.py → __init__.py} +0 -0
- {megadetector-5.0.28.dist-info → megadetector-10.0.0.dist-info}/licenses/LICENSE +0 -0
- {megadetector-5.0.28.dist-info → megadetector-10.0.0.dist-info}/top_level.txt +0 -0
|
@@ -5,30 +5,30 @@ run_inference_with_yolov5_val.py
|
|
|
5
5
|
Runs a folder of images through MegaDetector (or another YOLOv5/YOLOv8 model) with YOLO's
|
|
6
6
|
val.py, converting the output to the standard MD format. The reasons this script exists,
|
|
7
7
|
as an alternative to the standard run_detector_batch.py are:
|
|
8
|
-
|
|
8
|
+
|
|
9
9
|
* This script provides access to YOLO's test-time augmentation tools.
|
|
10
10
|
* This script serves a reference implementation: by any reasonable definition, YOLOv5's
|
|
11
|
-
val.py produces the "correct" result for any image, since it matches what was used in
|
|
11
|
+
val.py produces the "correct" result for any image, since it matches what was used in
|
|
12
12
|
training.
|
|
13
|
-
* This script works for any Ultralytics detection model, including YOLOv8 models
|
|
13
|
+
* This script works for any Ultralytics detection model, including YOLOv8 models
|
|
14
14
|
|
|
15
|
-
YOLOv5's val.py uses each file's base name as a unique identifier, which doesn't work
|
|
15
|
+
YOLOv5's val.py uses each file's base name as a unique identifier, which doesn't work
|
|
16
16
|
when you have typical camera trap images like:
|
|
17
17
|
|
|
18
18
|
* a/b/c/RECONYX0001.JPG
|
|
19
19
|
* d/e/f/RECONYX0001.JPG
|
|
20
20
|
|
|
21
|
-
...both of which would just be "RECONYX0001.JPG". So this script jumps through a bunch of
|
|
22
|
-
hoops to put a symlinks in a flat folder, run YOLOv5 on that folder, and map the results back
|
|
21
|
+
...both of which would just be "RECONYX0001.JPG". So this script jumps through a bunch of
|
|
22
|
+
hoops to put a symlinks in a flat folder, run YOLOv5 on that folder, and map the results back
|
|
23
23
|
to the real files.
|
|
24
24
|
|
|
25
25
|
If you are running a YOLOv5 model, this script currently requires the caller to supply the path
|
|
26
|
-
where a working YOLOv5 install lives, and assumes that the current conda environment is all set up for
|
|
26
|
+
where a working YOLOv5 install lives, and assumes that the current conda environment is all set up for
|
|
27
27
|
YOLOv5. If you are running a YOLOv8 model, the folder doesn't matter, but it assumes that ultralytics
|
|
28
28
|
tools are available in the current environment.
|
|
29
29
|
|
|
30
|
-
By default, this script uses symlinks to format the input images in a way that YOLO's
|
|
31
|
-
val.py likes, as per above. This requires admin privileges on Windows... actually technically this
|
|
30
|
+
By default, this script uses symlinks to format the input images in a way that YOLO's
|
|
31
|
+
val.py likes, as per above. This requires admin privileges on Windows... actually technically this
|
|
32
32
|
only requires permissions to create symbolic links, but I've never seen a case where someone has
|
|
33
33
|
that permission and *doesn't* have admin privileges. If you are running this script on
|
|
34
34
|
Windows and you don't have admin privileges, use --no_use_symlinks, which will make copies of images,
|
|
@@ -46,14 +46,17 @@ import tempfile
|
|
|
46
46
|
import shutil
|
|
47
47
|
import json
|
|
48
48
|
import copy
|
|
49
|
+
import argparse
|
|
49
50
|
|
|
50
51
|
from tqdm import tqdm
|
|
51
52
|
|
|
52
53
|
from megadetector.utils import path_utils
|
|
53
54
|
from megadetector.utils import process_utils
|
|
54
55
|
from megadetector.utils import string_utils
|
|
56
|
+
from megadetector.utils.ct_utils import args_to_object
|
|
55
57
|
|
|
56
58
|
from megadetector.utils.ct_utils import is_iterable, split_list_into_fixed_size_chunks
|
|
59
|
+
from megadetector.utils import ct_utils
|
|
57
60
|
from megadetector.utils.path_utils import path_is_abs
|
|
58
61
|
from megadetector.data_management import yolo_output_to_md_output
|
|
59
62
|
from megadetector.detection.run_detector import try_download_known_detector
|
|
@@ -67,129 +70,133 @@ default_image_size_with_no_augmentation = 1280
|
|
|
67
70
|
|
|
68
71
|
class YoloInferenceOptions:
|
|
69
72
|
"""
|
|
70
|
-
Parameters that control the behavior of run_inference_with_yolov5_val(), including
|
|
73
|
+
Parameters that control the behavior of run_inference_with_yolov5_val(), including
|
|
71
74
|
the input/output filenames.
|
|
72
75
|
"""
|
|
73
|
-
|
|
76
|
+
|
|
74
77
|
def __init__(self):
|
|
75
|
-
|
|
78
|
+
|
|
76
79
|
## Required-ish ##
|
|
77
|
-
|
|
80
|
+
|
|
78
81
|
#: Folder of images to process (can be None if image_filename_list contains absolute paths)
|
|
79
82
|
self.input_folder = None
|
|
80
|
-
|
|
83
|
+
|
|
81
84
|
#: If this is None, [input_folder] can't be None, we'll process all images in [input_folder].
|
|
82
|
-
#:
|
|
83
|
-
#: If this is not None, and [input_folder] is not None, this should be a list of relative image
|
|
84
|
-
#: paths within [input_folder] to process, or a .txt or .json file containing a list of
|
|
85
|
+
#:
|
|
86
|
+
#: If this is not None, and [input_folder] is not None, this should be a list of relative image
|
|
87
|
+
#: paths within [input_folder] to process, or a .txt or .json file containing a list of
|
|
85
88
|
#: relative image paths.
|
|
86
89
|
#:
|
|
87
90
|
#: If this is not None, and [input_folder] is None, this should be a list of absolute image
|
|
88
|
-
#: paths, or a .txt or .json file containing a list of absolute image paths.
|
|
91
|
+
#: paths, or a .txt or .json file containing a list of absolute image paths.
|
|
89
92
|
self.image_filename_list = None
|
|
90
|
-
|
|
93
|
+
|
|
91
94
|
#: Model filename (ending in .pt), or a well-known model name (e.g. "MDV5A")
|
|
92
95
|
self.model_filename = None
|
|
93
|
-
|
|
96
|
+
|
|
94
97
|
#: .json output file, in MD results format
|
|
95
98
|
self.output_file = None
|
|
96
|
-
|
|
97
|
-
|
|
99
|
+
|
|
100
|
+
|
|
98
101
|
## Optional ##
|
|
99
|
-
|
|
102
|
+
|
|
100
103
|
#: Required for older YOLOv5 inference, not for newer ulytralytics/YOLOv8 inference
|
|
101
104
|
self.yolo_working_folder = None
|
|
102
|
-
|
|
105
|
+
|
|
103
106
|
#: Currently 'yolov5' and 'ultralytics' are supported, and really these are proxies for
|
|
104
107
|
#: "the yolov5 repo" and "the ultralytics repo".
|
|
105
|
-
self.model_type = 'yolov5'
|
|
106
|
-
|
|
108
|
+
self.model_type = 'yolov5'
|
|
109
|
+
|
|
107
110
|
#: Image size to use; this is a single int, which in ultralytics's terminology means
|
|
108
111
|
#: "scale the long side of the image to this size, and preserve aspect ratio".
|
|
109
112
|
#:
|
|
110
113
|
#: If None, will choose based on whether augmentation is enabled.
|
|
111
114
|
self.image_size = None
|
|
112
|
-
|
|
115
|
+
|
|
113
116
|
#: Detections below this threshold will not be included in the output file
|
|
114
117
|
self.conf_thres = '0.001'
|
|
115
|
-
|
|
118
|
+
|
|
116
119
|
#: Batch size... has no impact on results, but may create memory issues if you set
|
|
117
120
|
#: this to large values
|
|
118
121
|
self.batch_size = 1
|
|
119
|
-
|
|
122
|
+
|
|
120
123
|
#: Device string: typically '0' for GPU 0, '1' for GPU 1, etc., or 'cpu'
|
|
121
124
|
self.device_string = '0'
|
|
122
|
-
|
|
125
|
+
|
|
123
126
|
#: Should we enable test-time augmentation?
|
|
124
127
|
self.augment = False
|
|
125
|
-
|
|
128
|
+
|
|
126
129
|
#: Should we enable half-precision inference?
|
|
127
130
|
self.half_precision_enabled = None
|
|
128
|
-
|
|
129
|
-
#: Where should we stash the temporary symlinks (or copies) used to give unique identifiers to image
|
|
131
|
+
|
|
132
|
+
#: Where should we stash the temporary symlinks (or copies) used to give unique identifiers to image
|
|
130
133
|
# files?
|
|
131
134
|
#:
|
|
132
135
|
#: If this is None, we'll create a folder in system temp space.
|
|
133
136
|
self.symlink_folder = None
|
|
134
|
-
|
|
137
|
+
|
|
135
138
|
#: Should we use symlinks to give unique identifiers to image files (vs. copies)?
|
|
136
139
|
self.use_symlinks = True
|
|
137
|
-
|
|
140
|
+
|
|
138
141
|
#: How should we guarantee that YOLO IDs (base filenames) are unique? Choices are:
|
|
139
142
|
#:
|
|
140
143
|
#: * 'verify': assume image IDs are unique, but verify and error if they're not
|
|
141
144
|
#: * 'links': create symlinks (or copies, depending on use_symlinks) to enforce uniqueness
|
|
142
145
|
#: * 'auto': check whether IDs are unique, create links if necessary
|
|
143
146
|
self.unique_id_strategy = 'links'
|
|
144
|
-
|
|
147
|
+
|
|
145
148
|
#: Temporary folder to stash intermediate YOLO results.
|
|
146
149
|
#:
|
|
147
|
-
#: If this is None, we'll create a folder in system temp space.
|
|
150
|
+
#: If this is None, we'll create a folder in system temp space.
|
|
148
151
|
self.yolo_results_folder = None
|
|
149
|
-
|
|
152
|
+
|
|
150
153
|
#: Should we remove the symlink folder when we're done?
|
|
151
154
|
self.remove_symlink_folder = True
|
|
152
|
-
|
|
155
|
+
|
|
153
156
|
#: Should we remove the intermediate results folder when we're done?
|
|
154
157
|
self.remove_yolo_results_folder = True
|
|
155
|
-
|
|
158
|
+
|
|
156
159
|
#: These are deliberately offset from the standard MD categories; YOLOv5
|
|
157
160
|
#: needs categories IDs to start at 0.
|
|
158
161
|
#:
|
|
159
162
|
#: This can also be a string that points to a YOLO dataset.yaml file.
|
|
160
163
|
self.yolo_category_id_to_name = {0:'animal',1:'person',2:'vehicle'}
|
|
161
|
-
|
|
164
|
+
|
|
162
165
|
#: What should we do if the output file already exists?
|
|
163
166
|
#:
|
|
164
167
|
#: Can be 'error', 'skip', or 'overwrite'.
|
|
165
168
|
self.overwrite_handling = 'skip'
|
|
166
|
-
|
|
169
|
+
|
|
167
170
|
#: If True, we'll do a dry run that lets you preview the YOLO val command, without
|
|
168
171
|
#: actually running it.
|
|
169
172
|
self.preview_yolo_command_only = False
|
|
170
|
-
|
|
173
|
+
|
|
171
174
|
#: By default, if any errors occur while we're copying images or creating symlinks, it's
|
|
172
175
|
#: game over. If this is True, those errors become warnings, and we plow ahead.
|
|
173
176
|
self.treat_copy_failures_as_warnings = False
|
|
174
|
-
|
|
177
|
+
|
|
175
178
|
#: Save YOLO console output
|
|
176
179
|
self.save_yolo_debug_output = False
|
|
177
|
-
|
|
180
|
+
|
|
178
181
|
#: Whether to search for images recursively within [input_folder]
|
|
179
182
|
#:
|
|
180
183
|
#: Ignored if a list of files is provided.
|
|
181
184
|
self.recursive = True
|
|
182
|
-
|
|
185
|
+
|
|
183
186
|
#: Maximum number of images to run in a single chunk
|
|
184
187
|
self.checkpoint_frequency = None
|
|
185
|
-
|
|
186
|
-
#: By default, if we're creating symlinks to images, we append a unique job ID to the
|
|
188
|
+
|
|
189
|
+
#: By default, if we're creating symlinks to images, we append a unique job ID to the
|
|
187
190
|
#: symlink folder. If the caller is 100% sure that the symlink folder can be re-used
|
|
188
191
|
#: across calls, this can be set to False.
|
|
189
192
|
self.append_job_id_to_symlink_folder = True
|
|
190
|
-
|
|
193
|
+
|
|
194
|
+
#: By default, we turn category ID 0 coming out of the YOLO .json file
|
|
195
|
+
#: into category 1 in the MD-formatted .json file.
|
|
196
|
+
self.offset_yolo_category_ids = True
|
|
197
|
+
|
|
191
198
|
# ...def __init__()
|
|
192
|
-
|
|
199
|
+
|
|
193
200
|
# ...YoloInferenceOptions()
|
|
194
201
|
|
|
195
202
|
|
|
@@ -201,13 +208,13 @@ def _clean_up_temporary_folders(options,
|
|
|
201
208
|
"""
|
|
202
209
|
Remove temporary symlink/results folders, unless the caller requested that we leave them in place.
|
|
203
210
|
"""
|
|
204
|
-
|
|
211
|
+
|
|
205
212
|
if options.remove_symlink_folder:
|
|
206
213
|
shutil.rmtree(symlink_folder)
|
|
207
214
|
elif symlink_folder_is_temp_folder:
|
|
208
215
|
print('Warning: using temporary symlink folder {}, but not removing it'.format(
|
|
209
216
|
symlink_folder))
|
|
210
|
-
|
|
217
|
+
|
|
211
218
|
if options.remove_yolo_results_folder:
|
|
212
219
|
shutil.rmtree(yolo_results_folder)
|
|
213
220
|
elif yolo_folder_is_temp_folder:
|
|
@@ -219,44 +226,44 @@ def get_stats_for_category(filename,category='all'):
|
|
|
219
226
|
"""
|
|
220
227
|
Retrieve statistics for a category from the YOLO console output
|
|
221
228
|
stored in [filenam].
|
|
222
|
-
|
|
229
|
+
|
|
223
230
|
Args:
|
|
224
231
|
filename (str): a text file containing console output from a YOLO val run
|
|
225
|
-
category (
|
|
226
|
-
|
|
232
|
+
category (str, optional): a category name
|
|
233
|
+
|
|
227
234
|
Returns:
|
|
228
235
|
dict: a dict with fields n_images, n_labels, P, R, mAP50, and mAP50-95
|
|
229
236
|
"""
|
|
230
|
-
|
|
237
|
+
|
|
231
238
|
with open(filename,'r',encoding='utf-8') as f:
|
|
232
239
|
lines = f.readlines()
|
|
233
|
-
|
|
240
|
+
|
|
234
241
|
# This is just a hedge to make sure there isn't some YOLO version floating
|
|
235
242
|
# around that used different IoU thresholds in the console output.
|
|
236
243
|
found_map50 = False
|
|
237
244
|
found_map5095 = False
|
|
238
|
-
|
|
245
|
+
|
|
239
246
|
for line in lines:
|
|
240
|
-
|
|
247
|
+
|
|
241
248
|
s = line.strip()
|
|
242
|
-
|
|
249
|
+
|
|
243
250
|
if ' map50 ' in s.lower() or ' map@.5 ' in s.lower():
|
|
244
251
|
found_map50 = True
|
|
245
252
|
if 'map50-95' in s.lower() or 'map@.5:.95' in s.lower():
|
|
246
253
|
found_map5095 = True
|
|
247
|
-
|
|
254
|
+
|
|
248
255
|
if not s.startswith(category):
|
|
249
256
|
continue
|
|
250
|
-
|
|
257
|
+
|
|
251
258
|
tokens = s.split(' ')
|
|
252
259
|
tokens_filtered = list(filter(None,tokens))
|
|
253
|
-
|
|
260
|
+
|
|
254
261
|
if len(tokens_filtered) != 7:
|
|
255
262
|
continue
|
|
256
|
-
|
|
263
|
+
|
|
257
264
|
assert found_map50 and found_map5095, \
|
|
258
265
|
'Parsing error in YOLO console output file {}'.format(filename)
|
|
259
|
-
|
|
266
|
+
|
|
260
267
|
to_return = {}
|
|
261
268
|
to_return['category'] = category
|
|
262
269
|
assert category == tokens_filtered[0]
|
|
@@ -267,68 +274,70 @@ def get_stats_for_category(filename,category='all'):
|
|
|
267
274
|
to_return['mAP50'] = float(tokens_filtered[5])
|
|
268
275
|
to_return['mAP50-95'] = float(tokens_filtered[6])
|
|
269
276
|
return to_return
|
|
270
|
-
|
|
277
|
+
|
|
271
278
|
# ...for each line
|
|
272
|
-
|
|
279
|
+
|
|
273
280
|
return None
|
|
274
281
|
|
|
275
|
-
|
|
282
|
+
|
|
276
283
|
#%% Main function
|
|
277
284
|
|
|
278
285
|
def run_inference_with_yolo_val(options):
|
|
279
286
|
"""
|
|
280
287
|
Runs a folder of images through MegaDetector (or another YOLOv5/YOLOv8 model) with YOLO's
|
|
281
288
|
val.py, converting the output to the standard MD format.
|
|
282
|
-
|
|
283
|
-
Args:
|
|
289
|
+
|
|
290
|
+
Args:
|
|
284
291
|
options (YoloInferenceOptions): all the parameters used to control this process,
|
|
285
|
-
including filenames; see YoloInferenceOptions for details
|
|
292
|
+
including filenames; see YoloInferenceOptions for details
|
|
286
293
|
"""
|
|
287
|
-
|
|
294
|
+
|
|
288
295
|
##%% Input and path handling
|
|
289
|
-
|
|
296
|
+
|
|
290
297
|
default_options = YoloInferenceOptions()
|
|
291
|
-
|
|
298
|
+
|
|
292
299
|
for k in options.__dict__.keys():
|
|
293
300
|
if k not in default_options.__dict__:
|
|
294
301
|
# Print warnings about unexpected variables, except for things like
|
|
295
302
|
# "no_append_job_id_to_symlink_folder", which just negate existing objects
|
|
296
303
|
if not k.startswith('no_'):
|
|
297
304
|
print('Warning: unexpected variable {} in options object'.format(k))
|
|
298
|
-
|
|
305
|
+
|
|
299
306
|
if options.model_type == 'yolov8':
|
|
300
|
-
|
|
301
|
-
print('Warning: model type "yolov8" supplied, "ultralytics" is the preferred model
|
|
307
|
+
|
|
308
|
+
print('Warning: model type "yolov8" supplied, "ultralytics" is the preferred model ' + \
|
|
309
|
+
'type string for YOLOv8 models')
|
|
302
310
|
options.model_type = 'ultralytics'
|
|
303
|
-
|
|
311
|
+
|
|
304
312
|
if (options.model_type == 'yolov5') and ('yolov8' in options.model_filename.lower()):
|
|
305
|
-
print('\n\n*** Warning: model type set as "yolov5", but your model filename contains "yolov8"...
|
|
306
|
-
|
|
313
|
+
print('\n\n*** Warning: model type set as "yolov5", but your model filename contains "yolov8"... ' + \
|
|
314
|
+
'did you mean to use --model_type yolov8?" ***\n\n')
|
|
315
|
+
|
|
307
316
|
if options.yolo_working_folder is None:
|
|
308
317
|
assert options.model_type == 'ultralytics', \
|
|
309
318
|
'A working folder is required to run YOLOv5 val.py'
|
|
310
319
|
else:
|
|
311
320
|
assert os.path.isdir(options.yolo_working_folder), \
|
|
312
321
|
'Could not find working folder {}'.format(options.yolo_working_folder)
|
|
313
|
-
|
|
322
|
+
|
|
314
323
|
if options.half_precision_enabled is not None:
|
|
315
324
|
assert options.half_precision_enabled in (0,1), \
|
|
316
325
|
'Invalid value {} for --half_precision_enabled (should be 0 or 1)'.format(
|
|
317
326
|
options.half_precision_enabled)
|
|
318
|
-
|
|
327
|
+
|
|
319
328
|
# If the model filename is a known model string (e.g. "MDv5A", download the model if necessary)
|
|
320
329
|
model_filename = try_download_known_detector(options.model_filename)
|
|
321
|
-
|
|
330
|
+
|
|
322
331
|
assert os.path.isfile(model_filename), \
|
|
323
332
|
'Could not find model file {}'.format(model_filename)
|
|
324
|
-
|
|
333
|
+
|
|
325
334
|
assert (options.input_folder is not None) or (options.image_filename_list is not None), \
|
|
326
335
|
'You must specify a folder and/or a file list'
|
|
327
|
-
|
|
336
|
+
|
|
328
337
|
if options.input_folder is not None:
|
|
329
338
|
assert os.path.isdir(options.input_folder), 'Could not find input folder {}'.format(
|
|
330
339
|
options.input_folder)
|
|
331
|
-
|
|
340
|
+
|
|
332
341
|
if os.path.exists(options.output_file):
|
|
333
342
|
if options.overwrite_handling == 'skip':
|
|
334
343
|
print('Warning: output file {} exists, skipping'.format(options.output_file))
|
|
@@ -339,17 +348,17 @@ def run_inference_with_yolo_val(options):
|
|
|
339
348
|
raise ValueError('Output file {} exists'.format(options.output_file))
|
|
340
349
|
else:
|
|
341
350
|
raise ValueError('Unknown output handling method {}'.format(options.overwrite_handling))
|
|
342
|
-
|
|
351
|
+
|
|
343
352
|
os.makedirs(os.path.dirname(options.output_file),exist_ok=True)
|
|
344
|
-
|
|
353
|
+
|
|
345
354
|
if options.input_folder is not None:
|
|
346
355
|
options.input_folder = options.input_folder.replace('\\','/')
|
|
347
|
-
|
|
348
|
-
|
|
356
|
+
|
|
357
|
+
|
|
349
358
|
##%% Other input handling
|
|
350
|
-
|
|
359
|
+
|
|
351
360
|
if isinstance(options.yolo_category_id_to_name,str):
|
|
352
|
-
|
|
361
|
+
|
|
353
362
|
assert os.path.isfile(options.yolo_category_id_to_name)
|
|
354
363
|
yolo_dataset_file = options.yolo_category_id_to_name
|
|
355
364
|
options.yolo_category_id_to_name = \
|
|
@@ -360,9 +369,9 @@ def run_inference_with_yolo_val(options):
|
|
|
360
369
|
temporary_folder = None
|
|
361
370
|
symlink_folder_is_temp_folder = False
|
|
362
371
|
yolo_folder_is_temp_folder = False
|
|
363
|
-
|
|
372
|
+
|
|
364
373
|
job_id = str(uuid.uuid1())
|
|
365
|
-
|
|
374
|
+
|
|
366
375
|
def get_job_temporary_folder(tf):
|
|
367
376
|
if tf is not None:
|
|
368
377
|
return tf
|
|
@@ -370,39 +379,39 @@ def run_inference_with_yolo_val(options):
|
|
|
370
379
|
tf = os.path.join(tempdir_base,'md_to_yolo','md_to_yolo_' + job_id)
|
|
371
380
|
os.makedirs(tf,exist_ok=True)
|
|
372
381
|
return tf
|
|
373
|
-
|
|
382
|
+
|
|
374
383
|
symlink_folder = options.symlink_folder
|
|
375
384
|
yolo_results_folder = options.yolo_results_folder
|
|
376
|
-
|
|
385
|
+
|
|
377
386
|
if symlink_folder is None:
|
|
378
387
|
temporary_folder = get_job_temporary_folder(temporary_folder)
|
|
379
388
|
symlink_folder = os.path.join(temporary_folder,'symlinks')
|
|
380
389
|
symlink_folder_is_temp_folder = True
|
|
381
|
-
|
|
390
|
+
|
|
382
391
|
if yolo_results_folder is None:
|
|
383
392
|
temporary_folder = get_job_temporary_folder(temporary_folder)
|
|
384
393
|
yolo_results_folder = os.path.join(temporary_folder,'yolo_results')
|
|
385
394
|
yolo_folder_is_temp_folder = True
|
|
386
|
-
|
|
395
|
+
|
|
387
396
|
if options.append_job_id_to_symlink_folder:
|
|
388
|
-
# Attach a GUID to the symlink folder, regardless of whether we created it
|
|
397
|
+
# Attach a GUID to the symlink folder, regardless of whether we created it
|
|
389
398
|
symlink_folder_inner = os.path.join(symlink_folder,job_id)
|
|
390
399
|
else:
|
|
391
400
|
print('Re-using existing symlink folder {}'.format(symlink_folder))
|
|
392
401
|
symlink_folder_inner = symlink_folder
|
|
393
|
-
|
|
402
|
+
|
|
394
403
|
os.makedirs(symlink_folder_inner,exist_ok=True)
|
|
395
404
|
os.makedirs(yolo_results_folder,exist_ok=True)
|
|
396
|
-
|
|
405
|
+
|
|
397
406
|
|
|
398
407
|
##%% Enumerate images
|
|
399
|
-
|
|
408
|
+
|
|
400
409
|
image_files_relative = None
|
|
401
410
|
image_files_absolute = None
|
|
402
|
-
|
|
411
|
+
|
|
403
412
|
# If the caller just provided a folder, not a list of files...
|
|
404
413
|
if options.image_filename_list is None:
|
|
405
|
-
|
|
414
|
+
|
|
406
415
|
assert options.input_folder is not None and os.path.isdir(options.input_folder), \
|
|
407
416
|
'Could not find input folder {}'.format(options.input_folder)
|
|
408
417
|
image_files_relative = path_utils.find_images(options.input_folder,
|
|
@@ -411,18 +420,18 @@ def run_inference_with_yolo_val(options):
|
|
|
411
420
|
convert_slashes=True)
|
|
412
421
|
image_files_absolute = [os.path.join(options.input_folder,fn) for \
|
|
413
422
|
fn in image_files_relative]
|
|
414
|
-
|
|
423
|
+
|
|
415
424
|
else:
|
|
416
|
-
|
|
417
|
-
# If the caller provided a list of image files (rather than a filename pointing
|
|
425
|
+
|
|
426
|
+
# If the caller provided a list of image files (rather than a filename pointing
|
|
418
427
|
# to a list of image files)...
|
|
419
428
|
if is_iterable(options.image_filename_list) and not isinstance(options.image_filename_list,str):
|
|
420
|
-
|
|
429
|
+
|
|
421
430
|
image_files_relative = options.image_filename_list
|
|
422
|
-
|
|
431
|
+
|
|
423
432
|
# If the caller provided a filename pointing to a list of image files...
|
|
424
433
|
else:
|
|
425
|
-
|
|
434
|
+
|
|
426
435
|
assert isinstance(options.image_filename_list,str), \
|
|
427
436
|
'Unrecognized image filename list object type: {}'.format(options.image_filename_list)
|
|
428
437
|
assert os.path.isfile(options.image_filename_list), \
|
|
@@ -439,152 +448,152 @@ def run_inference_with_yolo_val(options):
|
|
|
439
448
|
with open(options.image_filename_list,'r') as f:
|
|
440
449
|
image_files_relative = f.readlines()
|
|
441
450
|
image_files_relative = [s.strip() for s in image_files_relative]
|
|
442
|
-
|
|
451
|
+
|
|
443
452
|
# ...whether the image filename list was supplied as list vs. a filename
|
|
444
|
-
|
|
453
|
+
|
|
445
454
|
if options.input_folder is None:
|
|
446
|
-
|
|
455
|
+
|
|
447
456
|
image_files_absolute = image_files_relative
|
|
448
|
-
|
|
457
|
+
|
|
449
458
|
else:
|
|
450
|
-
|
|
459
|
+
|
|
451
460
|
# The list should be relative filenames
|
|
452
461
|
for fn in image_files_relative:
|
|
453
462
|
assert not path_is_abs(fn), \
|
|
454
463
|
'When providing a folder and a list, paths in the list should be relative'
|
|
455
|
-
|
|
464
|
+
|
|
456
465
|
image_files_absolute = \
|
|
457
466
|
[os.path.join(options.input_folder,fn) for fn in image_files_relative]
|
|
458
|
-
|
|
467
|
+
|
|
459
468
|
for fn in image_files_absolute:
|
|
460
469
|
assert os.path.isfile(fn), 'Could not find image file {}'.format(fn)
|
|
461
|
-
|
|
470
|
+
|
|
462
471
|
# ...whether the caller supplied a list of filenames
|
|
463
|
-
|
|
472
|
+
|
|
464
473
|
image_files_absolute = [fn.replace('\\','/') for fn in image_files_absolute]
|
|
465
|
-
|
|
474
|
+
|
|
466
475
|
del image_files_relative
|
|
467
|
-
|
|
468
|
-
|
|
476
|
+
|
|
477
|
+
|
|
469
478
|
##%% Recurse if necessary to handle checkpoints
|
|
470
|
-
|
|
479
|
+
|
|
471
480
|
if options.checkpoint_frequency is not None and options.checkpoint_frequency > 0:
|
|
472
|
-
|
|
481
|
+
|
|
473
482
|
chunks = split_list_into_fixed_size_chunks(image_files_absolute,options.checkpoint_frequency)
|
|
474
|
-
|
|
483
|
+
|
|
475
484
|
chunk_output_files = []
|
|
476
|
-
|
|
485
|
+
|
|
477
486
|
# i_chunk = 0; chunk_files_abs = chunks[i_chunk]
|
|
478
487
|
for i_chunk,chunk_files_abs in enumerate(chunks):
|
|
479
|
-
|
|
488
|
+
|
|
480
489
|
print('Processing {} images from chunk {} of {}'.format(
|
|
481
490
|
len(chunk_files_abs),i_chunk,len(chunks)))
|
|
482
|
-
|
|
491
|
+
|
|
483
492
|
chunk_options = copy.deepcopy(options)
|
|
484
|
-
|
|
493
|
+
|
|
485
494
|
# Run each chunk without checkpointing
|
|
486
495
|
chunk_options.checkpoint_frequency = None
|
|
487
|
-
|
|
496
|
+
|
|
488
497
|
if options.input_folder is not None:
|
|
489
498
|
chunk_files_relative = \
|
|
490
499
|
[os.path.relpath(fn,options.input_folder) for fn in chunk_files_abs]
|
|
491
500
|
chunk_options.image_filename_list = chunk_files_relative
|
|
492
501
|
else:
|
|
493
502
|
chunk_options.image_filename_list = chunk_files_abs
|
|
494
|
-
|
|
503
|
+
|
|
495
504
|
chunk_options.image_filename_list = \
|
|
496
505
|
[fn.replace('\\','/') for fn in chunk_options.image_filename_list]
|
|
497
|
-
|
|
506
|
+
|
|
498
507
|
chunk_string = 'chunk_{}'.format(str(i_chunk).zfill(5))
|
|
499
508
|
chunk_options.yolo_results_folder = yolo_results_folder + '_' + chunk_string
|
|
500
509
|
chunk_options.symlink_folder = symlink_folder + '_' + chunk_string
|
|
501
|
-
|
|
510
|
+
|
|
502
511
|
# Put the output file in the parent job's scratch folder
|
|
503
512
|
chunk_output_file = os.path.join(yolo_results_folder,chunk_string + '_results_md_format.json')
|
|
504
513
|
chunk_output_files.append(chunk_output_file)
|
|
505
514
|
chunk_options.output_file = chunk_output_file
|
|
506
|
-
|
|
515
|
+
|
|
507
516
|
if os.path.isfile(chunk_output_file):
|
|
508
|
-
|
|
517
|
+
|
|
509
518
|
print('Chunk output file {} exists, checking completeness'.format(chunk_output_file))
|
|
510
|
-
|
|
519
|
+
|
|
511
520
|
with open(chunk_output_file,'r') as f:
|
|
512
521
|
chunk_results = json.load(f)
|
|
513
|
-
images_in_this_chunk_results_file = [im['file'] for im in chunk_results['images']]
|
|
522
|
+
images_in_this_chunk_results_file = [im['file'] for im in chunk_results['images']]
|
|
514
523
|
assert len(images_in_this_chunk_results_file) == len(chunk_options.image_filename_list), \
|
|
515
|
-
'Expected {} images in
|
|
516
|
-
|
|
517
|
-
|
|
524
|
+
f'Expected {len(chunk_options.image_filename_list)} images in ' + \
|
|
525
|
+
f'chunk results file {chunk_output_file}, found {len(images_in_this_chunk_results_file)}, ' + \
|
|
526
|
+
'possibly this is left over from a previous job?'
|
|
518
527
|
for fn in images_in_this_chunk_results_file:
|
|
519
528
|
assert fn in chunk_options.image_filename_list, \
|
|
520
|
-
'Unexpected image {} in chunk results file {},
|
|
521
|
-
|
|
522
|
-
|
|
529
|
+
f'Unexpected image {fn} in chunk results file {chunk_output_file}, ' + \
|
|
530
|
+
'possibly this is left over from a previous job?'
|
|
531
|
+
|
|
523
532
|
print('Chunk output file {} exists and is complete, skipping this chunk'.format(
|
|
524
533
|
chunk_output_file))
|
|
525
|
-
|
|
534
|
+
|
|
526
535
|
# ...if the outptut file exists
|
|
527
|
-
|
|
536
|
+
|
|
528
537
|
else:
|
|
529
|
-
|
|
538
|
+
|
|
530
539
|
run_inference_with_yolo_val(chunk_options)
|
|
531
|
-
|
|
540
|
+
|
|
532
541
|
# ...if we do/don't have to run this chunk
|
|
533
|
-
|
|
542
|
+
|
|
534
543
|
assert os.path.isfile(chunk_options.output_file)
|
|
535
|
-
|
|
544
|
+
|
|
536
545
|
# ...for each chunk
|
|
537
|
-
|
|
546
|
+
|
|
538
547
|
# Merge
|
|
539
548
|
_ = combine_batch_output_files(input_files=chunk_output_files,
|
|
540
549
|
output_file=options.output_file,
|
|
541
550
|
require_uniqueness=True,
|
|
542
551
|
verbose=True)
|
|
543
|
-
|
|
552
|
+
|
|
544
553
|
# Validate
|
|
545
554
|
with open(options.output_file,'r') as f:
|
|
546
555
|
combined_results = json.load(f)
|
|
547
556
|
assert len(combined_results['images']) == len(image_files_absolute), \
|
|
548
557
|
'Expected {} images in merged output file, found {}'.format(
|
|
549
558
|
len(image_files_absolute),len(combined_results['images']))
|
|
550
|
-
|
|
559
|
+
|
|
551
560
|
# Clean up
|
|
552
561
|
_clean_up_temporary_folders(options,
|
|
553
562
|
symlink_folder,yolo_results_folder,
|
|
554
563
|
symlink_folder_is_temp_folder,yolo_folder_is_temp_folder)
|
|
555
|
-
|
|
564
|
+
|
|
556
565
|
return
|
|
557
|
-
|
|
566
|
+
|
|
558
567
|
# ...if we need to make recursive calls for file chunks
|
|
559
|
-
|
|
560
|
-
|
|
568
|
+
|
|
569
|
+
|
|
561
570
|
##%% Create symlinks (or copy images) to give a unique ID to each image
|
|
562
|
-
|
|
571
|
+
|
|
563
572
|
# Maps YOLO image IDs (base filename without extension as it will appear in YOLO .json output)
|
|
564
573
|
# to the *original full path* for each image (not the symlink path).
|
|
565
|
-
image_id_to_file = {}
|
|
566
|
-
|
|
574
|
+
image_id_to_file = {}
|
|
575
|
+
|
|
567
576
|
# Maps YOLO image IDs (base filename without extension as it will appear in YOLO .json output)
|
|
568
577
|
# to errors, including errors that happen before we run the model at all (e.g. file access errors).
|
|
569
578
|
image_id_to_error = {}
|
|
570
|
-
|
|
579
|
+
|
|
571
580
|
create_links = True
|
|
572
|
-
|
|
581
|
+
|
|
573
582
|
if options.unique_id_strategy == 'links':
|
|
574
|
-
|
|
583
|
+
|
|
575
584
|
create_links = True
|
|
576
|
-
|
|
585
|
+
|
|
577
586
|
else:
|
|
578
|
-
|
|
587
|
+
|
|
579
588
|
assert options.unique_id_strategy in ('auto','verify'), \
|
|
580
589
|
'Unknown unique ID strategy {}'.format(options.unique_id_strategy)
|
|
581
|
-
|
|
590
|
+
|
|
582
591
|
image_ids_are_unique = True
|
|
583
|
-
|
|
584
|
-
for i_image,image_fn in tqdm(enumerate(image_files_absolute),total=len(image_files_absolute)):
|
|
585
|
-
|
|
592
|
+
|
|
593
|
+
for i_image,image_fn in tqdm(enumerate(image_files_absolute),total=len(image_files_absolute)):
|
|
594
|
+
|
|
586
595
|
image_id = os.path.splitext(os.path.basename(image_fn))[0]
|
|
587
|
-
|
|
596
|
+
|
|
588
597
|
# Is this image ID unique?
|
|
589
598
|
if image_id in image_id_to_file:
|
|
590
599
|
if options.unique_id_strategy == 'verify':
|
|
@@ -596,20 +605,20 @@ def run_inference_with_yolo_val(options):
|
|
|
596
605
|
image_ids_are_unique = False
|
|
597
606
|
image_id_to_file = {}
|
|
598
607
|
break
|
|
599
|
-
|
|
608
|
+
|
|
600
609
|
image_id_to_file[image_id] = image_fn
|
|
601
|
-
|
|
610
|
+
|
|
602
611
|
# ...for each image
|
|
603
|
-
|
|
612
|
+
|
|
604
613
|
if image_ids_are_unique:
|
|
605
|
-
|
|
614
|
+
|
|
606
615
|
print('"{}" specified for image uniqueness and images are unique, skipping links'.format(
|
|
607
616
|
options.unique_id_strategy))
|
|
608
617
|
assert len(image_id_to_file) == len(image_files_absolute)
|
|
609
618
|
create_links = False
|
|
610
|
-
|
|
619
|
+
|
|
611
620
|
else:
|
|
612
|
-
|
|
621
|
+
|
|
613
622
|
assert options.unique_id_strategy == 'auto'
|
|
614
623
|
create_links = True
|
|
615
624
|
link_type = 'copies'
|
|
@@ -617,31 +626,31 @@ def run_inference_with_yolo_val(options):
|
|
|
617
626
|
link_type = 'links'
|
|
618
627
|
print('"auto" specified for image uniqueness and images are not unique, defaulting to {}'.format(
|
|
619
628
|
link_type))
|
|
620
|
-
|
|
629
|
+
|
|
621
630
|
# ...which unique ID strategy?
|
|
622
|
-
|
|
631
|
+
|
|
623
632
|
if create_links:
|
|
624
|
-
|
|
633
|
+
|
|
625
634
|
if options.use_symlinks:
|
|
626
635
|
print('Creating {} symlinks in {}'.format(len(image_files_absolute),symlink_folder_inner))
|
|
627
636
|
else:
|
|
628
637
|
print('Symlinks disabled, copying {} images to {}'.format(len(image_files_absolute),symlink_folder_inner))
|
|
629
|
-
|
|
638
|
+
|
|
630
639
|
link_full_paths = []
|
|
631
|
-
|
|
640
|
+
|
|
632
641
|
# i_image = 0; image_fn = image_files_absolute[i_image]
|
|
633
642
|
for i_image,image_fn in tqdm(enumerate(image_files_absolute),total=len(image_files_absolute)):
|
|
634
|
-
|
|
643
|
+
|
|
635
644
|
ext = os.path.splitext(image_fn)[1]
|
|
636
645
|
image_fn_without_extension = os.path.splitext(image_fn)[0]
|
|
637
|
-
|
|
646
|
+
|
|
638
647
|
# YOLO .json output identifies images by the base filename without the extension
|
|
639
648
|
image_id = str(i_image).zfill(10)
|
|
640
649
|
image_id_to_file[image_id] = image_fn
|
|
641
650
|
symlink_name = image_id + ext
|
|
642
651
|
symlink_full_path = os.path.join(symlink_folder_inner,symlink_name)
|
|
643
652
|
link_full_paths.append(symlink_full_path)
|
|
644
|
-
|
|
653
|
+
|
|
645
654
|
# If annotation files exist, link those too; only useful if we're reading the computed
|
|
646
655
|
# mAP value, but it doesn't hurt.
|
|
647
656
|
annotation_fn = image_fn_without_extension + '.txt'
|
|
@@ -649,10 +658,10 @@ def run_inference_with_yolo_val(options):
|
|
|
649
658
|
if os.path.isfile(annotation_fn):
|
|
650
659
|
annotation_file_exists = True
|
|
651
660
|
annotation_symlink_name = image_id + '.txt'
|
|
652
|
-
annotation_symlink_full_path = os.path.join(symlink_folder_inner,annotation_symlink_name)
|
|
653
|
-
|
|
661
|
+
annotation_symlink_full_path = os.path.join(symlink_folder_inner,annotation_symlink_name)
|
|
662
|
+
|
|
654
663
|
try:
|
|
655
|
-
|
|
664
|
+
|
|
656
665
|
if options.use_symlinks:
|
|
657
666
|
path_utils.safe_create_link(image_fn,symlink_full_path)
|
|
658
667
|
if annotation_file_exists:
|
|
@@ -661,74 +670,74 @@ def run_inference_with_yolo_val(options):
|
|
|
661
670
|
shutil.copyfile(image_fn,symlink_full_path)
|
|
662
671
|
if annotation_file_exists:
|
|
663
672
|
shutil.copyfile(annotation_fn,annotation_symlink_full_path)
|
|
664
|
-
|
|
673
|
+
|
|
665
674
|
except Exception as e:
|
|
666
|
-
|
|
675
|
+
|
|
667
676
|
error_string = str(e)
|
|
668
677
|
image_id_to_error[image_id] = error_string
|
|
669
|
-
|
|
678
|
+
|
|
670
679
|
# Always break if the user is trying to create symlinks on Windows without
|
|
671
680
|
# permission, 100% of images will always fail in this case.
|
|
672
681
|
if ('a required privilege is not held by the client' in error_string.lower()) or \
|
|
673
682
|
(not options.treat_copy_failures_as_warnings):
|
|
674
|
-
|
|
683
|
+
|
|
675
684
|
print('\nError copying/creating link for input file {}: {}'.format(
|
|
676
685
|
image_fn,error_string))
|
|
677
|
-
|
|
686
|
+
|
|
678
687
|
raise
|
|
679
|
-
|
|
688
|
+
|
|
680
689
|
else:
|
|
681
|
-
|
|
690
|
+
|
|
682
691
|
print('Warning: error copying/creating link for input file {}: {}'.format(
|
|
683
692
|
image_fn,error_string))
|
|
684
693
|
continue
|
|
685
|
-
|
|
694
|
+
|
|
686
695
|
# ...except
|
|
687
|
-
|
|
696
|
+
|
|
688
697
|
# ...for each image
|
|
689
|
-
|
|
698
|
+
|
|
690
699
|
# ...if we need to create links/copies
|
|
691
700
|
|
|
692
|
-
|
|
701
|
+
|
|
693
702
|
##%% Create the dataset file if necessary
|
|
694
|
-
|
|
703
|
+
|
|
695
704
|
# This may have been passed in as a string, but at this point, we should have
|
|
696
705
|
# loaded the dataset file.
|
|
697
706
|
assert isinstance(options.yolo_category_id_to_name,dict)
|
|
698
|
-
|
|
707
|
+
|
|
699
708
|
# Category IDs need to be continuous integers starting at 0
|
|
700
709
|
category_ids = sorted(list(options.yolo_category_id_to_name.keys()))
|
|
701
710
|
assert category_ids[0] == 0
|
|
702
711
|
assert len(category_ids) == 1 + category_ids[-1]
|
|
703
|
-
|
|
712
|
+
|
|
704
713
|
yolo_dataset_file = os.path.join(yolo_results_folder,'dataset.yaml')
|
|
705
|
-
yolo_image_list_file = os.path.join(yolo_results_folder,'images.txt')
|
|
706
|
-
|
|
714
|
+
yolo_image_list_file = os.path.join(yolo_results_folder,'images.txt')
|
|
715
|
+
|
|
707
716
|
with open(yolo_image_list_file,'w') as f:
|
|
708
|
-
|
|
717
|
+
|
|
709
718
|
if create_links:
|
|
710
719
|
image_files_to_write = link_full_paths
|
|
711
720
|
else:
|
|
712
721
|
image_files_to_write = image_files_absolute
|
|
713
|
-
|
|
722
|
+
|
|
714
723
|
for fn_abs in image_files_to_write:
|
|
715
|
-
# At least in YOLOv5 val (need to verify for YOLOv8 val), filenames in this
|
|
724
|
+
# At least in YOLOv5 val (need to verify for YOLOv8 val), filenames in this
|
|
716
725
|
# text file are treated as relative to the text file itself if they start with
|
|
717
726
|
# "./", otherwise they're treated as absolute paths. Since we don't want to put this
|
|
718
727
|
# text file in the image folder, we'll use absolute paths.
|
|
719
728
|
# fn_relative = os.path.relpath(fn_abs,options.input_folder)
|
|
720
729
|
# f.write(fn_relative + '\n')
|
|
721
730
|
f.write(fn_abs + '\n')
|
|
722
|
-
|
|
731
|
+
|
|
723
732
|
if create_links:
|
|
724
733
|
inference_folder = symlink_folder_inner
|
|
725
734
|
else:
|
|
726
735
|
# This doesn't matter, but it has to be a valid path
|
|
727
736
|
inference_folder = options.yolo_results_folder
|
|
728
|
-
|
|
737
|
+
|
|
729
738
|
with open(yolo_dataset_file,'w') as f:
|
|
730
|
-
|
|
731
|
-
f.write('path: {}\n'.format(inference_folder))
|
|
739
|
+
|
|
740
|
+
f.write('path: {}\n'.format(inference_folder))
|
|
732
741
|
# These need to be valid paths, even if you're not using them, and "." is always safe
|
|
733
742
|
f.write('train: .\n')
|
|
734
743
|
f.write('val: .\n')
|
|
@@ -744,7 +753,7 @@ def run_inference_with_yolo_val(options):
|
|
|
744
753
|
|
|
745
754
|
|
|
746
755
|
##%% Prepare Python command or YOLO CLI command
|
|
747
|
-
|
|
756
|
+
|
|
748
757
|
if options.image_size is None:
|
|
749
758
|
if options.augment:
|
|
750
759
|
image_size = default_image_size_with_augmentation
|
|
@@ -752,70 +761,70 @@ def run_inference_with_yolo_val(options):
|
|
|
752
761
|
image_size = default_image_size_with_no_augmentation
|
|
753
762
|
else:
|
|
754
763
|
image_size = options.image_size
|
|
755
|
-
|
|
764
|
+
|
|
756
765
|
image_size_string = str(round(image_size))
|
|
757
|
-
|
|
766
|
+
|
|
758
767
|
if options.model_type == 'yolov5':
|
|
759
|
-
|
|
768
|
+
|
|
760
769
|
cmd = 'python val.py --task test --data "{}"'.format(yolo_dataset_file)
|
|
761
770
|
cmd += ' --weights "{}"'.format(model_filename)
|
|
762
771
|
cmd += ' --batch-size {} --imgsz {} --conf-thres {}'.format(
|
|
763
772
|
options.batch_size,image_size_string,options.conf_thres)
|
|
764
773
|
cmd += ' --device "{}" --save-json'.format(options.device_string)
|
|
765
774
|
cmd += ' --project "{}" --name "{}" --exist-ok'.format(yolo_results_folder,'yolo_results')
|
|
766
|
-
|
|
775
|
+
|
|
767
776
|
# This is the NMS IoU threshold
|
|
768
777
|
# cmd += ' --iou-thres 0.6'
|
|
769
|
-
|
|
778
|
+
|
|
770
779
|
if options.augment:
|
|
771
780
|
cmd += ' --augment'
|
|
772
|
-
|
|
781
|
+
|
|
773
782
|
# --half is a store_true argument for YOLOv5's val.py
|
|
774
783
|
if (options.half_precision_enabled is not None) and (options.half_precision_enabled == 1):
|
|
775
784
|
cmd += ' --half'
|
|
776
|
-
|
|
785
|
+
|
|
777
786
|
# Sometimes useful for debugging
|
|
778
787
|
# cmd += ' --save_conf --save_txt'
|
|
779
|
-
|
|
788
|
+
|
|
780
789
|
elif options.model_type == 'ultralytics':
|
|
781
|
-
|
|
790
|
+
|
|
782
791
|
if options.augment:
|
|
783
792
|
augment_string = 'augment'
|
|
784
793
|
else:
|
|
785
794
|
augment_string = ''
|
|
786
|
-
|
|
795
|
+
|
|
787
796
|
cmd = 'yolo val {} model="{}" imgsz={} batch={} data="{}" project="{}" name="{}" device="{}"'.\
|
|
788
797
|
format(augment_string,model_filename,image_size_string,options.batch_size,
|
|
789
798
|
yolo_dataset_file,yolo_results_folder,'yolo_results',options.device_string)
|
|
790
799
|
cmd += ' save_json exist_ok'
|
|
791
|
-
|
|
800
|
+
|
|
792
801
|
if (options.half_precision_enabled is not None):
|
|
793
802
|
if options.half_precision_enabled == 1:
|
|
794
803
|
cmd += ' --half=True'
|
|
795
804
|
else:
|
|
796
805
|
assert options.half_precision_enabled == 0
|
|
797
806
|
cmd += ' --half=False'
|
|
798
|
-
|
|
807
|
+
|
|
799
808
|
# Sometimes useful for debugging
|
|
800
809
|
# cmd += ' save_conf save_txt'
|
|
801
|
-
|
|
810
|
+
|
|
802
811
|
else:
|
|
803
|
-
|
|
812
|
+
|
|
804
813
|
raise ValueError('Unrecognized model type {}'.format(options.model_type))
|
|
805
|
-
|
|
814
|
+
|
|
806
815
|
# print(cmd); import clipboard; clipboard.copy(cmd)
|
|
807
816
|
|
|
808
|
-
|
|
817
|
+
|
|
809
818
|
##%% Run YOLO command
|
|
810
|
-
|
|
819
|
+
|
|
811
820
|
if options.yolo_working_folder is not None:
|
|
812
821
|
current_dir = os.getcwd()
|
|
813
822
|
os.chdir(options.yolo_working_folder)
|
|
814
823
|
|
|
815
824
|
print('Running YOLO inference command:\n{}\n'.format(cmd))
|
|
816
|
-
|
|
825
|
+
|
|
817
826
|
if options.preview_yolo_command_only:
|
|
818
|
-
|
|
827
|
+
|
|
819
828
|
if options.remove_symlink_folder:
|
|
820
829
|
try:
|
|
821
830
|
print('Removing YOLO symlink folder {}'.format(symlink_folder))
|
|
@@ -830,34 +839,32 @@ def run_inference_with_yolo_val(options):
|
|
|
830
839
|
except Exception:
|
|
831
840
|
print('Warning: error removing YOLO results folder {}'.format(yolo_results_folder))
|
|
832
841
|
pass
|
|
833
|
-
|
|
842
|
+
|
|
834
843
|
# sys.exit()
|
|
835
844
|
return
|
|
836
|
-
|
|
845
|
+
|
|
837
846
|
execution_result = process_utils.execute_and_print(cmd,encoding='utf-8',verbose=True)
|
|
838
847
|
assert execution_result['status'] == 0, 'Error running {}'.format(options.model_type)
|
|
839
848
|
yolo_console_output = execution_result['output']
|
|
840
|
-
|
|
849
|
+
|
|
841
850
|
if options.save_yolo_debug_output:
|
|
842
|
-
|
|
851
|
+
|
|
843
852
|
with open(os.path.join(yolo_results_folder,'yolo_console_output.txt'),'w',encoding='utf-8') as f:
|
|
844
853
|
for s in yolo_console_output:
|
|
845
854
|
f.write(s + '\n')
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
855
|
+
ct_utils.write_json(os.path.join(yolo_results_folder,'image_id_to_file.json'), image_id_to_file)
|
|
856
|
+
ct_utils.write_json(os.path.join(yolo_results_folder,'image_id_to_error.json'), image_id_to_error)
|
|
857
|
+
|
|
858
|
+
|
|
852
859
|
# YOLO console output contains lots of ANSI escape codes, remove them for easier parsing
|
|
853
860
|
yolo_console_output = [string_utils.remove_ansi_codes(s) for s in yolo_console_output]
|
|
854
|
-
|
|
861
|
+
|
|
855
862
|
# Find errors that occurred during the initial corruption check; these will not be included in the
|
|
856
863
|
# output. Errors that occur during inference will be handled separately.
|
|
857
864
|
yolo_read_failures = []
|
|
858
|
-
|
|
865
|
+
|
|
859
866
|
for line in yolo_console_output:
|
|
860
|
-
|
|
867
|
+
|
|
861
868
|
#
|
|
862
869
|
# Lines indicating read failures look like:
|
|
863
870
|
#
|
|
@@ -869,30 +876,30 @@ def run_inference_with_yolo_val(options):
|
|
|
869
876
|
#
|
|
870
877
|
# line = "test: WARNING: a/b/c/d.jpg: ignoring corrupt image/label: cannot identify image file '/a/b/c/d.jpg'"
|
|
871
878
|
#
|
|
872
|
-
# In both cases, when we are using symlinks, the first filename is the symlink name, the
|
|
879
|
+
# In both cases, when we are using symlinks, the first filename is the symlink name, the
|
|
873
880
|
# second filename is the target, e.g.:
|
|
874
|
-
#
|
|
881
|
+
#
|
|
875
882
|
# line = "test: WARNING: /tmp/md_to_yolo/md_to_yolo_xyz/symlinks/xyz/0000000004.jpg: ignoring corrupt image/label: cannot identify image file '/tmp/md-tests/md-test-images/corrupt-images/real-file.jpg'"
|
|
876
883
|
#
|
|
877
884
|
# Windows example:
|
|
878
885
|
#
|
|
879
886
|
# line = "test: WARNING: g:\\temp\\md-test-images\\corrupt-images\\irfanview-can-still-read-me-caltech_camera_traps_5a0e37cc-23d2-11e8-a6a3-ec086b02610b.jpg: ignoring corrupt image/label: cannot identify image file 'g:\\\\temp\\\\md-test-images\\\\corrupt-images\\\\irfanview-can-still-read-me-caltech_camera_traps_5a0e37cc-23d2-11e8-a6a3-ec086b02610b.jpg'"
|
|
880
887
|
#
|
|
881
|
-
|
|
888
|
+
|
|
882
889
|
line = line.replace('⚠️',':')
|
|
883
890
|
if 'ignoring corrupt image/label' in line:
|
|
884
|
-
|
|
891
|
+
|
|
885
892
|
line_tokens = line.split('ignoring corrupt image/label')
|
|
886
893
|
assert len(line_tokens) == 2
|
|
887
|
-
|
|
894
|
+
|
|
888
895
|
tokens = line_tokens[0].split(':') # ,maxsplit=3)
|
|
889
896
|
tokens = [s.strip() for s in tokens]
|
|
890
|
-
|
|
897
|
+
|
|
891
898
|
# ['test', ' WARNING', ' a/b/c/d.jpg', ' ']
|
|
892
899
|
assert len(tokens[-1]) == 0
|
|
893
900
|
tokens = tokens[:-1]
|
|
894
901
|
assert 'warning' in tokens[1].lower()
|
|
895
|
-
|
|
902
|
+
|
|
896
903
|
if len(tokens) == 3:
|
|
897
904
|
image_name = tokens[2].strip()
|
|
898
905
|
else:
|
|
@@ -900,28 +907,28 @@ def run_inference_with_yolo_val(options):
|
|
|
900
907
|
assert len(tokens) == 4
|
|
901
908
|
assert len(tokens[2]) == 1
|
|
902
909
|
image_name = ':'.join(tokens[2:4])
|
|
903
|
-
|
|
910
|
+
|
|
904
911
|
yolo_read_failures.append(image_name)
|
|
905
|
-
|
|
912
|
+
|
|
906
913
|
# ...if this line indicated a corrupt image
|
|
907
|
-
|
|
914
|
+
|
|
908
915
|
# ...for each line in the console output
|
|
909
|
-
|
|
916
|
+
|
|
910
917
|
# image_file = yolo_read_failures[0]
|
|
911
918
|
for image_file in yolo_read_failures:
|
|
912
919
|
image_id = os.path.splitext(os.path.basename(image_file))[0]
|
|
913
920
|
assert image_id in image_id_to_file, 'Unexpected image ID {}'.format(image_id)
|
|
914
921
|
if image_id not in image_id_to_error:
|
|
915
922
|
image_id_to_error[image_id] = 'YOLO read failure'
|
|
916
|
-
|
|
923
|
+
|
|
917
924
|
if options.yolo_working_folder is not None:
|
|
918
925
|
os.chdir(current_dir)
|
|
919
|
-
|
|
920
|
-
|
|
926
|
+
|
|
927
|
+
|
|
921
928
|
##%% Convert results to MD format
|
|
922
|
-
|
|
929
|
+
|
|
923
930
|
json_files = glob.glob(yolo_results_folder + '/yolo_results/*.json')
|
|
924
|
-
assert len(json_files) == 1
|
|
931
|
+
assert len(json_files) == 1
|
|
925
932
|
yolo_json_file = json_files[0]
|
|
926
933
|
|
|
927
934
|
# Map YOLO image IDs to paths
|
|
@@ -939,14 +946,14 @@ def run_inference_with_yolo_val(options):
|
|
|
939
946
|
# as the base path in this case.
|
|
940
947
|
relative_path = fn
|
|
941
948
|
image_id_to_relative_path[image_id] = relative_path
|
|
942
|
-
|
|
949
|
+
|
|
943
950
|
# Are we working with a base folder?
|
|
944
951
|
if options.input_folder is not None:
|
|
945
952
|
assert os.path.isdir(options.input_folder)
|
|
946
953
|
image_base = options.input_folder
|
|
947
954
|
else:
|
|
948
955
|
image_base = '/'
|
|
949
|
-
|
|
956
|
+
|
|
950
957
|
yolo_output_to_md_output.yolo_json_output_to_md_output(
|
|
951
958
|
yolo_json_file=yolo_json_file,
|
|
952
959
|
image_folder=image_base,
|
|
@@ -954,38 +961,37 @@ def run_inference_with_yolo_val(options):
|
|
|
954
961
|
yolo_category_id_to_name=options.yolo_category_id_to_name,
|
|
955
962
|
detector_name=os.path.basename(model_filename),
|
|
956
963
|
image_id_to_relative_path=image_id_to_relative_path,
|
|
957
|
-
image_id_to_error=image_id_to_error
|
|
964
|
+
image_id_to_error=image_id_to_error,
|
|
965
|
+
offset_yolo_class_ids=options.offset_yolo_category_ids)
|
|
958
966
|
|
|
959
967
|
|
|
960
968
|
##%% Clean up
|
|
961
|
-
|
|
969
|
+
|
|
962
970
|
_clean_up_temporary_folders(options,
|
|
963
971
|
symlink_folder,yolo_results_folder,
|
|
964
972
|
symlink_folder_is_temp_folder,yolo_folder_is_temp_folder)
|
|
965
|
-
|
|
973
|
+
|
|
966
974
|
# ...def run_inference_with_yolo_val()
|
|
967
975
|
|
|
968
976
|
|
|
969
977
|
#%% Command-line driver
|
|
970
978
|
|
|
971
|
-
|
|
972
|
-
from megadetector.utils.ct_utils import args_to_object
|
|
979
|
+
def main(): # noqa
|
|
973
980
|
|
|
974
|
-
def main():
|
|
975
|
-
|
|
976
981
|
options = YoloInferenceOptions()
|
|
977
|
-
|
|
982
|
+
|
|
978
983
|
parser = argparse.ArgumentParser()
|
|
979
984
|
parser.add_argument(
|
|
980
985
|
'model_filename',type=str,
|
|
981
986
|
help='model file name')
|
|
982
987
|
parser.add_argument(
|
|
983
988
|
'input_folder',type=str,
|
|
984
|
-
help='folder on which to recursively run the model, or a .json or .txt file
|
|
989
|
+
help='folder on which to recursively run the model, or a .json or .txt file ' + \
|
|
990
|
+
'containing a list of absolute image paths')
|
|
985
991
|
parser.add_argument(
|
|
986
992
|
'output_file',type=str,
|
|
987
993
|
help='.json file where output will be written')
|
|
988
|
-
|
|
994
|
+
|
|
989
995
|
parser.add_argument(
|
|
990
996
|
'--image_filename_list',type=str,default=None,
|
|
991
997
|
help='.json or .txt file containing a list of relative image filenames within [input_folder]')
|
|
@@ -1005,10 +1011,12 @@ def main():
|
|
|
1005
1011
|
help='inference batch size (default {})'.format(options.batch_size))
|
|
1006
1012
|
parser.add_argument(
|
|
1007
1013
|
'--half_precision_enabled', default=None, type=int,
|
|
1008
|
-
help='use half-precision-inference (1 or 0) (default is the underlying model\'s default,
|
|
1014
|
+
help='use half-precision-inference (1 or 0) (default is the underlying model\'s default, ' + \
|
|
1015
|
+
'probably full for YOLOv8 and half for YOLOv5')
|
|
1009
1016
|
parser.add_argument(
|
|
1010
1017
|
'--device_string', default=options.device_string, type=str,
|
|
1011
|
-
help='CUDA device specifier, typically "0" or "1" for CUDA devices, "mps" for
|
|
1018
|
+
help='CUDA device specifier, typically "0" or "1" for CUDA devices, "mps" for ' + \
|
|
1019
|
+
'M1/M2 devices, or "cpu" (default {})'.format(
|
|
1012
1020
|
options.device_string))
|
|
1013
1021
|
parser.add_argument(
|
|
1014
1022
|
'--overwrite_handling', default=options.overwrite_handling, type=str,
|
|
@@ -1048,36 +1056,39 @@ def main():
|
|
|
1048
1056
|
parser.add_argument(
|
|
1049
1057
|
'--checkpoint_frequency', default=options.checkpoint_frequency, type=int,
|
|
1050
1058
|
help='break the job into chunks with no more than this many images (default {})'.format(
|
|
1051
|
-
options.checkpoint_frequency))
|
|
1059
|
+
options.checkpoint_frequency))
|
|
1052
1060
|
parser.add_argument(
|
|
1053
1061
|
'--no_append_job_id_to_symlink_folder', action='store_true',
|
|
1054
1062
|
help="don't append a unique job ID to the symlink folder name")
|
|
1055
1063
|
parser.add_argument(
|
|
1056
1064
|
'--nonrecursive', action='store_true',
|
|
1057
1065
|
help='disable recursive folder processing')
|
|
1058
|
-
|
|
1066
|
+
parser.add_argument(
|
|
1067
|
+
'--no_offset_class_ids', action='store_true',
|
|
1068
|
+
help='disable class ID offsetting')
|
|
1069
|
+
|
|
1059
1070
|
parser.add_argument(
|
|
1060
1071
|
'--preview_yolo_command_only', action='store_true',
|
|
1061
1072
|
help='don\'t run inference, just preview the YOLO inference command (still creates symlinks)')
|
|
1062
|
-
|
|
1073
|
+
|
|
1063
1074
|
if options.augment:
|
|
1064
1075
|
default_augment_enabled = 1
|
|
1065
1076
|
else:
|
|
1066
1077
|
default_augment_enabled = 0
|
|
1067
|
-
|
|
1078
|
+
|
|
1068
1079
|
parser.add_argument(
|
|
1069
1080
|
'--augment_enabled', default=default_augment_enabled, type=int,
|
|
1070
1081
|
help='enable/disable augmentation (default {})'.format(default_augment_enabled))
|
|
1071
|
-
|
|
1082
|
+
|
|
1072
1083
|
if len(sys.argv[1:]) == 0:
|
|
1073
1084
|
parser.print_help()
|
|
1074
1085
|
parser.exit()
|
|
1075
1086
|
|
|
1076
1087
|
args = parser.parse_args()
|
|
1077
|
-
|
|
1088
|
+
|
|
1078
1089
|
# If the caller hasn't specified an image size, choose one based on whether augmentation
|
|
1079
1090
|
# is enabled.
|
|
1080
|
-
if args.image_size is None:
|
|
1091
|
+
if args.image_size is None:
|
|
1081
1092
|
assert args.augment_enabled in (0,1), \
|
|
1082
1093
|
'Illegal augment_enabled value {}'.format(args.augment_enabled)
|
|
1083
1094
|
if args.augment_enabled == 1:
|
|
@@ -1089,38 +1100,40 @@ def main():
|
|
|
1089
1100
|
augment_enabled_string = 'disabled'
|
|
1090
1101
|
print('Augmentation is {}, using default image size {}'.format(
|
|
1091
1102
|
augment_enabled_string,args.image_size))
|
|
1092
|
-
|
|
1103
|
+
|
|
1093
1104
|
args_to_object(args, options)
|
|
1094
|
-
|
|
1105
|
+
|
|
1095
1106
|
if args.yolo_dataset_file is not None:
|
|
1096
1107
|
options.yolo_category_id_to_name = args.yolo_dataset_file
|
|
1097
|
-
|
|
1098
|
-
# The function convention is that input_folder should be None when we want to use a list of
|
|
1099
|
-
# absolute paths, but the CLI convention is that the required argument is always valid, whether
|
|
1108
|
+
|
|
1109
|
+
# The function convention is that input_folder should be None when we want to use a list of
|
|
1110
|
+
# absolute paths, but the CLI convention is that the required argument is always valid, whether
|
|
1100
1111
|
# it's a folder or a list of absolute paths.
|
|
1101
1112
|
if os.path.isfile(options.input_folder):
|
|
1102
1113
|
assert options.image_filename_list is None, \
|
|
1103
1114
|
'image_filename_list should not be specified when input_folder is a file'
|
|
1104
1115
|
options.image_filename_list = options.input_folder
|
|
1105
|
-
options.input_folder = None
|
|
1106
|
-
|
|
1116
|
+
options.input_folder = None
|
|
1117
|
+
|
|
1107
1118
|
options.recursive = (not options.nonrecursive)
|
|
1108
1119
|
options.append_job_id_to_symlink_folder = (not options.no_append_job_id_to_symlink_folder)
|
|
1109
1120
|
options.remove_symlink_folder = (not options.no_remove_symlink_folder)
|
|
1110
1121
|
options.remove_yolo_results_folder = (not options.no_remove_yolo_results_folder)
|
|
1111
1122
|
options.use_symlinks = (not options.no_use_symlinks)
|
|
1112
|
-
options.augment = (options.augment_enabled > 0)
|
|
1113
|
-
|
|
1123
|
+
options.augment = (options.augment_enabled > 0)
|
|
1124
|
+
options.offset_yolo_category_ids = (not options.no_offset_class_ids)
|
|
1125
|
+
|
|
1114
1126
|
del options.nonrecursive
|
|
1115
1127
|
del options.no_remove_symlink_folder
|
|
1116
1128
|
del options.no_remove_yolo_results_folder
|
|
1117
1129
|
del options.no_use_symlinks
|
|
1118
1130
|
del options.augment_enabled
|
|
1119
1131
|
del options.yolo_dataset_file
|
|
1120
|
-
|
|
1132
|
+
del options.no_offset_class_ids
|
|
1133
|
+
|
|
1121
1134
|
print(options.__dict__)
|
|
1122
|
-
|
|
1123
|
-
run_inference_with_yolo_val(options)
|
|
1135
|
+
|
|
1136
|
+
run_inference_with_yolo_val(options)
|
|
1124
1137
|
|
|
1125
1138
|
if __name__ == '__main__':
|
|
1126
1139
|
main()
|
|
@@ -1133,7 +1146,7 @@ if False:
|
|
|
1133
1146
|
|
|
1134
1147
|
#%% Debugging
|
|
1135
1148
|
|
|
1136
|
-
input_folder = r'g:\temp\md-test-images'
|
|
1149
|
+
input_folder = r'g:\temp\md-test-images'
|
|
1137
1150
|
model_filename = 'MDV5A'
|
|
1138
1151
|
output_folder = r'g:\temp\yolo-test-out'
|
|
1139
1152
|
yolo_working_folder = r'c:\git\yolov5-md'
|
|
@@ -1142,16 +1155,16 @@ if False:
|
|
|
1142
1155
|
symlink_folder = os.path.join(output_folder,'symlinks')
|
|
1143
1156
|
yolo_results_folder = os.path.join(output_folder,'yolo_results')
|
|
1144
1157
|
model_name = os.path.splitext(os.path.basename(model_filename))[0]
|
|
1145
|
-
|
|
1158
|
+
|
|
1146
1159
|
output_file = os.path.join(output_folder,'{}_{}-md_format.json'.format(
|
|
1147
1160
|
job_name,model_name))
|
|
1148
|
-
|
|
1161
|
+
|
|
1149
1162
|
options = YoloInferenceOptions()
|
|
1150
|
-
|
|
1163
|
+
|
|
1151
1164
|
options.yolo_working_folder = yolo_working_folder
|
|
1152
1165
|
options.input_folder = input_folder
|
|
1153
1166
|
options.output_file = output_file
|
|
1154
|
-
|
|
1167
|
+
|
|
1155
1168
|
options.yolo_category_id_to_name = dataset_file
|
|
1156
1169
|
options.augment = False
|
|
1157
1170
|
options.conf_thres = '0.001'
|
|
@@ -1164,18 +1177,18 @@ if False:
|
|
|
1164
1177
|
options.image_size = round(1280 * 1.3)
|
|
1165
1178
|
else:
|
|
1166
1179
|
options.image_size = 1280
|
|
1167
|
-
|
|
1180
|
+
|
|
1168
1181
|
options.model_filename = model_filename
|
|
1169
|
-
|
|
1170
|
-
options.yolo_results_folder = yolo_results_folder # os.path.join(output_folder + 'yolo_results')
|
|
1182
|
+
|
|
1183
|
+
options.yolo_results_folder = yolo_results_folder # os.path.join(output_folder + 'yolo_results')
|
|
1171
1184
|
options.symlink_folder = symlink_folder # os.path.join(output_folder,'symlinks')
|
|
1172
1185
|
options.use_symlinks = False
|
|
1173
|
-
|
|
1186
|
+
|
|
1174
1187
|
options.remove_symlink_folder = True
|
|
1175
1188
|
options.remove_yolo_results_folder = True
|
|
1176
|
-
|
|
1189
|
+
|
|
1177
1190
|
options.checkpoint_frequency = None
|
|
1178
|
-
|
|
1191
|
+
|
|
1179
1192
|
cmd = f'python run_inference_with_yolov5_val.py {model_filename} {input_folder} ' + \
|
|
1180
1193
|
f'{output_file} --yolo_working_folder {yolo_working_folder} ' + \
|
|
1181
1194
|
f' --image_size {options.image_size} --conf_thres {options.conf_thres} ' + \
|
|
@@ -1183,10 +1196,10 @@ if False:
|
|
|
1183
1196
|
f' --symlink_folder {options.symlink_folder} --yolo_results_folder {options.yolo_results_folder} ' + \
|
|
1184
1197
|
f' --yolo_dataset_file {options.yolo_category_id_to_name} ' + \
|
|
1185
1198
|
f' --unique_id_strategy {options.unique_id_strategy} --overwrite_handling {options.overwrite_handling}'
|
|
1186
|
-
|
|
1199
|
+
|
|
1187
1200
|
if not options.remove_symlink_folder:
|
|
1188
1201
|
cmd += ' --no_remove_symlink_folder'
|
|
1189
|
-
if not options.remove_yolo_results_folder:
|
|
1202
|
+
if not options.remove_yolo_results_folder:
|
|
1190
1203
|
cmd += ' --no_remove_yolo_results_folder'
|
|
1191
1204
|
if options.checkpoint_frequency is not None:
|
|
1192
1205
|
cmd += f' --checkpoint_frequency {options.checkpoint_frequency}'
|
|
@@ -1194,7 +1207,7 @@ if False:
|
|
|
1194
1207
|
cmd += ' --no_use_symlinks'
|
|
1195
1208
|
if not options.augment:
|
|
1196
1209
|
cmd += ' --augment_enabled 0'
|
|
1197
|
-
|
|
1210
|
+
|
|
1198
1211
|
print(cmd)
|
|
1199
1212
|
execute_in_python = False
|
|
1200
1213
|
if execute_in_python:
|
|
@@ -1205,47 +1218,47 @@ if False:
|
|
|
1205
1218
|
|
|
1206
1219
|
|
|
1207
1220
|
#%% Run inference on a folder
|
|
1208
|
-
|
|
1221
|
+
|
|
1209
1222
|
input_folder = r'g:\temp\tegu-val-mini'.replace('\\','/')
|
|
1210
1223
|
model_filename = r'g:\temp\usgs-tegus-yolov5x-231003-b8-img1280-e3002-best.pt'
|
|
1211
1224
|
output_folder = r'g:\temp\tegu-scratch'
|
|
1212
1225
|
yolo_working_folder = r'c:\git\yolov5-tegus'
|
|
1213
1226
|
dataset_file = r'g:\temp\dataset.yaml'
|
|
1214
|
-
|
|
1227
|
+
|
|
1215
1228
|
# This only impacts the output file name, it's not passed to the inference function
|
|
1216
1229
|
job_name = 'yolo-inference-test'
|
|
1217
|
-
|
|
1230
|
+
|
|
1218
1231
|
model_name = os.path.splitext(os.path.basename(model_filename))[0]
|
|
1219
|
-
|
|
1232
|
+
|
|
1220
1233
|
symlink_folder = os.path.join(output_folder,'symlinks')
|
|
1221
1234
|
yolo_results_folder = os.path.join(output_folder,'yolo_results')
|
|
1222
|
-
|
|
1235
|
+
|
|
1223
1236
|
output_file = os.path.join(output_folder,'{}_{}-md_format.json'.format(
|
|
1224
1237
|
job_name,model_name))
|
|
1225
|
-
|
|
1238
|
+
|
|
1226
1239
|
options = YoloInferenceOptions()
|
|
1227
|
-
|
|
1240
|
+
|
|
1228
1241
|
options.yolo_working_folder = yolo_working_folder
|
|
1229
1242
|
options.input_folder = input_folder
|
|
1230
1243
|
options.output_file = output_file
|
|
1231
|
-
|
|
1232
|
-
pass_image_filename_list = False
|
|
1244
|
+
|
|
1245
|
+
pass_image_filename_list = False
|
|
1233
1246
|
pass_relative_paths = True
|
|
1234
|
-
|
|
1247
|
+
|
|
1235
1248
|
if pass_image_filename_list:
|
|
1236
1249
|
if pass_relative_paths:
|
|
1237
1250
|
options.image_filename_list = [
|
|
1238
1251
|
r"val#american_cardinal#american_cardinal#CaCa#31W.01_C83#2017-2019#C90 and C83_31W.01#(05) 18AUG17 - 05SEP17 FTC AEG#MFDC1949_000065.JPG",
|
|
1239
1252
|
r"val#american_cardinal#american_cardinal#CaCa#31W.01_C83#2017-2019#C90 and C83_31W.01#(04) 27JUL17 - 18AUG17 FTC AEG#MFDC1902_000064.JPG"
|
|
1240
|
-
]
|
|
1253
|
+
]
|
|
1241
1254
|
else:
|
|
1242
1255
|
options.image_filename_list = [
|
|
1243
1256
|
r"g:/temp/tegu-val-mini/val#american_cardinal#american_cardinal#CaCa#31W.01_C83#2017-2019#C90 and C83_31W.01#(05) 18AUG17 - 05SEP17 FTC AEG#MFDC1949_000065.JPG",
|
|
1244
1257
|
r"g:/temp/tegu-val-mini/val#american_cardinal#american_cardinal#CaCa#31W.01_C83#2017-2019#C90 and C83_31W.01#(04) 27JUL17 - 18AUG17 FTC AEG#MFDC1902_000064.JPG"
|
|
1245
1258
|
]
|
|
1246
1259
|
else:
|
|
1247
|
-
options.image_filename_list = None
|
|
1248
|
-
|
|
1260
|
+
options.image_filename_list = None
|
|
1261
|
+
|
|
1249
1262
|
options.yolo_category_id_to_name = dataset_file
|
|
1250
1263
|
options.augment = False
|
|
1251
1264
|
options.conf_thres = '0.001'
|
|
@@ -1258,18 +1271,18 @@ if False:
|
|
|
1258
1271
|
options.image_size = round(1280 * 1.3)
|
|
1259
1272
|
else:
|
|
1260
1273
|
options.image_size = 1280
|
|
1261
|
-
|
|
1274
|
+
|
|
1262
1275
|
options.model_filename = model_filename
|
|
1263
|
-
|
|
1264
|
-
options.yolo_results_folder = yolo_results_folder # os.path.join(output_folder + 'yolo_results')
|
|
1276
|
+
|
|
1277
|
+
options.yolo_results_folder = yolo_results_folder # os.path.join(output_folder + 'yolo_results')
|
|
1265
1278
|
options.symlink_folder = symlink_folder # os.path.join(output_folder,'symlinks')
|
|
1266
1279
|
options.use_symlinks = False
|
|
1267
|
-
|
|
1280
|
+
|
|
1268
1281
|
options.remove_symlink_folder = True
|
|
1269
1282
|
options.remove_yolo_results_folder = True
|
|
1270
|
-
|
|
1283
|
+
|
|
1271
1284
|
options.checkpoint_frequency = 5
|
|
1272
|
-
|
|
1285
|
+
|
|
1273
1286
|
cmd = f'python run_inference_with_yolov5_val.py {model_filename} {input_folder} ' + \
|
|
1274
1287
|
f'{output_file} --yolo_working_folder {yolo_working_folder} ' + \
|
|
1275
1288
|
f' --image_size {options.image_size} --conf_thres {options.conf_thres} ' + \
|
|
@@ -1277,10 +1290,10 @@ if False:
|
|
|
1277
1290
|
f' --symlink_folder {options.symlink_folder} --yolo_results_folder {options.yolo_results_folder} ' + \
|
|
1278
1291
|
f' --yolo_dataset_file {options.yolo_category_id_to_name} ' + \
|
|
1279
1292
|
f' --unique_id_strategy {options.unique_id_strategy} --overwrite_handling {options.overwrite_handling}'
|
|
1280
|
-
|
|
1293
|
+
|
|
1281
1294
|
if not options.remove_symlink_folder:
|
|
1282
1295
|
cmd += ' --no_remove_symlink_folder'
|
|
1283
|
-
if not options.remove_yolo_results_folder:
|
|
1296
|
+
if not options.remove_yolo_results_folder:
|
|
1284
1297
|
cmd += ' --no_remove_yolo_results_folder'
|
|
1285
1298
|
if options.checkpoint_frequency is not None:
|
|
1286
1299
|
cmd += f' --checkpoint_frequency {options.checkpoint_frequency}'
|
|
@@ -1288,7 +1301,7 @@ if False:
|
|
|
1288
1301
|
cmd += ' --no_use_symlinks'
|
|
1289
1302
|
if not options.augment:
|
|
1290
1303
|
cmd += ' --augment_enabled 0'
|
|
1291
|
-
|
|
1304
|
+
|
|
1292
1305
|
print(cmd)
|
|
1293
1306
|
execute_in_python = False
|
|
1294
1307
|
if execute_in_python:
|