ultralytics 8.0.237__py3-none-any.whl → 8.0.239__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ultralytics might be problematic. Click here for more details.
- ultralytics/__init__.py +2 -2
- ultralytics/cfg/__init__.py +241 -138
- ultralytics/cfg/datasets/DOTAv1.5.yaml +1 -1
- ultralytics/cfg/datasets/DOTAv1.yaml +1 -1
- ultralytics/cfg/datasets/dota8.yaml +34 -0
- ultralytics/data/__init__.py +9 -2
- ultralytics/data/annotator.py +4 -4
- ultralytics/data/augment.py +186 -169
- ultralytics/data/base.py +54 -48
- ultralytics/data/build.py +34 -23
- ultralytics/data/converter.py +242 -70
- ultralytics/data/dataset.py +117 -95
- ultralytics/data/explorer/__init__.py +5 -0
- ultralytics/data/explorer/explorer.py +170 -97
- ultralytics/data/explorer/gui/__init__.py +1 -0
- ultralytics/data/explorer/gui/dash.py +146 -76
- ultralytics/data/explorer/utils.py +87 -25
- ultralytics/data/loaders.py +75 -62
- ultralytics/data/split_dota.py +44 -36
- ultralytics/data/utils.py +160 -142
- ultralytics/engine/exporter.py +348 -292
- ultralytics/engine/model.py +102 -66
- ultralytics/engine/predictor.py +74 -55
- ultralytics/engine/results.py +63 -40
- ultralytics/engine/trainer.py +192 -144
- ultralytics/engine/tuner.py +66 -59
- ultralytics/engine/validator.py +31 -26
- ultralytics/hub/__init__.py +54 -31
- ultralytics/hub/auth.py +28 -25
- ultralytics/hub/session.py +282 -133
- ultralytics/hub/utils.py +64 -42
- ultralytics/models/__init__.py +1 -1
- ultralytics/models/fastsam/__init__.py +1 -1
- ultralytics/models/fastsam/model.py +6 -6
- ultralytics/models/fastsam/predict.py +3 -2
- ultralytics/models/fastsam/prompt.py +55 -48
- ultralytics/models/fastsam/val.py +1 -1
- ultralytics/models/nas/__init__.py +1 -1
- ultralytics/models/nas/model.py +9 -8
- ultralytics/models/nas/predict.py +8 -6
- ultralytics/models/nas/val.py +11 -9
- ultralytics/models/rtdetr/__init__.py +1 -1
- ultralytics/models/rtdetr/model.py +11 -9
- ultralytics/models/rtdetr/train.py +18 -16
- ultralytics/models/rtdetr/val.py +25 -19
- ultralytics/models/sam/__init__.py +1 -1
- ultralytics/models/sam/amg.py +13 -14
- ultralytics/models/sam/build.py +44 -42
- ultralytics/models/sam/model.py +6 -6
- ultralytics/models/sam/modules/decoders.py +6 -4
- ultralytics/models/sam/modules/encoders.py +37 -35
- ultralytics/models/sam/modules/sam.py +5 -4
- ultralytics/models/sam/modules/tiny_encoder.py +95 -73
- ultralytics/models/sam/modules/transformer.py +3 -2
- ultralytics/models/sam/predict.py +39 -27
- ultralytics/models/utils/loss.py +99 -95
- ultralytics/models/utils/ops.py +34 -31
- ultralytics/models/yolo/__init__.py +1 -1
- ultralytics/models/yolo/classify/__init__.py +1 -1
- ultralytics/models/yolo/classify/predict.py +8 -6
- ultralytics/models/yolo/classify/train.py +37 -31
- ultralytics/models/yolo/classify/val.py +26 -24
- ultralytics/models/yolo/detect/__init__.py +1 -1
- ultralytics/models/yolo/detect/predict.py +8 -6
- ultralytics/models/yolo/detect/train.py +47 -37
- ultralytics/models/yolo/detect/val.py +100 -82
- ultralytics/models/yolo/model.py +31 -25
- ultralytics/models/yolo/obb/__init__.py +1 -1
- ultralytics/models/yolo/obb/predict.py +13 -12
- ultralytics/models/yolo/obb/train.py +3 -3
- ultralytics/models/yolo/obb/val.py +80 -58
- ultralytics/models/yolo/pose/__init__.py +1 -1
- ultralytics/models/yolo/pose/predict.py +17 -12
- ultralytics/models/yolo/pose/train.py +28 -25
- ultralytics/models/yolo/pose/val.py +91 -64
- ultralytics/models/yolo/segment/__init__.py +1 -1
- ultralytics/models/yolo/segment/predict.py +10 -8
- ultralytics/models/yolo/segment/train.py +16 -15
- ultralytics/models/yolo/segment/val.py +90 -68
- ultralytics/nn/__init__.py +26 -6
- ultralytics/nn/autobackend.py +144 -112
- ultralytics/nn/modules/__init__.py +96 -13
- ultralytics/nn/modules/block.py +28 -7
- ultralytics/nn/modules/conv.py +41 -23
- ultralytics/nn/modules/head.py +67 -59
- ultralytics/nn/modules/transformer.py +49 -32
- ultralytics/nn/modules/utils.py +20 -15
- ultralytics/nn/tasks.py +215 -141
- ultralytics/solutions/ai_gym.py +59 -47
- ultralytics/solutions/distance_calculation.py +22 -15
- ultralytics/solutions/heatmap.py +76 -54
- ultralytics/solutions/object_counter.py +46 -39
- ultralytics/solutions/speed_estimation.py +13 -16
- ultralytics/trackers/__init__.py +1 -1
- ultralytics/trackers/basetrack.py +1 -0
- ultralytics/trackers/bot_sort.py +2 -1
- ultralytics/trackers/byte_tracker.py +10 -7
- ultralytics/trackers/track.py +7 -7
- ultralytics/trackers/utils/gmc.py +25 -25
- ultralytics/trackers/utils/kalman_filter.py +85 -42
- ultralytics/trackers/utils/matching.py +8 -7
- ultralytics/utils/__init__.py +173 -151
- ultralytics/utils/autobatch.py +10 -10
- ultralytics/utils/benchmarks.py +76 -86
- ultralytics/utils/callbacks/__init__.py +1 -1
- ultralytics/utils/callbacks/base.py +29 -29
- ultralytics/utils/callbacks/clearml.py +51 -43
- ultralytics/utils/callbacks/comet.py +81 -66
- ultralytics/utils/callbacks/dvc.py +33 -26
- ultralytics/utils/callbacks/hub.py +44 -26
- ultralytics/utils/callbacks/mlflow.py +31 -24
- ultralytics/utils/callbacks/neptune.py +35 -25
- ultralytics/utils/callbacks/raytune.py +9 -4
- ultralytics/utils/callbacks/tensorboard.py +16 -11
- ultralytics/utils/callbacks/wb.py +39 -33
- ultralytics/utils/checks.py +189 -141
- ultralytics/utils/dist.py +15 -12
- ultralytics/utils/downloads.py +112 -96
- ultralytics/utils/errors.py +1 -1
- ultralytics/utils/files.py +11 -11
- ultralytics/utils/instance.py +22 -22
- ultralytics/utils/loss.py +117 -67
- ultralytics/utils/metrics.py +224 -158
- ultralytics/utils/ops.py +39 -29
- ultralytics/utils/patches.py +3 -3
- ultralytics/utils/plotting.py +217 -120
- ultralytics/utils/tal.py +19 -13
- ultralytics/utils/torch_utils.py +138 -109
- ultralytics/utils/triton.py +12 -10
- ultralytics/utils/tuner.py +49 -47
- {ultralytics-8.0.237.dist-info → ultralytics-8.0.239.dist-info}/METADATA +5 -4
- ultralytics-8.0.239.dist-info/RECORD +188 -0
- ultralytics-8.0.237.dist-info/RECORD +0 -187
- {ultralytics-8.0.237.dist-info → ultralytics-8.0.239.dist-info}/LICENSE +0 -0
- {ultralytics-8.0.237.dist-info → ultralytics-8.0.239.dist-info}/WHEEL +0 -0
- {ultralytics-8.0.237.dist-info → ultralytics-8.0.239.dist-info}/entry_points.txt +0 -0
- {ultralytics-8.0.237.dist-info → ultralytics-8.0.239.dist-info}/top_level.txt +0 -0
|
@@ -4,20 +4,20 @@ from ultralytics.utils import LOGGER, RANK, SETTINGS, TESTS_RUNNING, ops
|
|
|
4
4
|
|
|
5
5
|
try:
|
|
6
6
|
assert not TESTS_RUNNING # do not log pytest
|
|
7
|
-
assert SETTINGS[
|
|
7
|
+
assert SETTINGS["comet"] is True # verify integration is enabled
|
|
8
8
|
import comet_ml
|
|
9
9
|
|
|
10
|
-
assert hasattr(comet_ml,
|
|
10
|
+
assert hasattr(comet_ml, "__version__") # verify package is not directory
|
|
11
11
|
|
|
12
12
|
import os
|
|
13
13
|
from pathlib import Path
|
|
14
14
|
|
|
15
15
|
# Ensures certain logging functions only run for supported tasks
|
|
16
|
-
COMET_SUPPORTED_TASKS = [
|
|
16
|
+
COMET_SUPPORTED_TASKS = ["detect"]
|
|
17
17
|
|
|
18
18
|
# Names of plots created by YOLOv8 that are logged to Comet
|
|
19
|
-
EVALUATION_PLOT_NAMES =
|
|
20
|
-
LABEL_PLOT_NAMES =
|
|
19
|
+
EVALUATION_PLOT_NAMES = "F1_curve", "P_curve", "R_curve", "PR_curve", "confusion_matrix"
|
|
20
|
+
LABEL_PLOT_NAMES = "labels", "labels_correlogram"
|
|
21
21
|
|
|
22
22
|
_comet_image_prediction_count = 0
|
|
23
23
|
|
|
@@ -27,43 +27,43 @@ except (ImportError, AssertionError):
|
|
|
27
27
|
|
|
28
28
|
def _get_comet_mode():
|
|
29
29
|
"""Returns the mode of comet set in the environment variables, defaults to 'online' if not set."""
|
|
30
|
-
return os.getenv(
|
|
30
|
+
return os.getenv("COMET_MODE", "online")
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
def _get_comet_model_name():
|
|
34
34
|
"""Returns the model name for Comet from the environment variable 'COMET_MODEL_NAME' or defaults to 'YOLOv8'."""
|
|
35
|
-
return os.getenv(
|
|
35
|
+
return os.getenv("COMET_MODEL_NAME", "YOLOv8")
|
|
36
36
|
|
|
37
37
|
|
|
38
38
|
def _get_eval_batch_logging_interval():
|
|
39
39
|
"""Get the evaluation batch logging interval from environment variable or use default value 1."""
|
|
40
|
-
return int(os.getenv(
|
|
40
|
+
return int(os.getenv("COMET_EVAL_BATCH_LOGGING_INTERVAL", 1))
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
def _get_max_image_predictions_to_log():
|
|
44
44
|
"""Get the maximum number of image predictions to log from the environment variables."""
|
|
45
|
-
return int(os.getenv(
|
|
45
|
+
return int(os.getenv("COMET_MAX_IMAGE_PREDICTIONS", 100))
|
|
46
46
|
|
|
47
47
|
|
|
48
48
|
def _scale_confidence_score(score):
|
|
49
49
|
"""Scales the given confidence score by a factor specified in an environment variable."""
|
|
50
|
-
scale = float(os.getenv(
|
|
50
|
+
scale = float(os.getenv("COMET_MAX_CONFIDENCE_SCORE", 100.0))
|
|
51
51
|
return score * scale
|
|
52
52
|
|
|
53
53
|
|
|
54
54
|
def _should_log_confusion_matrix():
|
|
55
55
|
"""Determines if the confusion matrix should be logged based on the environment variable settings."""
|
|
56
|
-
return os.getenv(
|
|
56
|
+
return os.getenv("COMET_EVAL_LOG_CONFUSION_MATRIX", "false").lower() == "true"
|
|
57
57
|
|
|
58
58
|
|
|
59
59
|
def _should_log_image_predictions():
|
|
60
60
|
"""Determines whether to log image predictions based on a specified environment variable."""
|
|
61
|
-
return os.getenv(
|
|
61
|
+
return os.getenv("COMET_EVAL_LOG_IMAGE_PREDICTIONS", "true").lower() == "true"
|
|
62
62
|
|
|
63
63
|
|
|
64
64
|
def _get_experiment_type(mode, project_name):
|
|
65
65
|
"""Return an experiment based on mode and project name."""
|
|
66
|
-
if mode ==
|
|
66
|
+
if mode == "offline":
|
|
67
67
|
return comet_ml.OfflineExperiment(project_name=project_name)
|
|
68
68
|
|
|
69
69
|
return comet_ml.Experiment(project_name=project_name)
|
|
@@ -75,18 +75,21 @@ def _create_experiment(args):
|
|
|
75
75
|
return
|
|
76
76
|
try:
|
|
77
77
|
comet_mode = _get_comet_mode()
|
|
78
|
-
_project_name = os.getenv(
|
|
78
|
+
_project_name = os.getenv("COMET_PROJECT_NAME", args.project)
|
|
79
79
|
experiment = _get_experiment_type(comet_mode, _project_name)
|
|
80
80
|
experiment.log_parameters(vars(args))
|
|
81
|
-
experiment.log_others(
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
81
|
+
experiment.log_others(
|
|
82
|
+
{
|
|
83
|
+
"eval_batch_logging_interval": _get_eval_batch_logging_interval(),
|
|
84
|
+
"log_confusion_matrix_on_eval": _should_log_confusion_matrix(),
|
|
85
|
+
"log_image_predictions": _should_log_image_predictions(),
|
|
86
|
+
"max_image_predictions": _get_max_image_predictions_to_log(),
|
|
87
|
+
}
|
|
88
|
+
)
|
|
89
|
+
experiment.log_other("Created from", "yolov8")
|
|
87
90
|
|
|
88
91
|
except Exception as e:
|
|
89
|
-
LOGGER.warning(f
|
|
92
|
+
LOGGER.warning(f"WARNING ⚠️ Comet installed but not initialized correctly, not logging this run. {e}")
|
|
90
93
|
|
|
91
94
|
|
|
92
95
|
def _fetch_trainer_metadata(trainer):
|
|
@@ -134,29 +137,32 @@ def _scale_bounding_box_to_original_image_shape(box, resized_image_shape, origin
|
|
|
134
137
|
|
|
135
138
|
def _format_ground_truth_annotations_for_detection(img_idx, image_path, batch, class_name_map=None):
|
|
136
139
|
"""Format ground truth annotations for detection."""
|
|
137
|
-
indices = batch[
|
|
138
|
-
bboxes = batch[
|
|
140
|
+
indices = batch["batch_idx"] == img_idx
|
|
141
|
+
bboxes = batch["bboxes"][indices]
|
|
139
142
|
if len(bboxes) == 0:
|
|
140
|
-
LOGGER.debug(f
|
|
143
|
+
LOGGER.debug(f"COMET WARNING: Image: {image_path} has no bounding boxes labels")
|
|
141
144
|
return None
|
|
142
145
|
|
|
143
|
-
cls_labels = batch[
|
|
146
|
+
cls_labels = batch["cls"][indices].squeeze(1).tolist()
|
|
144
147
|
if class_name_map:
|
|
145
148
|
cls_labels = [str(class_name_map[label]) for label in cls_labels]
|
|
146
149
|
|
|
147
|
-
original_image_shape = batch[
|
|
148
|
-
resized_image_shape = batch[
|
|
149
|
-
ratio_pad = batch[
|
|
150
|
+
original_image_shape = batch["ori_shape"][img_idx]
|
|
151
|
+
resized_image_shape = batch["resized_shape"][img_idx]
|
|
152
|
+
ratio_pad = batch["ratio_pad"][img_idx]
|
|
150
153
|
|
|
151
154
|
data = []
|
|
152
155
|
for box, label in zip(bboxes, cls_labels):
|
|
153
156
|
box = _scale_bounding_box_to_original_image_shape(box, resized_image_shape, original_image_shape, ratio_pad)
|
|
154
|
-
data.append(
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
157
|
+
data.append(
|
|
158
|
+
{
|
|
159
|
+
"boxes": [box],
|
|
160
|
+
"label": f"gt_{label}",
|
|
161
|
+
"score": _scale_confidence_score(1.0),
|
|
162
|
+
}
|
|
163
|
+
)
|
|
158
164
|
|
|
159
|
-
return {
|
|
165
|
+
return {"name": "ground_truth", "data": data}
|
|
160
166
|
|
|
161
167
|
|
|
162
168
|
def _format_prediction_annotations_for_detection(image_path, metadata, class_label_map=None):
|
|
@@ -166,31 +172,34 @@ def _format_prediction_annotations_for_detection(image_path, metadata, class_lab
|
|
|
166
172
|
|
|
167
173
|
predictions = metadata.get(image_id)
|
|
168
174
|
if not predictions:
|
|
169
|
-
LOGGER.debug(f
|
|
175
|
+
LOGGER.debug(f"COMET WARNING: Image: {image_path} has no bounding boxes predictions")
|
|
170
176
|
return None
|
|
171
177
|
|
|
172
178
|
data = []
|
|
173
179
|
for prediction in predictions:
|
|
174
|
-
boxes = prediction[
|
|
175
|
-
score = _scale_confidence_score(prediction[
|
|
176
|
-
cls_label = prediction[
|
|
180
|
+
boxes = prediction["bbox"]
|
|
181
|
+
score = _scale_confidence_score(prediction["score"])
|
|
182
|
+
cls_label = prediction["category_id"]
|
|
177
183
|
if class_label_map:
|
|
178
184
|
cls_label = str(class_label_map[cls_label])
|
|
179
185
|
|
|
180
|
-
data.append({
|
|
186
|
+
data.append({"boxes": [boxes], "label": cls_label, "score": score})
|
|
181
187
|
|
|
182
|
-
return {
|
|
188
|
+
return {"name": "prediction", "data": data}
|
|
183
189
|
|
|
184
190
|
|
|
185
191
|
def _fetch_annotations(img_idx, image_path, batch, prediction_metadata_map, class_label_map):
|
|
186
192
|
"""Join the ground truth and prediction annotations if they exist."""
|
|
187
|
-
ground_truth_annotations = _format_ground_truth_annotations_for_detection(
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
193
|
+
ground_truth_annotations = _format_ground_truth_annotations_for_detection(
|
|
194
|
+
img_idx, image_path, batch, class_label_map
|
|
195
|
+
)
|
|
196
|
+
prediction_annotations = _format_prediction_annotations_for_detection(
|
|
197
|
+
image_path, prediction_metadata_map, class_label_map
|
|
198
|
+
)
|
|
191
199
|
|
|
192
200
|
annotations = [
|
|
193
|
-
annotation for annotation in [ground_truth_annotations, prediction_annotations] if annotation is not None
|
|
201
|
+
annotation for annotation in [ground_truth_annotations, prediction_annotations] if annotation is not None
|
|
202
|
+
]
|
|
194
203
|
return [annotations] if annotations else None
|
|
195
204
|
|
|
196
205
|
|
|
@@ -198,8 +207,8 @@ def _create_prediction_metadata_map(model_predictions):
|
|
|
198
207
|
"""Create metadata map for model predictions by groupings them based on image ID."""
|
|
199
208
|
pred_metadata_map = {}
|
|
200
209
|
for prediction in model_predictions:
|
|
201
|
-
pred_metadata_map.setdefault(prediction[
|
|
202
|
-
pred_metadata_map[prediction[
|
|
210
|
+
pred_metadata_map.setdefault(prediction["image_id"], [])
|
|
211
|
+
pred_metadata_map[prediction["image_id"]].append(prediction)
|
|
203
212
|
|
|
204
213
|
return pred_metadata_map
|
|
205
214
|
|
|
@@ -207,7 +216,7 @@ def _create_prediction_metadata_map(model_predictions):
|
|
|
207
216
|
def _log_confusion_matrix(experiment, trainer, curr_step, curr_epoch):
|
|
208
217
|
"""Log the confusion matrix to Comet experiment."""
|
|
209
218
|
conf_mat = trainer.validator.confusion_matrix.matrix
|
|
210
|
-
names = list(trainer.data[
|
|
219
|
+
names = list(trainer.data["names"].values()) + ["background"]
|
|
211
220
|
experiment.log_confusion_matrix(
|
|
212
221
|
matrix=conf_mat,
|
|
213
222
|
labels=names,
|
|
@@ -251,7 +260,7 @@ def _log_image_predictions(experiment, validator, curr_step):
|
|
|
251
260
|
if (batch_idx + 1) % batch_logging_interval != 0:
|
|
252
261
|
continue
|
|
253
262
|
|
|
254
|
-
image_paths = batch[
|
|
263
|
+
image_paths = batch["im_file"]
|
|
255
264
|
for img_idx, image_path in enumerate(image_paths):
|
|
256
265
|
if _comet_image_prediction_count >= max_image_predictions:
|
|
257
266
|
return
|
|
@@ -275,10 +284,10 @@ def _log_image_predictions(experiment, validator, curr_step):
|
|
|
275
284
|
|
|
276
285
|
def _log_plots(experiment, trainer):
|
|
277
286
|
"""Logs evaluation plots and label plots for the experiment."""
|
|
278
|
-
plot_filenames = [trainer.save_dir / f
|
|
287
|
+
plot_filenames = [trainer.save_dir / f"{plots}.png" for plots in EVALUATION_PLOT_NAMES]
|
|
279
288
|
_log_images(experiment, plot_filenames, None)
|
|
280
289
|
|
|
281
|
-
label_plot_filenames = [trainer.save_dir / f
|
|
290
|
+
label_plot_filenames = [trainer.save_dir / f"{labels}.jpg" for labels in LABEL_PLOT_NAMES]
|
|
282
291
|
_log_images(experiment, label_plot_filenames, None)
|
|
283
292
|
|
|
284
293
|
|
|
@@ -288,7 +297,7 @@ def _log_model(experiment, trainer):
|
|
|
288
297
|
experiment.log_model(
|
|
289
298
|
model_name,
|
|
290
299
|
file_or_folder=str(trainer.best),
|
|
291
|
-
file_name=
|
|
300
|
+
file_name="best.pt",
|
|
292
301
|
overwrite=True,
|
|
293
302
|
)
|
|
294
303
|
|
|
@@ -296,7 +305,7 @@ def _log_model(experiment, trainer):
|
|
|
296
305
|
def on_pretrain_routine_start(trainer):
|
|
297
306
|
"""Creates or resumes a CometML experiment at the start of a YOLO pre-training routine."""
|
|
298
307
|
experiment = comet_ml.get_global_experiment()
|
|
299
|
-
is_alive = getattr(experiment,
|
|
308
|
+
is_alive = getattr(experiment, "alive", False)
|
|
300
309
|
if not experiment or not is_alive:
|
|
301
310
|
_create_experiment(trainer.args)
|
|
302
311
|
|
|
@@ -308,17 +317,17 @@ def on_train_epoch_end(trainer):
|
|
|
308
317
|
return
|
|
309
318
|
|
|
310
319
|
metadata = _fetch_trainer_metadata(trainer)
|
|
311
|
-
curr_epoch = metadata[
|
|
312
|
-
curr_step = metadata[
|
|
320
|
+
curr_epoch = metadata["curr_epoch"]
|
|
321
|
+
curr_step = metadata["curr_step"]
|
|
313
322
|
|
|
314
323
|
experiment.log_metrics(
|
|
315
|
-
trainer.label_loss_items(trainer.tloss, prefix=
|
|
324
|
+
trainer.label_loss_items(trainer.tloss, prefix="train"),
|
|
316
325
|
step=curr_step,
|
|
317
326
|
epoch=curr_epoch,
|
|
318
327
|
)
|
|
319
328
|
|
|
320
329
|
if curr_epoch == 1:
|
|
321
|
-
_log_images(experiment, trainer.save_dir.glob(
|
|
330
|
+
_log_images(experiment, trainer.save_dir.glob("train_batch*.jpg"), curr_step)
|
|
322
331
|
|
|
323
332
|
|
|
324
333
|
def on_fit_epoch_end(trainer):
|
|
@@ -328,14 +337,15 @@ def on_fit_epoch_end(trainer):
|
|
|
328
337
|
return
|
|
329
338
|
|
|
330
339
|
metadata = _fetch_trainer_metadata(trainer)
|
|
331
|
-
curr_epoch = metadata[
|
|
332
|
-
curr_step = metadata[
|
|
333
|
-
save_assets = metadata[
|
|
340
|
+
curr_epoch = metadata["curr_epoch"]
|
|
341
|
+
curr_step = metadata["curr_step"]
|
|
342
|
+
save_assets = metadata["save_assets"]
|
|
334
343
|
|
|
335
344
|
experiment.log_metrics(trainer.metrics, step=curr_step, epoch=curr_epoch)
|
|
336
345
|
experiment.log_metrics(trainer.lr, step=curr_step, epoch=curr_epoch)
|
|
337
346
|
if curr_epoch == 1:
|
|
338
347
|
from ultralytics.utils.torch_utils import model_info_for_loggers
|
|
348
|
+
|
|
339
349
|
experiment.log_metrics(model_info_for_loggers(trainer), step=curr_step, epoch=curr_epoch)
|
|
340
350
|
|
|
341
351
|
if not save_assets:
|
|
@@ -355,8 +365,8 @@ def on_train_end(trainer):
|
|
|
355
365
|
return
|
|
356
366
|
|
|
357
367
|
metadata = _fetch_trainer_metadata(trainer)
|
|
358
|
-
curr_epoch = metadata[
|
|
359
|
-
curr_step = metadata[
|
|
368
|
+
curr_epoch = metadata["curr_epoch"]
|
|
369
|
+
curr_step = metadata["curr_step"]
|
|
360
370
|
plots = trainer.args.plots
|
|
361
371
|
|
|
362
372
|
_log_model(experiment, trainer)
|
|
@@ -371,8 +381,13 @@ def on_train_end(trainer):
|
|
|
371
381
|
_comet_image_prediction_count = 0
|
|
372
382
|
|
|
373
383
|
|
|
374
|
-
callbacks =
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
384
|
+
callbacks = (
|
|
385
|
+
{
|
|
386
|
+
"on_pretrain_routine_start": on_pretrain_routine_start,
|
|
387
|
+
"on_train_epoch_end": on_train_epoch_end,
|
|
388
|
+
"on_fit_epoch_end": on_fit_epoch_end,
|
|
389
|
+
"on_train_end": on_train_end,
|
|
390
|
+
}
|
|
391
|
+
if comet_ml
|
|
392
|
+
else {}
|
|
393
|
+
)
|
|
@@ -4,9 +4,10 @@ from ultralytics.utils import LOGGER, SETTINGS, TESTS_RUNNING, checks
|
|
|
4
4
|
|
|
5
5
|
try:
|
|
6
6
|
assert not TESTS_RUNNING # do not log pytest
|
|
7
|
-
assert SETTINGS[
|
|
7
|
+
assert SETTINGS["dvc"] is True # verify integration is enabled
|
|
8
8
|
import dvclive
|
|
9
|
-
|
|
9
|
+
|
|
10
|
+
assert checks.check_version("dvclive", "2.11.0", verbose=True)
|
|
10
11
|
|
|
11
12
|
import os
|
|
12
13
|
import re
|
|
@@ -24,24 +25,24 @@ except (ImportError, AssertionError, TypeError):
|
|
|
24
25
|
dvclive = None
|
|
25
26
|
|
|
26
27
|
|
|
27
|
-
def _log_images(path, prefix=
|
|
28
|
+
def _log_images(path, prefix=""):
|
|
28
29
|
"""Logs images at specified path with an optional prefix using DVCLive."""
|
|
29
30
|
if live:
|
|
30
31
|
name = path.name
|
|
31
32
|
|
|
32
33
|
# Group images by batch to enable sliders in UI
|
|
33
|
-
if m := re.search(r
|
|
34
|
+
if m := re.search(r"_batch(\d+)", name):
|
|
34
35
|
ni = m[1]
|
|
35
|
-
new_stem = re.sub(r
|
|
36
|
+
new_stem = re.sub(r"_batch(\d+)", "_batch", path.stem)
|
|
36
37
|
name = (Path(new_stem) / ni).with_suffix(path.suffix)
|
|
37
38
|
|
|
38
39
|
live.log_image(os.path.join(prefix, name), path)
|
|
39
40
|
|
|
40
41
|
|
|
41
|
-
def _log_plots(plots, prefix=
|
|
42
|
+
def _log_plots(plots, prefix=""):
|
|
42
43
|
"""Logs plot images for training progress if they have not been previously processed."""
|
|
43
44
|
for name, params in plots.items():
|
|
44
|
-
timestamp = params[
|
|
45
|
+
timestamp = params["timestamp"]
|
|
45
46
|
if _processed_plots.get(name) != timestamp:
|
|
46
47
|
_log_images(name, prefix)
|
|
47
48
|
_processed_plots[name] = timestamp
|
|
@@ -53,15 +54,15 @@ def _log_confusion_matrix(validator):
|
|
|
53
54
|
preds = []
|
|
54
55
|
matrix = validator.confusion_matrix.matrix
|
|
55
56
|
names = list(validator.names.values())
|
|
56
|
-
if validator.confusion_matrix.task ==
|
|
57
|
-
names += [
|
|
57
|
+
if validator.confusion_matrix.task == "detect":
|
|
58
|
+
names += ["background"]
|
|
58
59
|
|
|
59
60
|
for ti, pred in enumerate(matrix.T.astype(int)):
|
|
60
61
|
for pi, num in enumerate(pred):
|
|
61
62
|
targets.extend([names[ti]] * num)
|
|
62
63
|
preds.extend([names[pi]] * num)
|
|
63
64
|
|
|
64
|
-
live.log_sklearn_plot(
|
|
65
|
+
live.log_sklearn_plot("confusion_matrix", targets, preds, name="cf.json", normalized=True)
|
|
65
66
|
|
|
66
67
|
|
|
67
68
|
def on_pretrain_routine_start(trainer):
|
|
@@ -71,12 +72,12 @@ def on_pretrain_routine_start(trainer):
|
|
|
71
72
|
live = dvclive.Live(save_dvc_exp=True, cache_images=True)
|
|
72
73
|
LOGGER.info("DVCLive is detected and auto logging is enabled (run 'yolo settings dvc=False' to disable).")
|
|
73
74
|
except Exception as e:
|
|
74
|
-
LOGGER.warning(f
|
|
75
|
+
LOGGER.warning(f"WARNING ⚠️ DVCLive installed but not initialized correctly, not logging this run. {e}")
|
|
75
76
|
|
|
76
77
|
|
|
77
78
|
def on_pretrain_routine_end(trainer):
|
|
78
79
|
"""Logs plots related to the training process at the end of the pretraining routine."""
|
|
79
|
-
_log_plots(trainer.plots,
|
|
80
|
+
_log_plots(trainer.plots, "train")
|
|
80
81
|
|
|
81
82
|
|
|
82
83
|
def on_train_start(trainer):
|
|
@@ -95,17 +96,18 @@ def on_fit_epoch_end(trainer):
|
|
|
95
96
|
"""Logs training metrics and model info, and advances to next step on the end of each fit epoch."""
|
|
96
97
|
global _training_epoch
|
|
97
98
|
if live and _training_epoch:
|
|
98
|
-
all_metrics = {**trainer.label_loss_items(trainer.tloss, prefix=
|
|
99
|
+
all_metrics = {**trainer.label_loss_items(trainer.tloss, prefix="train"), **trainer.metrics, **trainer.lr}
|
|
99
100
|
for metric, value in all_metrics.items():
|
|
100
101
|
live.log_metric(metric, value)
|
|
101
102
|
|
|
102
103
|
if trainer.epoch == 0:
|
|
103
104
|
from ultralytics.utils.torch_utils import model_info_for_loggers
|
|
105
|
+
|
|
104
106
|
for metric, value in model_info_for_loggers(trainer).items():
|
|
105
107
|
live.log_metric(metric, value, plot=False)
|
|
106
108
|
|
|
107
|
-
_log_plots(trainer.plots,
|
|
108
|
-
_log_plots(trainer.validator.plots,
|
|
109
|
+
_log_plots(trainer.plots, "train")
|
|
110
|
+
_log_plots(trainer.validator.plots, "val")
|
|
109
111
|
|
|
110
112
|
live.next_step()
|
|
111
113
|
_training_epoch = False
|
|
@@ -115,24 +117,29 @@ def on_train_end(trainer):
|
|
|
115
117
|
"""Logs the best metrics, plots, and confusion matrix at the end of training if DVCLive is active."""
|
|
116
118
|
if live:
|
|
117
119
|
# At the end log the best metrics. It runs validator on the best model internally.
|
|
118
|
-
all_metrics = {**trainer.label_loss_items(trainer.tloss, prefix=
|
|
120
|
+
all_metrics = {**trainer.label_loss_items(trainer.tloss, prefix="train"), **trainer.metrics, **trainer.lr}
|
|
119
121
|
for metric, value in all_metrics.items():
|
|
120
122
|
live.log_metric(metric, value, plot=False)
|
|
121
123
|
|
|
122
|
-
_log_plots(trainer.plots,
|
|
123
|
-
_log_plots(trainer.validator.plots,
|
|
124
|
+
_log_plots(trainer.plots, "val")
|
|
125
|
+
_log_plots(trainer.validator.plots, "val")
|
|
124
126
|
_log_confusion_matrix(trainer.validator)
|
|
125
127
|
|
|
126
128
|
if trainer.best.exists():
|
|
127
|
-
live.log_artifact(trainer.best, copy=True, type=
|
|
129
|
+
live.log_artifact(trainer.best, copy=True, type="model")
|
|
128
130
|
|
|
129
131
|
live.end()
|
|
130
132
|
|
|
131
133
|
|
|
132
|
-
callbacks =
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
134
|
+
callbacks = (
|
|
135
|
+
{
|
|
136
|
+
"on_pretrain_routine_start": on_pretrain_routine_start,
|
|
137
|
+
"on_pretrain_routine_end": on_pretrain_routine_end,
|
|
138
|
+
"on_train_start": on_train_start,
|
|
139
|
+
"on_train_epoch_start": on_train_epoch_start,
|
|
140
|
+
"on_fit_epoch_end": on_fit_epoch_end,
|
|
141
|
+
"on_train_end": on_train_end,
|
|
142
|
+
}
|
|
143
|
+
if dvclive
|
|
144
|
+
else {}
|
|
145
|
+
)
|
|
@@ -3,57 +3,70 @@
|
|
|
3
3
|
import json
|
|
4
4
|
from time import time
|
|
5
5
|
|
|
6
|
-
from
|
|
6
|
+
from hub_sdk.config import HUB_WEB_ROOT
|
|
7
|
+
|
|
8
|
+
from ultralytics.hub.utils import PREFIX, events
|
|
7
9
|
from ultralytics.utils import LOGGER, SETTINGS
|
|
8
10
|
|
|
9
11
|
|
|
10
12
|
def on_pretrain_routine_end(trainer):
|
|
11
13
|
"""Logs info before starting timer for upload rate limit."""
|
|
12
|
-
session = getattr(trainer,
|
|
14
|
+
session = getattr(trainer, "hub_session", None)
|
|
13
15
|
if session:
|
|
14
16
|
# Start timer for upload rate limit
|
|
15
|
-
|
|
16
|
-
|
|
17
|
+
session.timers = {
|
|
18
|
+
"metrics": time(),
|
|
19
|
+
"ckpt": time(),
|
|
20
|
+
} # start timer on session.rate_limit
|
|
17
21
|
|
|
18
22
|
|
|
19
23
|
def on_fit_epoch_end(trainer):
|
|
20
24
|
"""Uploads training progress metrics at the end of each epoch."""
|
|
21
|
-
session = getattr(trainer,
|
|
25
|
+
session = getattr(trainer, "hub_session", None)
|
|
22
26
|
if session:
|
|
23
27
|
# Upload metrics after val end
|
|
24
|
-
all_plots = {
|
|
28
|
+
all_plots = {
|
|
29
|
+
**trainer.label_loss_items(trainer.tloss, prefix="train"),
|
|
30
|
+
**trainer.metrics,
|
|
31
|
+
}
|
|
25
32
|
if trainer.epoch == 0:
|
|
26
33
|
from ultralytics.utils.torch_utils import model_info_for_loggers
|
|
34
|
+
|
|
27
35
|
all_plots = {**all_plots, **model_info_for_loggers(trainer)}
|
|
36
|
+
|
|
28
37
|
session.metrics_queue[trainer.epoch] = json.dumps(all_plots)
|
|
29
|
-
if time() - session.timers[
|
|
38
|
+
if time() - session.timers["metrics"] > session.rate_limits["metrics"]:
|
|
30
39
|
session.upload_metrics()
|
|
31
|
-
session.timers[
|
|
40
|
+
session.timers["metrics"] = time() # reset timer
|
|
32
41
|
session.metrics_queue = {} # reset queue
|
|
33
42
|
|
|
34
43
|
|
|
35
44
|
def on_model_save(trainer):
|
|
36
45
|
"""Saves checkpoints to Ultralytics HUB with rate limiting."""
|
|
37
|
-
session = getattr(trainer,
|
|
46
|
+
session = getattr(trainer, "hub_session", None)
|
|
38
47
|
if session:
|
|
39
48
|
# Upload checkpoints with rate limiting
|
|
40
49
|
is_best = trainer.best_fitness == trainer.fitness
|
|
41
|
-
if time() - session.timers[
|
|
42
|
-
LOGGER.info(f
|
|
50
|
+
if time() - session.timers["ckpt"] > session.rate_limits["ckpt"]:
|
|
51
|
+
LOGGER.info(f"{PREFIX}Uploading checkpoint {HUB_WEB_ROOT}/models/{session.model_file}")
|
|
43
52
|
session.upload_model(trainer.epoch, trainer.last, is_best)
|
|
44
|
-
session.timers[
|
|
53
|
+
session.timers["ckpt"] = time() # reset timer
|
|
45
54
|
|
|
46
55
|
|
|
47
56
|
def on_train_end(trainer):
|
|
48
57
|
"""Upload final model and metrics to Ultralytics HUB at the end of training."""
|
|
49
|
-
session = getattr(trainer,
|
|
58
|
+
session = getattr(trainer, "hub_session", None)
|
|
50
59
|
if session:
|
|
51
60
|
# Upload final model and metrics with exponential standoff
|
|
52
|
-
LOGGER.info(f
|
|
53
|
-
session.upload_model(
|
|
61
|
+
LOGGER.info(f"{PREFIX}Syncing final model...")
|
|
62
|
+
session.upload_model(
|
|
63
|
+
trainer.epoch,
|
|
64
|
+
trainer.best,
|
|
65
|
+
map=trainer.metrics.get("metrics/mAP50-95(B)", 0),
|
|
66
|
+
final=True,
|
|
67
|
+
)
|
|
54
68
|
session.alive = False # stop heartbeats
|
|
55
|
-
LOGGER.info(f
|
|
56
|
-
f'{PREFIX}View model at {HUB_WEB_ROOT}/models/{session.model_id} 🚀')
|
|
69
|
+
LOGGER.info(f"{PREFIX}Done ✅\n" f"{PREFIX}View model at {session.model_url} 🚀")
|
|
57
70
|
|
|
58
71
|
|
|
59
72
|
def on_train_start(trainer):
|
|
@@ -76,12 +89,17 @@ def on_export_start(exporter):
|
|
|
76
89
|
events(exporter.args)
|
|
77
90
|
|
|
78
91
|
|
|
79
|
-
callbacks =
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
92
|
+
callbacks = (
|
|
93
|
+
{
|
|
94
|
+
"on_pretrain_routine_end": on_pretrain_routine_end,
|
|
95
|
+
"on_fit_epoch_end": on_fit_epoch_end,
|
|
96
|
+
"on_model_save": on_model_save,
|
|
97
|
+
"on_train_end": on_train_end,
|
|
98
|
+
"on_train_start": on_train_start,
|
|
99
|
+
"on_val_start": on_val_start,
|
|
100
|
+
"on_predict_start": on_predict_start,
|
|
101
|
+
"on_export_start": on_export_start,
|
|
102
|
+
}
|
|
103
|
+
if SETTINGS["hub"] is True
|
|
104
|
+
else {}
|
|
105
|
+
) # verify enabled
|