ultralytics 8.3.212__tar.gz → 8.3.213__tar.gz

{ultralytics-8.3.212 → ultralytics-8.3.213}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ultralytics
-Version: 8.3.212
+Version: 8.3.213
 Summary: Ultralytics YOLO 🚀 for SOTA object detection, multi-object tracking, instance segmentation, pose estimation and image classification.
 Author-email: Glenn Jocher <glenn.jocher@ultralytics.com>, Jing Qiu <jing.qiu@ultralytics.com>
 Maintainer-email: Ultralytics <hello@ultralytics.com>

{ultralytics-8.3.212 → ultralytics-8.3.213}/tests/test_engine.py

@@ -3,6 +3,8 @@
 import sys
 from unittest import mock
 
+import torch
+
 from tests import MODEL
 from ultralytics import YOLO
 from ultralytics.cfg import get_cfg
@@ -136,3 +138,20 @@ def test_classify():
     assert test_func in pred.callbacks["on_predict_start"], "callback test failed"
     result = pred(source=ASSETS, model=trainer.best)
     assert len(result), "predictor test failed"
+
+
+def test_nan_recovery():
+    """Test NaN loss detection and recovery during training."""
+    nan_injected = [False]
+
+    def inject_nan(trainer):
+        """Inject NaN into loss during batch processing to test recovery mechanism."""
+        if trainer.epoch == 1 and trainer.tloss is not None and not nan_injected[0]:
+            trainer.tloss *= torch.tensor(float("nan"))
+            nan_injected[0] = True
+
+    overrides = {"data": "coco8.yaml", "model": "yolo11n.yaml", "imgsz": 32, "epochs": 3}
+    trainer = detect.DetectionTrainer(overrides=overrides)
+    trainer.add_callback("on_train_batch_end", inject_nan)
+    trainer.train()
+    assert nan_injected[0], "NaN injection failed"

{ultralytics-8.3.212 → ultralytics-8.3.213}/tests/test_python.py

@@ -23,6 +23,8 @@ from ultralytics.utils import (
     ASSETS_URL,
     DEFAULT_CFG,
     DEFAULT_CFG_PATH,
+    IS_JETSON,
+    IS_RASPBERRYPI,
     LINUX,
     LOGGER,
     ONLINE,
@@ -217,6 +219,7 @@ def test_val(task: str, weight: str, data: str) -> None:
         metrics.confusion_matrix.to_json()
 
 
+@pytest.mark.skipif(IS_JETSON or IS_RASPBERRYPI, reason="Edge devices not intended for training")
 def test_train_scratch():
     """Test training the YOLO model from scratch using the provided configuration."""
     model = YOLO(CFG)

{ultralytics-8.3.212 → ultralytics-8.3.213}/ultralytics/__init__.py

@@ -1,6 +1,6 @@
 # Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
 
-__version__ = "8.3.212"
+__version__ = "8.3.213"
 
 import importlib
 import os

{ultralytics-8.3.212 → ultralytics-8.3.213}/ultralytics/cfg/datasets/Objects365.yaml

@@ -384,6 +384,7 @@ names:
 
 # Download script/URL (optional) ---------------------------------------------------------------------------------------
 download: |
+  from concurrent.futures import ThreadPoolExecutor
   from pathlib import Path
 
   import numpy as np
@@ -396,31 +397,28 @@ download: |
   check_requirements("faster-coco-eval")
   from faster_coco_eval import COCO
 
-  # Make Directories
-  dir = Path(yaml["path"])  # dataset root dir
-  for p in "images", "labels":
-      (dir / p).mkdir(parents=True, exist_ok=True)
-      for q in "train", "val":
-          (dir / p / q).mkdir(parents=True, exist_ok=True)
-
   # Train, Val Splits
+  dir = Path(yaml["path"])
   for split, patches in [("train", 50 + 1), ("val", 43 + 1)]:
       print(f"Processing {split} in {patches} patches ...")
       images, labels = dir / "images" / split, dir / "labels" / split
+      images.mkdir(parents=True, exist_ok=True)
+      labels.mkdir(parents=True, exist_ok=True)
 
       # Download
       url = f"https://dorc.ks3-cn-beijing.ksyun.com/data-set/2020Objects365%E6%95%B0%E6%8D%AE%E9%9B%86/{split}/"
       if split == "train":
           download([f"{url}zhiyuan_objv2_{split}.tar.gz"], dir=dir)  # annotations json
-          download([f"{url}patch{i}.tar.gz" for i in range(patches)], dir=images, curl=True, threads=8)
+          download([f"{url}patch{i}.tar.gz" for i in range(patches)], dir=images, threads=17)  # 51 patches / 17 threads = 3
       elif split == "val":
           download([f"{url}zhiyuan_objv2_{split}.json"], dir=dir)  # annotations json
-          download([f"{url}images/v1/patch{i}.tar.gz" for i in range(15 + 1)], dir=images, curl=True, threads=8)
-          download([f"{url}images/v2/patch{i}.tar.gz" for i in range(16, patches)], dir=images, curl=True, threads=8)
+          download([f"{url}images/v1/patch{i}.tar.gz" for i in range(15 + 1)], dir=images, threads=16)
+          download([f"{url}images/v2/patch{i}.tar.gz" for i in range(16, patches)], dir=images, threads=16)
 
       # Move
-      for f in TQDM(images.rglob("*.jpg"), desc=f"Moving {split} images"):
-          f.rename(images / f.name)  # move to /images/{split}
+      files = list(images.rglob("*.jpg"))
+      with ThreadPoolExecutor(max_workers=16) as executor:
+          list(TQDM(executor.map(lambda f: f.rename(images / f.name), files), total=len(files), desc=f"Moving {split} images"))
 
       # Labels
       coco = COCO(dir / f"zhiyuan_objv2_{split}.json")
@@ -428,10 +426,12 @@ download: |
       for cid, cat in enumerate(names):
           catIds = coco.getCatIds(catNms=[cat])
           imgIds = coco.getImgIds(catIds=catIds)
-          for im in TQDM(coco.loadImgs(imgIds), desc=f"Class {cid + 1}/{len(names)} {cat}"):
-              width, height = im["width"], im["height"]
-              path = Path(im["file_name"])  # image filename
+
+          def process_annotation(im):
+              """Process and write annotations for a single image."""
               try:
+                  width, height = im["width"], im["height"]
+                  path = Path(im["file_name"])
                   with open(labels / path.with_suffix(".txt").name, "a", encoding="utf-8") as file:
                       annIds = coco.getAnnIds(imgIds=im["id"], catIds=catIds, iscrowd=None)
                       for a in coco.loadAnns(annIds):
@@ -441,3 +441,7 @@ download: |
                           file.write(f"{cid} {x:.5f} {y:.5f} {w:.5f} {h:.5f}\n")
               except Exception as e:
                   print(e)
+
+          images_list = coco.loadImgs(imgIds)
+          with ThreadPoolExecutor(max_workers=16) as executor:
+              list(TQDM(executor.map(process_annotation, images_list), total=len(images_list), desc=f"Class {cid + 1}/{len(names)} {cat}"))

{ultralytics-8.3.212 → ultralytics-8.3.213}/ultralytics/cfg/datasets/VOC.yaml

@@ -49,7 +49,7 @@ download: |
   from pathlib import Path
 
   from ultralytics.utils.downloads import download
-  from ultralytics.utils import TQDM
+  from ultralytics.utils import ASSETS_URL, TQDM
 
   def convert_label(path, lb_path, year, image_id):
       """Converts XML annotations from VOC format to YOLO format by extracting bounding boxes and class IDs."""
@@ -79,11 +79,10 @@ download: |
 
   # Download
   dir = Path(yaml["path"])  # dataset root dir
-  url = "https://github.com/ultralytics/assets/releases/download/v0.0.0/"
   urls = [
-      f"{url}VOCtrainval_06-Nov-2007.zip",  # 446MB, 5012 images
-      f"{url}VOCtest_06-Nov-2007.zip",  # 438MB, 4953 images
-      f"{url}VOCtrainval_11-May-2012.zip",  # 1.95GB, 17126 images
+      f"{ASSETS_URL}/VOCtrainval_06-Nov-2007.zip",  # 446MB, 5012 images
+      f"{ASSETS_URL}/VOCtest_06-Nov-2007.zip",  # 438MB, 4953 images
+      f"{ASSETS_URL}/VOCtrainval_11-May-2012.zip",  # 1.95GB, 17126 images
   ]
   download(urls, dir=dir / "images", threads=3, exist_ok=True)  # download and unzip over existing (required)
 

{ultralytics-8.3.212 → ultralytics-8.3.213}/ultralytics/cfg/datasets/VisDrone.yaml

@@ -34,7 +34,7 @@ download: |
   import shutil
 
   from ultralytics.utils.downloads import download
-  from ultralytics.utils import TQDM
+  from ultralytics.utils import ASSETS_URL, TQDM
 
 
   def visdrone2yolo(dir, split, source_name=None):
@@ -73,10 +73,10 @@ download: |
   # Download (ignores test-challenge split)
   dir = Path(yaml["path"])  # dataset root dir
   urls = [
-      "https://github.com/ultralytics/assets/releases/download/v0.0.0/VisDrone2019-DET-train.zip",
-      "https://github.com/ultralytics/assets/releases/download/v0.0.0/VisDrone2019-DET-val.zip",
-      "https://github.com/ultralytics/assets/releases/download/v0.0.0/VisDrone2019-DET-test-dev.zip",
-      # "https://github.com/ultralytics/assets/releases/download/v0.0.0/VisDrone2019-DET-test-challenge.zip",
+      f"{ASSETS_URL}/VisDrone2019-DET-train.zip",
+      f"{ASSETS_URL}/VisDrone2019-DET-val.zip",
+      f"{ASSETS_URL}/VisDrone2019-DET-test-dev.zip",
+      # f"{ASSETS_URL}/VisDrone2019-DET-test-challenge.zip",
   ]
   download(urls, dir=dir, threads=4)
 

{ultralytics-8.3.212 → ultralytics-8.3.213}/ultralytics/cfg/datasets/coco-pose.yaml

@@ -26,12 +26,13 @@ names:
 download: |
   from pathlib import Path
 
+  from ultralytics.utils import ASSETS_URL
   from ultralytics.utils.downloads import download
 
   # Download labels
   dir = Path(yaml["path"])  # dataset root dir
-  url = "https://github.com/ultralytics/assets/releases/download/v0.0.0/"
-  urls = [f"{url}coco2017labels-pose.zip"]
+
+  urls = [f"{ASSETS_URL}/coco2017labels-pose.zip"]
   download(urls, dir=dir.parent)
   # Download data
   urls = [

{ultralytics-8.3.212 → ultralytics-8.3.213}/ultralytics/cfg/datasets/coco.yaml

@@ -101,13 +101,13 @@ names:
 download: |
   from pathlib import Path
 
+  from ultralytics.utils import ASSETS_URL
   from ultralytics.utils.downloads import download
 
   # Download labels
   segments = True  # segment or box labels
   dir = Path(yaml["path"])  # dataset root dir
-  url = "https://github.com/ultralytics/assets/releases/download/v0.0.0/"
-  urls = [url + ("coco2017labels-segments.zip" if segments else "coco2017labels.zip")]  # labels
+  urls = [ASSETS_URL + ("/coco2017labels-segments.zip" if segments else "/coco2017labels.zip")]  # labels
   download(urls, dir=dir.parent)
   # Download data
   urls = [

{ultralytics-8.3.212 → ultralytics-8.3.213}/ultralytics/cfg/datasets/lvis.yaml

@@ -1223,12 +1223,12 @@ names:
 download: |
   from pathlib import Path
 
+  from ultralytics.utils import ASSETS_URL
   from ultralytics.utils.downloads import download
 
   # Download labels
   dir = Path(yaml["path"])  # dataset root dir
-  url = "https://github.com/ultralytics/assets/releases/download/v0.0.0/"
-  urls = [f"{url}lvis-labels-segments.zip"]
+  urls = [f"{ASSETS_URL}/lvis-labels-segments.zip"]
   download(urls, dir=dir.parent)
 
   # Download data

{ultralytics-8.3.212 → ultralytics-8.3.213}/ultralytics/engine/trainer.py

@@ -171,6 +171,7 @@ class BaseTrainer:
         self.loss_names = ["Loss"]
         self.csv = self.save_dir / "results.csv"
         self.plot_idx = [0, 1, 2]
+        self.nan_recovery_attempts = 0
 
         # Callbacks
         self.callbacks = _callbacks or callbacks.get_default_callbacks()
@@ -458,6 +459,7 @@ class BaseTrainer:
                 self.run_callbacks("on_train_batch_end")
 
             self.lr = {f"lr/pg{ir}": x["lr"] for ir, x in enumerate(self.optimizer.param_groups)}  # for loggers
+
             self.run_callbacks("on_train_epoch_end")
             if RANK in {-1, 0}:
                 final_epoch = epoch + 1 >= self.epochs
@@ -467,6 +469,13 @@ class BaseTrainer:
                 if self.args.val or final_epoch or self.stopper.possible_stop or self.stop:
                     self._clear_memory(threshold=0.5)  # prevent VRAM spike
                     self.metrics, self.fitness = self.validate()
+
+            # NaN recovery
+            if self._handle_nan_recovery(epoch):
+                continue
+
+            self.nan_recovery_attempts = 0
+            if RANK in {-1, 0}:
                 self.save_metrics(metrics={**self.label_loss_items(self.tloss), **self.metrics, **self.lr})
                 self.stop |= self.stopper(epoch + 1, self.fitness) or final_epoch
                 if self.args.time:
@@ -804,20 +813,52 @@ class BaseTrainer:
                 ) from e
         self.resume = resume
 
-    def resume_training(self, ckpt):
-        """Resume YOLO training from given epoch and best fitness."""
-        if ckpt is None or not self.resume:
-            return
-        best_fitness = 0.0
-        start_epoch = ckpt.get("epoch", -1) + 1
+    def _load_checkpoint_state(self, ckpt):
+        """Load optimizer, scaler, EMA, and best_fitness from checkpoint."""
         if ckpt.get("optimizer") is not None:
-            self.optimizer.load_state_dict(ckpt["optimizer"])  # optimizer
-            best_fitness = ckpt["best_fitness"]
+            self.optimizer.load_state_dict(ckpt["optimizer"])
         if ckpt.get("scaler") is not None:
             self.scaler.load_state_dict(ckpt["scaler"])
         if self.ema and ckpt.get("ema"):
-            self.ema.ema.load_state_dict(ckpt["ema"].float().state_dict())  # EMA
+            self.ema.ema.load_state_dict(ckpt["ema"].float().state_dict())
             self.ema.updates = ckpt["updates"]
+        self.best_fitness = ckpt.get("best_fitness", 0.0)
+
+    def _handle_nan_recovery(self, epoch):
+        """Detect and recover from NaN/Inf loss or fitness collapse by loading last checkpoint."""
+        loss_nan = self.tloss is not None and not torch.isfinite(self.tloss).all()
+        fitness_nan = self.fitness is not None and not np.isfinite(self.fitness)
+        fitness_collapse = self.best_fitness and self.best_fitness > 0 and self.fitness == 0
+        corrupted = RANK in {-1, 0} and (loss_nan or fitness_nan or fitness_collapse)
+        reason = "Loss NaN/Inf" if loss_nan else "Fitness NaN/Inf" if fitness_nan else "Fitness collapse"
+        if RANK != -1:  # DDP: broadcast to all ranks
+            broadcast_list = [corrupted if RANK == 0 else None]
+            dist.broadcast_object_list(broadcast_list, 0)
+            corrupted = broadcast_list[0]
+        if not corrupted:
+            return False
+        if epoch == self.start_epoch or not self.last.exists():
+            LOGGER.warning(f"{reason} detected but can not recover from last.pt...")
+            return False  # Cannot recover on first epoch, let training continue
+        self.nan_recovery_attempts += 1
+        if self.nan_recovery_attempts > 3:
+            raise RuntimeError(f"Training failed: NaN persisted for {self.nan_recovery_attempts} epochs")
+        LOGGER.warning(f"{reason} detected (attempt {self.nan_recovery_attempts}/3), recovering from last.pt...")
+        _, ckpt = load_checkpoint(self.last)
+        ema_state = ckpt["ema"].float().state_dict()
+        if not all(torch.isfinite(v).all() for v in ema_state.values() if isinstance(v, torch.Tensor)):
+            raise RuntimeError(f"Checkpoint {self.last} is corrupted with NaN/Inf weights")
+        unwrap_model(self.model).load_state_dict(ema_state)  # Load EMA weights into model
+        self._load_checkpoint_state(ckpt)  # Load optimizer/scaler/EMA/best_fitness
+        del ckpt, ema_state
+        self.scheduler.last_epoch = epoch - 1
+        return True
+
+    def resume_training(self, ckpt):
+        """Resume YOLO training from given epoch and best fitness."""
+        if ckpt is None or not self.resume:
+            return
+        start_epoch = ckpt.get("epoch", -1) + 1
         assert start_epoch > 0, (
             f"{self.args.model} training to {self.epochs} epochs is finished, nothing to resume.\n"
             f"Start a new training without resuming, i.e. 'yolo train model={self.args.model}'"
@@ -828,7 +869,7 @@ class BaseTrainer:
                 f"{self.model} has been trained for {ckpt['epoch']} epochs. Fine-tuning for {self.epochs} more epochs."
             )
             self.epochs += ckpt["epoch"]  # finetune additional epochs
-        self.best_fitness = best_fitness
+        self._load_checkpoint_state(ckpt)
         self.start_epoch = start_epoch
         if start_epoch > (self.epochs - self.args.close_mosaic):
             self._close_dataloader_mosaic()

{ultralytics-8.3.212 → ultralytics-8.3.213}/ultralytics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ultralytics
-Version: 8.3.212
+Version: 8.3.213
 Summary: Ultralytics YOLO 🚀 for SOTA object detection, multi-object tracking, instance segmentation, pose estimation and image classification.
 Author-email: Glenn Jocher <glenn.jocher@ultralytics.com>, Jing Qiu <jing.qiu@ultralytics.com>
 Maintainer-email: Ultralytics <hello@ultralytics.com>