dgenerate-ultralytics-headless 8.3.182__tar.gz → 8.3.184__tar.gz

{dgenerate_ultralytics_headless-8.3.182 → dgenerate_ultralytics_headless-8.3.184}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dgenerate-ultralytics-headless
-Version: 8.3.182
+Version: 8.3.184
 Summary: Automatically built Ultralytics package with python-opencv-headless dependency instead of python-opencv
 Author-email: Glenn Jocher <glenn.jocher@ultralytics.com>, Jing Qiu <jing.qiu@ultralytics.com>
 Maintainer-email: Ultralytics <hello@ultralytics.com>

{dgenerate_ultralytics_headless-8.3.182 → dgenerate_ultralytics_headless-8.3.184}/dgenerate_ultralytics_headless.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dgenerate-ultralytics-headless
-Version: 8.3.182
+Version: 8.3.184
 Summary: Automatically built Ultralytics package with python-opencv-headless dependency instead of python-opencv
 Author-email: Glenn Jocher <glenn.jocher@ultralytics.com>, Jing Qiu <jing.qiu@ultralytics.com>
 Maintainer-email: Ultralytics <hello@ultralytics.com>

{dgenerate_ultralytics_headless-8.3.182 → dgenerate_ultralytics_headless-8.3.184}/dgenerate_ultralytics_headless.egg-info/SOURCES.txt

@@ -262,6 +262,7 @@ ultralytics/utils/errors.py
 ultralytics/utils/export.py
 ultralytics/utils/files.py
 ultralytics/utils/instance.py
+ultralytics/utils/logger.py
 ultralytics/utils/loss.py
 ultralytics/utils/metrics.py
 ultralytics/utils/ops.py
@@ -279,6 +280,7 @@ ultralytics/utils/callbacks/dvc.py
 ultralytics/utils/callbacks/hub.py
 ultralytics/utils/callbacks/mlflow.py
 ultralytics/utils/callbacks/neptune.py
+ultralytics/utils/callbacks/platform.py
 ultralytics/utils/callbacks/raytune.py
 ultralytics/utils/callbacks/tensorboard.py
 ultralytics/utils/callbacks/wb.py

{dgenerate_ultralytics_headless-8.3.182 → dgenerate_ultralytics_headless-8.3.184}/tests/test_cuda.py

@@ -67,7 +67,7 @@ def test_export_onnx_matrix(task, dynamic, int8, half, batch, simplify, nms):
         half=half,
         batch=batch,
         simplify=simplify,
-        nms=nms,
+        nms=nms and task != "obb",  # disable NMS for OBB task for now on T4 instance
         device=DEVICES[0],
     )
     YOLO(file)([SOURCE] * batch, imgsz=64 if dynamic else 32, device=DEVICES[0])  # exported model inference
@@ -163,7 +163,7 @@ def test_autobatch():
 
 
 @pytest.mark.slow
-@pytest.mark.skipif(not DEVICES, reason="No CUDA devices available")
+@pytest.mark.skipif(True, reason="Skip for now since T4 instance does not support TensorRT > 10.0")
 def test_utils_benchmarks():
     """Profile YOLO models for performance benchmarks."""
     from ultralytics.utils.benchmarks import ProfileModels

{dgenerate_ultralytics_headless-8.3.182 → dgenerate_ultralytics_headless-8.3.184}/tests/test_python.py

@@ -228,6 +228,15 @@ def test_train_scratch():
     model(SOURCE)
 
 
+@pytest.mark.skipif(not ONLINE, reason="environment is offline")
+def test_train_ndjson():
+    """Test training the YOLO model using NDJSON format dataset."""
+    model = YOLO(WEIGHTS_DIR / "yolo11n.pt")
+    model.train(
+        data="https://github.com/ultralytics/assets/releases/download/v0.0.0/coco8-ndjson.ndjson", epochs=1, imgsz=32
+    )
+
+
 @pytest.mark.parametrize("scls", [False, True])
 def test_train_pretrained(scls):
     """Test training of the YOLO model starting from a pre-trained checkpoint."""

{dgenerate_ultralytics_headless-8.3.182 → dgenerate_ultralytics_headless-8.3.184}/ultralytics/__init__.py

@@ -1,6 +1,6 @@
 # Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
 
-__version__ = "8.3.182"
+__version__ = "8.3.184"
 
 import os
 

{dgenerate_ultralytics_headless-8.3.182 → dgenerate_ultralytics_headless-8.3.184}/ultralytics/data/converter.py

@@ -1,5 +1,6 @@
 # Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
 
+import asyncio
 import json
 import random
 import shutil
@@ -12,7 +13,8 @@ import cv2
 import numpy as np
 from PIL import Image
 
-from ultralytics.utils import DATASETS_DIR, LOGGER, NUM_THREADS, TQDM
+from ultralytics.utils import DATASETS_DIR, LOGGER, NUM_THREADS, TQDM, YAML
+from ultralytics.utils.checks import check_file, check_requirements
 from ultralytics.utils.downloads import download, zip_directory
 from ultralytics.utils.files import increment_path
 
@@ -754,3 +756,113 @@ def convert_to_multispectral(path: Union[str, Path], n_channels: int = 10, repla
         multispectral = f(target_wavelengths)
         cv2.imwritemulti(str(output_path), np.clip(multispectral, 0, 255).astype(np.uint8).transpose(2, 0, 1))
         LOGGER.info(f"Converted {output_path}")
+
+
+async def convert_ndjson_to_yolo(ndjson_path: Union[str, Path], output_path: Optional[Union[str, Path]] = None) -> Path:
+    """
+    Convert NDJSON dataset format to Ultralytics YOLO11 dataset structure.
+
+    This function converts datasets stored in NDJSON (Newline Delimited JSON) format to the standard YOLO
+    format with separate directories for images and labels. It supports parallel processing for efficient
+    conversion of large datasets and can download images from URLs if they don't exist locally.
+
+    The NDJSON format consists of:
+    - First line: Dataset metadata with class names and configuration
+    - Subsequent lines: Individual image records with annotations and optional URLs
+
+    Args:
+        ndjson_path (Union[str, Path]): Path to the input NDJSON file containing dataset information.
+        output_path (Optional[Union[str, Path]], optional): Directory where the converted YOLO dataset
+            will be saved. If None, uses the parent directory of the NDJSON file. Defaults to None.
+
+    Returns:
+        (Path): Path to the generated data.yaml file that can be used for YOLO training.
+
+    Examples:
+        Convert a local NDJSON file:
+        >>> yaml_path = convert_ndjson_to_yolo("dataset.ndjson")
+        >>> print(f"Dataset converted to: {yaml_path}")
+
+        Convert with custom output directory:
+        >>> yaml_path = convert_ndjson_to_yolo("dataset.ndjson", output_path="./converted_datasets")
+
+        Use with YOLO training
+        >>> from ultralytics import YOLO
+        >>> model = YOLO("yolo11n.pt")
+        >>> model.train(data="https://github.com/ultralytics/assets/releases/download/v0.0.0/coco8-ndjson.ndjson")
+    """
+    check_requirements("aiohttp")
+    import aiohttp
+
+    ndjson_path = Path(check_file(ndjson_path))
+    output_path = Path(output_path or DATASETS_DIR)
+    with open(ndjson_path) as f:
+        lines = [json.loads(line.strip()) for line in f if line.strip()]
+
+    dataset_record, image_records = lines[0], lines[1:]
+    dataset_dir = output_path / ndjson_path.stem
+    splits = {record["split"] for record in image_records}
+
+    # Create directories and prepare YAML structure
+    dataset_dir.mkdir(parents=True, exist_ok=True)
+    data_yaml = dict(dataset_record)
+    data_yaml["names"] = {int(k): v for k, v in dataset_record.get("class_names", {}).items()}
+    data_yaml.pop("class_names")
+
+    for split in sorted(splits):
+        (dataset_dir / "images" / split).mkdir(parents=True, exist_ok=True)
+        (dataset_dir / "labels" / split).mkdir(parents=True, exist_ok=True)
+        data_yaml[split] = f"images/{split}"
+
+    async def process_record(session, semaphore, record):
+        """Process single image record with async session."""
+        async with semaphore:
+            split, original_name = record["split"], record["file"]
+            label_path = dataset_dir / "labels" / split / f"{Path(original_name).stem}.txt"
+            image_path = dataset_dir / "images" / split / original_name
+
+            annotations = record.get("annotations", {})
+            lines_to_write = []
+            for key in annotations.keys():
+                lines_to_write = [" ".join(map(str, item)) for item in annotations[key]]
+                break
+            if "classification" in annotations:
+                lines_to_write = [str(cls) for cls in annotations["classification"]]
+
+            label_path.write_text("\n".join(lines_to_write) + "\n" if lines_to_write else "")
+
+            if http_url := record.get("url"):
+                if not image_path.exists():
+                    try:
+                        async with session.get(http_url, timeout=aiohttp.ClientTimeout(total=30)) as response:
+                            response.raise_for_status()
+                            with open(image_path, "wb") as f:
+                                async for chunk in response.content.iter_chunked(8192):
+                                    f.write(chunk)
+                        return True
+                    except Exception as e:
+                        LOGGER.warning(f"Failed to download {http_url}: {e}")
+                        return False
+            return True
+
+    # Process all images with async downloads
+    semaphore = asyncio.Semaphore(64)
+    async with aiohttp.ClientSession() as session:
+        pbar = TQDM(
+            total=len(image_records),
+            desc=f"Converting {ndjson_path.name} → {dataset_dir} ({len(image_records)} images)",
+        )
+
+        async def tracked_process(record):
+            result = await process_record(session, semaphore, record)
+            pbar.update(1)
+            return result
+
+        await asyncio.gather(*[tracked_process(record) for record in image_records])
+        pbar.close()
+
+    # Write data.yaml
+    yaml_path = dataset_dir / "data.yaml"
+    YAML.save(yaml_path, data_yaml)
+
+    return yaml_path

{dgenerate_ultralytics_headless-8.3.182 → dgenerate_ultralytics_headless-8.3.184}/ultralytics/engine/trainer.py

@@ -174,6 +174,8 @@ class BaseTrainer:
         self.callbacks = _callbacks or callbacks.get_default_callbacks()
         if RANK in {-1, 0}:
             callbacks.add_integration_callbacks(self)
+            # Start console logging immediately at trainer initialization
+            self.run_callbacks("on_pretrain_routine_start")
 
     def add_callback(self, event: str, callback):
         """Append the given callback to the event's callback list."""
@@ -249,8 +251,6 @@ class BaseTrainer:
 
     def _setup_train(self, world_size):
         """Build dataloaders and optimizer on correct rank process."""
-        # Model
-        self.run_callbacks("on_pretrain_routine_start")
         ckpt = self.setup_model()
         self.model = self.model.to(self.device)
         self.set_model_attributes()
@@ -598,6 +598,15 @@ class BaseTrainer:
         try:
             if self.args.task == "classify":
                 data = check_cls_dataset(self.args.data)
+            elif self.args.data.rsplit(".", 1)[-1] == "ndjson":
+                # Convert NDJSON to YOLO format
+                import asyncio
+
+                from ultralytics.data.converter import convert_ndjson_to_yolo
+
+                yaml_path = asyncio.run(convert_ndjson_to_yolo(self.args.data))
+                self.args.data = str(yaml_path)
+                data = check_det_dataset(self.args.data)
             elif self.args.data.rsplit(".", 1)[-1] in {"yaml", "yml"} or self.args.task in {
                 "detect",
                 "segment",

{dgenerate_ultralytics_headless-8.3.182 → dgenerate_ultralytics_headless-8.3.184}/ultralytics/models/yolo/yoloe/val.py

@@ -181,7 +181,7 @@ class YOLOEDetectValidator(DetectionValidator):
         else:
             if refer_data is not None:
                 assert load_vp, "Refer data is only used for visual prompt validation."
-            self.device = select_device(self.args.device)
+            self.device = select_device(self.args.device, verbose=False)
 
             if isinstance(model, (str, Path)):
                 from ultralytics.nn.tasks import attempt_load_weights

{dgenerate_ultralytics_headless-8.3.182 → dgenerate_ultralytics_headless-8.3.184}/ultralytics/utils/__init__.py

@@ -170,6 +170,7 @@ class TQDM(rich.tqdm if TQDM_RICH else tqdm.tqdm):
         Notes:
             - The progress bar is disabled if VERBOSE is False or if 'disable' is explicitly set to True in kwargs.
             - The default bar format is set to TQDM_BAR_FORMAT unless overridden in kwargs.
+            - In GitHub Actions, progress bars only update at completion to keep CI logs clean.
 
         Examples:
             >>> from ultralytics.utils import TQDM
@@ -178,6 +179,8 @@ class TQDM(rich.tqdm if TQDM_RICH else tqdm.tqdm):
             ...     pass
         """
         warnings.filterwarnings("ignore", category=tqdm.TqdmExperimentalWarning)  # suppress tqdm.rich warning
+        if is_github_action_running():
+            kwargs["mininterval"] = 60  # only update every 60 seconds
         kwargs["disable"] = not VERBOSE or kwargs.get("disable", False) or LOGGER.getEffectiveLevel() > 20
         kwargs.setdefault("bar_format", TQDM_BAR_FORMAT)  # override default value if passed
         super().__init__(*args, **kwargs)

{dgenerate_ultralytics_headless-8.3.182 → dgenerate_ultralytics_headless-8.3.184}/ultralytics/utils/callbacks/base.py

@@ -209,10 +209,11 @@ def add_integration_callbacks(instance):
         >>> trainer = BaseTrainer()
         >>> add_integration_callbacks(trainer)
     """
-    # Load HUB callbacks
     from .hub import callbacks as hub_cb
+    from .platform import callbacks as platform_cb
 
-    callbacks_list = [hub_cb]
+    # Load Ultralytics callbacks
+    callbacks_list = [hub_cb, platform_cb]
 
     # Load training callbacks
     if "Trainer" in instance.__class__.__name__:

{dgenerate_ultralytics_headless-8.3.182 → dgenerate_ultralytics_headless-8.3.184}/ultralytics/utils/callbacks/hub.py

@@ -106,4 +106,4 @@ callbacks = (
     }
     if SETTINGS["hub"] is True
     else {}
-)  # verify hub is enabled before registering callbacks
+)

dgenerate_ultralytics_headless-8.3.184/ultralytics/utils/callbacks/platform.py

@@ -0,0 +1,72 @@
+# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
+
+from ultralytics.utils import RANK, SETTINGS
+from ultralytics.utils.logger import DEFAULT_LOG_PATH, ConsoleLogger, SystemLogger
+
+
+def on_pretrain_routine_start(trainer):
+    """Initialize and start console logging immediately at the very beginning."""
+    if RANK in {-1, 0}:
+        trainer.system_logger = SystemLogger()
+        trainer.console_logger = ConsoleLogger(DEFAULT_LOG_PATH)
+        trainer.console_logger.start_capture()
+
+
+def on_pretrain_routine_end(trainer):
+    """Handle pre-training routine completion event."""
+    pass
+
+
+def on_fit_epoch_end(trainer):
+    """Handle end of training epoch event and collect system metrics."""
+    if RANK in {-1, 0} and hasattr(trainer, "system_logger"):
+        system_metrics = trainer.system_logger.get_metrics()
+        print(system_metrics)  # for debug
+
+
+def on_model_save(trainer):
+    """Handle model checkpoint save event."""
+    pass
+
+
+def on_train_end(trainer):
+    """Stop console capture and finalize logs."""
+    if logger := getattr(trainer, "console_logger", None):
+        logger.stop_capture()
+
+
+def on_train_start(trainer):
+    """Handle training start event."""
+    pass
+
+
+def on_val_start(validator):
+    """Handle validation start event."""
+    pass
+
+
+def on_predict_start(predictor):
+    """Handle prediction start event."""
+    pass
+
+
+def on_export_start(exporter):
+    """Handle model export start event."""
+    pass
+
+
+callbacks = (
+    {
+        "on_pretrain_routine_start": on_pretrain_routine_start,
+        "on_pretrain_routine_end": on_pretrain_routine_end,
+        "on_fit_epoch_end": on_fit_epoch_end,
+        "on_model_save": on_model_save,
+        "on_train_end": on_train_end,
+        "on_train_start": on_train_start,
+        "on_val_start": on_val_start,
+        "on_predict_start": on_predict_start,
+        "on_export_start": on_export_start,
+    }
+    if SETTINGS.get("platform", False) is True  # disabled for debugging
+    else {}
+)