dgenerate-ultralytics-headless 8.3.183__tar.gz → 8.3.184__tar.gz

{dgenerate_ultralytics_headless-8.3.183 → dgenerate_ultralytics_headless-8.3.184}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dgenerate-ultralytics-headless
-Version: 8.3.183
+Version: 8.3.184
 Summary: Automatically built Ultralytics package with python-opencv-headless dependency instead of python-opencv
 Author-email: Glenn Jocher <glenn.jocher@ultralytics.com>, Jing Qiu <jing.qiu@ultralytics.com>
 Maintainer-email: Ultralytics <hello@ultralytics.com>

{dgenerate_ultralytics_headless-8.3.183 → dgenerate_ultralytics_headless-8.3.184}/dgenerate_ultralytics_headless.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dgenerate-ultralytics-headless
-Version: 8.3.183
+Version: 8.3.184
 Summary: Automatically built Ultralytics package with python-opencv-headless dependency instead of python-opencv
 Author-email: Glenn Jocher <glenn.jocher@ultralytics.com>, Jing Qiu <jing.qiu@ultralytics.com>
 Maintainer-email: Ultralytics <hello@ultralytics.com>

{dgenerate_ultralytics_headless-8.3.183 → dgenerate_ultralytics_headless-8.3.184}/dgenerate_ultralytics_headless.egg-info/SOURCES.txt

@@ -262,6 +262,7 @@ ultralytics/utils/errors.py
 ultralytics/utils/export.py
 ultralytics/utils/files.py
 ultralytics/utils/instance.py
+ultralytics/utils/logger.py
 ultralytics/utils/loss.py
 ultralytics/utils/metrics.py
 ultralytics/utils/ops.py
@@ -279,6 +280,7 @@ ultralytics/utils/callbacks/dvc.py
 ultralytics/utils/callbacks/hub.py
 ultralytics/utils/callbacks/mlflow.py
 ultralytics/utils/callbacks/neptune.py
+ultralytics/utils/callbacks/platform.py
 ultralytics/utils/callbacks/raytune.py
 ultralytics/utils/callbacks/tensorboard.py
 ultralytics/utils/callbacks/wb.py

{dgenerate_ultralytics_headless-8.3.183 → dgenerate_ultralytics_headless-8.3.184}/ultralytics/__init__.py

@@ -1,6 +1,6 @@
 # Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
 
-__version__ = "8.3.183"
+__version__ = "8.3.184"
 
 import os
 

{dgenerate_ultralytics_headless-8.3.183 → dgenerate_ultralytics_headless-8.3.184}/ultralytics/data/converter.py

@@ -786,7 +786,7 @@ async def convert_ndjson_to_yolo(ndjson_path: Union[str, Path], output_path: Opt
         Convert with custom output directory:
         >>> yaml_path = convert_ndjson_to_yolo("dataset.ndjson", output_path="./converted_datasets")
 
-        # Use with YOLO training
+        Use with YOLO training
         >>> from ultralytics import YOLO
         >>> model = YOLO("yolo11n.pt")
         >>> model.train(data="https://github.com/ultralytics/assets/releases/download/v0.0.0/coco8-ndjson.ndjson")
@@ -807,6 +807,7 @@ async def convert_ndjson_to_yolo(ndjson_path: Union[str, Path], output_path: Opt
     dataset_dir.mkdir(parents=True, exist_ok=True)
     data_yaml = dict(dataset_record)
     data_yaml["names"] = {int(k): v for k, v in dataset_record.get("class_names", {}).items()}
+    data_yaml.pop("class_names")
 
     for split in sorted(splits):
         (dataset_dir / "images" / split).mkdir(parents=True, exist_ok=True)

{dgenerate_ultralytics_headless-8.3.183 → dgenerate_ultralytics_headless-8.3.184}/ultralytics/engine/trainer.py

@@ -174,6 +174,8 @@ class BaseTrainer:
         self.callbacks = _callbacks or callbacks.get_default_callbacks()
         if RANK in {-1, 0}:
             callbacks.add_integration_callbacks(self)
+            # Start console logging immediately at trainer initialization
+            self.run_callbacks("on_pretrain_routine_start")
 
     def add_callback(self, event: str, callback):
         """Append the given callback to the event's callback list."""
@@ -249,8 +251,6 @@ class BaseTrainer:
 
     def _setup_train(self, world_size):
         """Build dataloaders and optimizer on correct rank process."""
-        # Model
-        self.run_callbacks("on_pretrain_routine_start")
         ckpt = self.setup_model()
         self.model = self.model.to(self.device)
         self.set_model_attributes()

{dgenerate_ultralytics_headless-8.3.183 → dgenerate_ultralytics_headless-8.3.184}/ultralytics/utils/__init__.py

@@ -170,6 +170,7 @@ class TQDM(rich.tqdm if TQDM_RICH else tqdm.tqdm):
         Notes:
             - The progress bar is disabled if VERBOSE is False or if 'disable' is explicitly set to True in kwargs.
             - The default bar format is set to TQDM_BAR_FORMAT unless overridden in kwargs.
+            - In GitHub Actions, progress bars only update at completion to keep CI logs clean.
 
         Examples:
             >>> from ultralytics.utils import TQDM
@@ -178,6 +179,8 @@ class TQDM(rich.tqdm if TQDM_RICH else tqdm.tqdm):
             ...     pass
         """
         warnings.filterwarnings("ignore", category=tqdm.TqdmExperimentalWarning)  # suppress tqdm.rich warning
+        if is_github_action_running():
+            kwargs["mininterval"] = 60  # only update every 60 seconds
         kwargs["disable"] = not VERBOSE or kwargs.get("disable", False) or LOGGER.getEffectiveLevel() > 20
         kwargs.setdefault("bar_format", TQDM_BAR_FORMAT)  # override default value if passed
         super().__init__(*args, **kwargs)

{dgenerate_ultralytics_headless-8.3.183 → dgenerate_ultralytics_headless-8.3.184}/ultralytics/utils/callbacks/base.py

@@ -209,10 +209,11 @@ def add_integration_callbacks(instance):
         >>> trainer = BaseTrainer()
         >>> add_integration_callbacks(trainer)
     """
-    # Load HUB callbacks
     from .hub import callbacks as hub_cb
+    from .platform import callbacks as platform_cb
 
-    callbacks_list = [hub_cb]
+    # Load Ultralytics callbacks
+    callbacks_list = [hub_cb, platform_cb]
 
     # Load training callbacks
     if "Trainer" in instance.__class__.__name__:

{dgenerate_ultralytics_headless-8.3.183 → dgenerate_ultralytics_headless-8.3.184}/ultralytics/utils/callbacks/hub.py

@@ -106,4 +106,4 @@ callbacks = (
     }
     if SETTINGS["hub"] is True
     else {}
-)  # verify hub is enabled before registering callbacks
+)

dgenerate_ultralytics_headless-8.3.184/ultralytics/utils/callbacks/platform.py

@@ -0,0 +1,72 @@
+# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
+
+from ultralytics.utils import RANK, SETTINGS
+from ultralytics.utils.logger import DEFAULT_LOG_PATH, ConsoleLogger, SystemLogger
+
+
+def on_pretrain_routine_start(trainer):
+    """Initialize and start console logging immediately at the very beginning."""
+    if RANK in {-1, 0}:
+        trainer.system_logger = SystemLogger()
+        trainer.console_logger = ConsoleLogger(DEFAULT_LOG_PATH)
+        trainer.console_logger.start_capture()
+
+
+def on_pretrain_routine_end(trainer):
+    """Handle pre-training routine completion event."""
+    pass
+
+
+def on_fit_epoch_end(trainer):
+    """Handle end of training epoch event and collect system metrics."""
+    if RANK in {-1, 0} and hasattr(trainer, "system_logger"):
+        system_metrics = trainer.system_logger.get_metrics()
+        print(system_metrics)  # for debug
+
+
+def on_model_save(trainer):
+    """Handle model checkpoint save event."""
+    pass
+
+
+def on_train_end(trainer):
+    """Stop console capture and finalize logs."""
+    if logger := getattr(trainer, "console_logger", None):
+        logger.stop_capture()
+
+
+def on_train_start(trainer):
+    """Handle training start event."""
+    pass
+
+
+def on_val_start(validator):
+    """Handle validation start event."""
+    pass
+
+
+def on_predict_start(predictor):
+    """Handle prediction start event."""
+    pass
+
+
+def on_export_start(exporter):
+    """Handle model export start event."""
+    pass
+
+
+callbacks = (
+    {
+        "on_pretrain_routine_start": on_pretrain_routine_start,
+        "on_pretrain_routine_end": on_pretrain_routine_end,
+        "on_fit_epoch_end": on_fit_epoch_end,
+        "on_model_save": on_model_save,
+        "on_train_end": on_train_end,
+        "on_train_start": on_train_start,
+        "on_val_start": on_val_start,
+        "on_predict_start": on_predict_start,
+        "on_export_start": on_export_start,
+    }
+    if SETTINGS.get("platform", False) is True  # disabled for debugging
+    else {}
+)

dgenerate_ultralytics_headless-8.3.184/ultralytics/utils/logger.py

@@ -0,0 +1,408 @@
+# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
+
+import logging
+import queue
+import shutil
+import sys
+import threading
+import time
+from datetime import datetime
+from pathlib import Path
+
+import psutil
+import requests
+
+from ultralytics.utils import MACOS, RANK
+from ultralytics.utils.checks import check_requirements
+
+# Initialize default log file
+DEFAULT_LOG_PATH = Path("train.log")
+if RANK in {-1, 0} and DEFAULT_LOG_PATH.exists():
+    DEFAULT_LOG_PATH.unlink(missing_ok=True)
+
+
+class ConsoleLogger:
+    """
+    Console output capture with API/file streaming and deduplication.
+
+    Captures stdout/stderr output and streams it to either an API endpoint or local file, with intelligent
+    deduplication to reduce noise from repetitive console output.
+
+    Attributes:
+        destination (str | Path): Target destination for streaming (URL or Path object).
+        is_api (bool): Whether destination is an API endpoint (True) or local file (False).
+        original_stdout: Reference to original sys.stdout for restoration.
+        original_stderr: Reference to original sys.stderr for restoration.
+        log_queue (queue.Queue): Thread-safe queue for buffering log messages.
+        active (bool): Whether console capture is currently active.
+        worker_thread (threading.Thread): Background thread for processing log queue.
+        last_line (str): Last processed line for deduplication.
+        last_time (float): Timestamp of last processed line.
+        last_progress_line (str): Last progress bar line for progress deduplication.
+        last_was_progress (bool): Whether the last line was a progress bar.
+
+    Examples:
+        Basic file logging:
+        >>> logger = ConsoleLogger("training.log")
+        >>> logger.start_capture()
+        >>> print("This will be logged")
+        >>> logger.stop_capture()
+
+        API streaming:
+        >>> logger = ConsoleLogger("https://api.example.com/logs")
+        >>> logger.start_capture()
+        >>> # All output streams to API
+        >>> logger.stop_capture()
+    """
+
+    def __init__(self, destination):
+        """
+        Initialize with API endpoint or local file path.
+
+        Args:
+            destination (str | Path): API endpoint URL (http/https) or local file path for streaming output.
+        """
+        self.destination = destination
+        self.is_api = isinstance(destination, str) and destination.startswith(("http://", "https://"))
+        if not self.is_api:
+            self.destination = Path(destination)
+
+        # Console capture
+        self.original_stdout = sys.stdout
+        self.original_stderr = sys.stderr
+        self.log_queue = queue.Queue(maxsize=1000)
+        self.active = False
+        self.worker_thread = None
+
+        # State tracking
+        self.last_line = ""
+        self.last_time = 0.0
+        self.last_progress_line = ""  # Track 100% progress lines separately
+        self.last_was_progress = False  # Track if last line was a progress bar
+
+    def start_capture(self):
+        """Start capturing console output and redirect stdout/stderr to custom capture objects."""
+        if self.active:
+            return
+
+        self.active = True
+        sys.stdout = self._ConsoleCapture(self.original_stdout, self._queue_log)
+        sys.stderr = self._ConsoleCapture(self.original_stderr, self._queue_log)
+
+        # Hook Ultralytics logger
+        try:
+            handler = self._LogHandler(self._queue_log)
+            logging.getLogger("ultralytics").addHandler(handler)
+        except Exception:
+            pass
+
+        self.worker_thread = threading.Thread(target=self._stream_worker, daemon=True)
+        self.worker_thread.start()
+
+    def stop_capture(self):
+        """Stop capturing console output and restore original stdout/stderr."""
+        if not self.active:
+            return
+
+        self.active = False
+        sys.stdout = self.original_stdout
+        sys.stderr = self.original_stderr
+        self.log_queue.put(None)
+
+    def _queue_log(self, text):
+        """Queue console text with deduplication and timestamp processing."""
+        if not self.active:
+            return
+
+        current_time = time.time()
+
+        # Handle carriage returns and process lines
+        if "\r" in text:
+            text = text.split("\r")[-1]
+
+        lines = text.split("\n")
+        if lines and lines[-1] == "":
+            lines.pop()
+
+        for line in lines:
+            line = line.rstrip()
+
+            # Handle progress bars - only show 100% completions
+            if ("it/s" in line and ("%|" in line or "━" in line)) or (
+                "100%" in line and ("it/s" in line or "[" in line)
+            ):
+                if "100%" not in line:
+                    continue
+                # Dedupe 100% lines by core content (strip timing)
+                progress_core = line.split("[")[0].split("]")[0].strip()
+                if progress_core == self.last_progress_line:
+                    continue
+                self.last_progress_line = progress_core
+                self.last_was_progress = True
+            else:
+                # Skip empty line after progress bar
+                if not line and self.last_was_progress:
+                    self.last_was_progress = False
+                    continue
+                self.last_was_progress = False
+
+            # General deduplication
+            if line == self.last_line and current_time - self.last_time < 0.1:
+                continue
+
+            self.last_line = line
+            self.last_time = current_time
+
+            # Add timestamp if needed
+            if not line.startswith("[20"):
+                timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                line = f"[{timestamp}] {line}"
+
+            # Queue with overflow protection
+            if not self._safe_put(f"{line}\n"):
+                continue  # Skip if queue handling fails
+
+    def _safe_put(self, item):
+        """Safely put item in queue with overflow handling."""
+        try:
+            self.log_queue.put_nowait(item)
+            return True
+        except queue.Full:
+            try:
+                self.log_queue.get_nowait()  # Drop oldest
+                self.log_queue.put_nowait(item)
+                return True
+            except queue.Empty:
+                return False
+
+    def _stream_worker(self):
+        """Background worker for streaming logs to destination."""
+        while self.active:
+            try:
+                log_text = self.log_queue.get(timeout=1)
+                if log_text is None:
+                    break
+                self._write_log(log_text)
+            except queue.Empty:
+                continue
+
+    def _write_log(self, text):
+        """Write log to API endpoint or local file destination."""
+        try:
+            if self.is_api:
+                payload = {"timestamp": datetime.now().isoformat(), "message": text.strip()}
+                requests.post(self.destination, json=payload, timeout=5)
+            else:
+                self.destination.parent.mkdir(parents=True, exist_ok=True)
+                with self.destination.open("a", encoding="utf-8") as f:
+                    f.write(text)
+        except Exception as e:
+            print(f"Platform logging error: {e}", file=self.original_stderr)
+
+    class _ConsoleCapture:
+        """Lightweight stdout/stderr capture."""
+
+        __slots__ = ("original", "callback")
+
+        def __init__(self, original, callback):
+            self.original = original
+            self.callback = callback
+
+        def write(self, text):
+            self.original.write(text)
+            self.callback(text)
+
+        def flush(self):
+            self.original.flush()
+
+    class _LogHandler(logging.Handler):
+        """Lightweight logging handler."""
+
+        __slots__ = ("callback",)
+
+        def __init__(self, callback):
+            super().__init__()
+            self.callback = callback
+
+        def emit(self, record):
+            self.callback(self.format(record) + "\n")
+
+
+class SystemLogger:
+    """
+    Log dynamic system metrics for training monitoring.
+
+    Captures real-time system metrics including CPU, RAM, disk I/O, network I/O, and NVIDIA GPU statistics for
+    training performance monitoring and analysis.
+
+    Attributes:
+        pynvml: NVIDIA pynvml module instance if successfully imported, None otherwise.
+        nvidia_initialized (bool): Whether NVIDIA GPU monitoring is available and initialized.
+        process (psutil.Process): Current psutil.Process instance for process-specific metrics.
+        net_start: Initial network I/O counters for calculating cumulative usage.
+        disk_start: Initial disk I/O counters for calculating cumulative usage.
+
+    Examples:
+        Basic usage:
+        >>> logger = SystemLogger()
+        >>> metrics = logger.get_metrics()
+        >>> print(f"CPU: {metrics['cpu']}%, RAM: {metrics['ram']}%")
+        >>> if metrics["gpus"]:
+        ...     gpu0 = metrics["gpus"]["0"]
+        ...     print(f"GPU0: {gpu0['usage']}% usage, {gpu0['temp']}°C")
+
+        Training loop integration:
+        >>> system_logger = SystemLogger()
+        >>> for epoch in range(epochs):
+        ...     # Training code here
+        ...     metrics = system_logger.get_metrics()
+        ...     # Log to database/file
+    """
+
+    def __init__(self):
+        """Initialize the system logger."""
+        self.pynvml = None
+        self.nvidia_initialized = self._init_nvidia()
+        self.process = psutil.Process()
+        self.net_start = psutil.net_io_counters()
+        self.disk_start = psutil.disk_io_counters()
+
+    def _init_nvidia(self):
+        """Initialize NVIDIA GPU monitoring with pynvml."""
+        try:
+            assert not MACOS
+            check_requirements("pynvml>=12.0.0")
+            self.pynvml = __import__("pynvml")
+            self.pynvml.nvmlInit()
+            return True
+        except Exception:
+            return False
+
+    def get_metrics(self):
+        """
+        Get current system metrics.
+
+        Collects comprehensive system metrics including CPU usage, RAM usage, disk I/O statistics,
+        network I/O statistics, and GPU metrics (if available). Example output:
+
+        ```json
+        {
+            'cpu': 45.2,
+            'ram': 78.9,
+            'disk': {'read_mb': 156.7, 'write_mb': 89.3, 'used_gb': 256.8},
+            'network': {'recv_mb': 157.2, 'sent_mb': 89.1},
+            'gpus': {
+                0: {'usage': 95.6, 'memory': 85.4, 'temp': 72, 'power': 285},
+                1: {'usage': 94.1, 'memory': 82.7, 'temp': 70, 'power': 278}
+            }
+        }
+        ```
+
+        - cpu (float): CPU usage percentage (0-100%)
+        - ram (float): RAM usage percentage (0-100%)
+        - disk (dict):
+            - read_mb (float): Cumulative disk read in MB since initialization
+            - write_mb (float): Cumulative disk write in MB since initialization
+            - used_gb (float): Total disk space used in GB
+        - network (dict):
+            - recv_mb (float): Cumulative network received in MB since initialization
+            - sent_mb (float): Cumulative network sent in MB since initialization
+        - gpus (dict): GPU metrics by device index (e.g., 0, 1) containing:
+            - usage (int): GPU utilization percentage (0-100%)
+            - memory (float): CUDA memory usage percentage (0-100%)
+            - temp (int): GPU temperature in degrees Celsius
+            - power (int): GPU power consumption in watts
+
+        Returns:
+            (dict): System metrics containing 'cpu', 'ram', 'disk', 'network', 'gpus' with respective usage data.
+        """
+        net = psutil.net_io_counters()
+        disk = psutil.disk_io_counters()
+        memory = psutil.virtual_memory()
+        disk_usage = shutil.disk_usage("/")
+
+        metrics = {
+            "cpu": round(psutil.cpu_percent(), 3),
+            "ram": round(memory.percent, 3),
+            "disk": {
+                "read_mb": round((disk.read_bytes - self.disk_start.read_bytes) / (1 << 20), 3),
+                "write_mb": round((disk.write_bytes - self.disk_start.write_bytes) / (1 << 20), 3),
+                "used_gb": round(disk_usage.used / (1 << 30), 3),
+            },
+            "network": {
+                "recv_mb": round((net.bytes_recv - self.net_start.bytes_recv) / (1 << 20), 3),
+                "sent_mb": round((net.bytes_sent - self.net_start.bytes_sent) / (1 << 20), 3),
+            },
+            "gpus": {},
+        }
+
+        # Add GPU metrics (NVIDIA only)
+        if self.nvidia_initialized:
+            metrics["gpus"].update(self._get_nvidia_metrics())
+
+        return metrics
+
+    def _get_nvidia_metrics(self):
+        """Get NVIDIA GPU metrics including utilization, memory, temperature, and power."""
+        gpus = {}
+        if not self.nvidia_initialized or not self.pynvml:
+            return gpus
+        try:
+            device_count = self.pynvml.nvmlDeviceGetCount()
+            for i in range(device_count):
+                handle = self.pynvml.nvmlDeviceGetHandleByIndex(i)
+                util = self.pynvml.nvmlDeviceGetUtilizationRates(handle)
+                memory = self.pynvml.nvmlDeviceGetMemoryInfo(handle)
+                temp = self.pynvml.nvmlDeviceGetTemperature(handle, self.pynvml.NVML_TEMPERATURE_GPU)
+                power = self.pynvml.nvmlDeviceGetPowerUsage(handle) // 1000
+
+                gpus[str(i)] = {
+                    "usage": round(util.gpu, 3),
+                    "memory": round((memory.used / memory.total) * 100, 3),
+                    "temp": temp,
+                    "power": power,
+                }
+        except Exception:
+            pass
+        return gpus
+
+
+if __name__ == "__main__":
+    print("SystemLogger Real-time Metrics Monitor")
+    print("Press Ctrl+C to stop\n")
+
+    logger = SystemLogger()
+
+    try:
+        while True:
+            metrics = logger.get_metrics()
+
+            # Clear screen (works on most terminals)
+            print("\033[H\033[J", end="")
+
+            # Display system metrics
+            print(f"CPU: {metrics['cpu']:5.1f}%")
+            print(f"RAM: {metrics['ram']:5.1f}%")
+            print(f"Disk Read: {metrics['disk']['read_mb']:8.1f} MB")
+            print(f"Disk Write: {metrics['disk']['write_mb']:7.1f} MB")
+            print(f"Disk Used: {metrics['disk']['used_gb']:8.1f} GB")
+            print(f"Net Recv: {metrics['network']['recv_mb']:9.1f} MB")
+            print(f"Net Sent: {metrics['network']['sent_mb']:9.1f} MB")
+
+            # Display GPU metrics if available
+            if metrics["gpus"]:
+                print("\nGPU Metrics:")
+                for gpu_id, gpu_data in metrics["gpus"].items():
+                    print(
+                        f"  GPU {gpu_id}: {gpu_data['usage']:3}% | "
+                        f"Mem: {gpu_data['memory']:5.1f}% | "
+                        f"Temp: {gpu_data['temp']:2}°C | "
+                        f"Power: {gpu_data['power']:3}W"
+                    )
+            else:
+                print("\nGPU: No NVIDIA GPUs detected")
+
+            time.sleep(1)
+
+    except KeyboardInterrupt:
+        print("\n\nStopped monitoring.")

{dgenerate_ultralytics_headless-8.3.183 → dgenerate_ultralytics_headless-8.3.184}/ultralytics/utils/metrics.py

@@ -1152,7 +1152,9 @@ class DetMetrics(SimpleClass, DataExportMixin):
     @property
     def results_dict(self) -> Dict[str, float]:
         """Return dictionary of computed performance metrics and statistics."""
-        return dict(zip(self.keys + ["fitness"], self.mean_results() + [self.fitness]))
+        keys = self.keys + ["fitness"]
+        values = ((float(x) if hasattr(x, "item") else x) for x in (self.mean_results() + [self.fitness]))
+        return dict(zip(keys, values))
 
     @property
     def curves(self) -> List[str]: