YoloDatasetCustomizer 0.4.1__tar.gz → 0.5.0__tar.gz

{yolodatasetcustomizer-0.4.1 → yolodatasetcustomizer-0.5.0}/LICENSE

@@ -1,7 +1,7 @@
 GNU GENERAL PUBLIC LICENSE
 Version 3, 29 June 2007
 
-Copyright (C) 2024 Luca Mazzon
+Copyright (C) 2026 Luca Mazzon
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

{yolodatasetcustomizer-0.4.1 → yolodatasetcustomizer-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: YoloDatasetCustomizer
-Version: 0.4.1
+Version: 0.5.0
 Summary: Load one or multiple datasets. Reduce, mix or merge into a custom dataset.
 Project-URL: Homepage, https://github.com/mLuca/YoloDatasetCustomizer
 Project-URL: Repository, https://github.com/mLuca/YoloDatasetCustomizer

{yolodatasetcustomizer-0.4.1 → yolodatasetcustomizer-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "YoloDatasetCustomizer"
-version = "0.4.1"
+version = "0.5.0"
 description = "Load one or multiple datasets. Reduce, mix or merge into a custom dataset."
 readme = "README.md"
 requires-python = ">=3.10"

{yolodatasetcustomizer-0.4.1 → yolodatasetcustomizer-0.5.0}/src/YoloDatasetCustomizer/__init__.py

@@ -5,7 +5,7 @@ from .labels import LabelFile
 from .reader import YoloDataFileReader
 from .writer import YoloDataFileWriter
 
-__version__ = "0.4.1"
+__version__ = "0.5.0"
 __author__ = "Luca Mazzon"
 __email__ = "mazzon.luca@gmail.com"
 

{yolodatasetcustomizer-0.4.1 → yolodatasetcustomizer-0.5.0}/src/YoloDatasetCustomizer/customizer.py

@@ -1,8 +1,11 @@
 import glob
 import os
 import shutil
+import sys
+from concurrent.futures import ThreadPoolExecutor
 
 import yaml
+from rich.console import Console
 from rich.progress import BarColumn, MofNCompleteColumn, Progress, TaskProgressColumn, TextColumn
 
 from .labels import LabelFile
@@ -46,8 +49,9 @@ class YoloDatasetCustomizer:
         paths_without_file_ending = [
             path for path in data_set_paths if os.path.basename(path).lower() != "data.yaml"
         ]
-        for path in paths_without_file_ending:
-            self.__found_data_file_paths.update(glob.glob(os.path.join(path, "**", "data.yaml"), recursive=True))
+        with Console().status("Searching for data.yaml files..."):
+            for path in paths_without_file_ending:
+                self.__found_data_file_paths.update(glob.glob(os.path.join(path, "**", "data.yaml"), recursive=True))
 
         for path in self.__found_data_file_paths:
             try:
@@ -58,6 +62,23 @@ class YoloDatasetCustomizer:
             except FileNotFoundError:
                 print(f"ERROR: Yaml file does not exist at {path}")
 
+    @staticmethod
+    def __read_label_files(label_paths: list[str], description: str) -> list[LabelFile]:
+        """Parse label files in parallel (I/O bound) while showing a progress bar."""
+        label_files: list[LabelFile] = []
+        with Progress(
+            TextColumn("{task.description}"),
+            BarColumn(),
+            MofNCompleteColumn(),
+            transient=True,
+        ) as progress:
+            task = progress.add_task(description, total=len(label_paths))
+            with ThreadPoolExecutor(max_workers=16) as pool:
+                for label_file in pool.map(LabelFile, label_paths):
+                    label_files.append(label_file)
+                    progress.advance(task)
+        return label_files
+
     def get_found_data_file_paths(self) -> set[str]:
         """Return all discovered YOLO dataset YAML file paths."""
         return self.__found_data_file_paths
@@ -97,6 +118,7 @@ class YoloDatasetCustomizer:
         planned_files: dict[str, list[tuple[LabelFile, dict[str, list[str]], dict[str, str]]]] = {
             split_name: [] for split_name in _SPLIT_NAMES
         }
+        fallback_choice: str | None = None  # Sticky answer: "all" or "none"
         for data_set in self.__data_sets:
             common_class_names = set(data_set.get_class_names()) & class_names
             if not common_class_names:
@@ -120,22 +142,53 @@ class YoloDatasetCustomizer:
                         (path for path in (old_labels_path, old_images_path) if not os.path.exists(path)), None
                     )
                     if missing_path is not None:
-                        print(f"WARNING: Path '{missing_path}' does not exist! Skipping it.")
-                        continue
+                        fallback_images_path = self.__find_fallback_images_path(data_set, split_path)
+                        if fallback_images_path is None:
+                            print(
+                                f"WARNING: Path '{missing_path}' from '{data_set.get_file_path()}' "
+                                f"does not exist! Skipping it."
+                            )
+                            continue
+                        try:
+                            choice = fallback_choice or self.__ask_use_fallback(
+                                data_set.get_file_path(), split_path, fallback_images_path
+                            )
+                        except EOFError:
+                            print(
+                                "ERROR: Cannot prompt for Roboflow fallback because standard input is closed. "
+                                "Dataset creation aborted."
+                            )
+                            return False
+                        if choice in ("all", "none"):
+                            fallback_choice = choice
+                        if choice == "abort":
+                            print("INFO: Aborted by user.")
+                            return False
+                        if choice in ("no", "none"):
+                            print(f"WARNING: Skipping '{split_path}' of '{data_set.get_file_path()}'.")
+                            continue
+                        old_images_path = fallback_images_path
+                        old_labels_path = self.__generate_label_path_from_img_path(old_images_path)
 
                     # Index images by exact stem so 'car.*' can't pick up e.g. 'car.v2.jpg'.
                     # Only regular files with an extension count, like the old 'car.*' lookup.
                     images_by_stem: dict[str, list[str]] = {}
-                    for image_name in os.listdir(old_images_path):
-                        stem, ext = os.path.splitext(image_name)
-                        image_path = os.path.join(old_images_path, image_name)
-                        if not ext or not os.path.isfile(image_path):
-                            continue
-                        images_by_stem.setdefault(stem, []).append(image_path)
+                    with os.scandir(old_images_path) as entries:
+                        for entry in entries:
+                            stem, ext = os.path.splitext(entry.name)
+                            if ext and entry.is_file():
+                                images_by_stem.setdefault(stem, []).append(entry.path)
 
-                    for original_label_file_path in glob.glob(os.path.join(old_labels_path, "*.txt")):
-                        label_file = LabelFile(original_label_file_path)
-                        if label_file.get_class_indices() & set(old_to_new_index_match):
+                    with os.scandir(old_labels_path) as entries:
+                        label_paths = [
+                            e.path for e in entries
+                            if os.path.normcase(e.name).endswith(".txt") and e.is_file()
+                        ]
+                    wanted_indices = set(old_to_new_index_match)
+                    for label_file in self.__read_label_files(
+                        label_paths, f"Scanning {split_name:<5} labels of {data_set.get_file_path()}"
+                    ):
+                        if label_file.get_class_indices() & wanted_indices:
                             planned_files[split_name].append((label_file, images_by_stem, old_to_new_index_match))
 
         # One bar per stage (white, filled green), all visible from the start
@@ -222,6 +275,43 @@ class YoloDatasetCustomizer:
                 return False
         return True
 
+    def __find_fallback_images_path(self, data_set: YoloDataFileReader, split_path: str) -> str | None:
+        """Return the images path with leading '../' removed (legacy Roboflow style) if it and its labels exist."""
+        stripped = split_path.replace("\\", "/")
+        while stripped.startswith("../"):
+            stripped = stripped[3:]
+        if stripped == split_path.replace("\\", "/"):
+            return None
+        images_path = os.path.join(_normalize_path(data_set.get_file_dir()), stripped)
+        labels_path = self.__generate_label_path_from_img_path(images_path)
+        if os.path.isdir(images_path) and os.path.isdir(labels_path):
+            return os.path.normpath(images_path)
+        return None
+
+    @staticmethod
+    def __ask_use_fallback(yaml_path: str, split_path: str, fallback_path: str) -> str:
+        """Ask the user whether to use the fallback folder. Returns yes/all/no/none/abort."""
+        options = {"1": "yes", "2": "all", "3": "no", "4": "none", "5": "abort"}
+        print(
+            f"'{split_path}' in '{yaml_path}' does not exist, but was found at:\n  {fallback_path}\n"
+            f"Use it? [1] Yes  [2] All  [3] No  [4] None  [5] Abort: ",
+            end="",
+            flush=True,
+        )
+        while True:
+            # getwch() bypasses stdin, so only use it for an interactive console
+            if sys.platform == "win32" and sys.stdin is not None and sys.stdin.isatty():
+                import msvcrt
+
+                key = msvcrt.getwch()
+                if key == "\x03":
+                    raise KeyboardInterrupt
+            else:
+                key = (input() or " ")[0]
+            if key in options:
+                print(key)
+                return options[key]
+
     def __generate_new_class_indices(self, class_names: set[str]) -> dict[str, str]:
         """Generate a fresh label index mapping for the selected class names."""
         ret: dict[str, str] = {}

{yolodatasetcustomizer-0.4.1 → yolodatasetcustomizer-0.5.0}/test/YoloDatasetCustomizer_test.py

@@ -3,6 +3,8 @@ import shutil
 import sys
 import tempfile
 import unittest
+from contextlib import redirect_stdout
+from io import StringIO
 from unittest.mock import patch
 
 import yaml
@@ -316,6 +318,29 @@ nc: 2
             )
         self.assertFalse(result)
 
+    def test_create_new_dataset_reports_closed_stdin_for_roboflow_fallback(self):
+        with tempfile.TemporaryDirectory(prefix="yolodatasetcustomizer_roboflow_") as tmp:
+            dataset_dir = os.path.join(tmp, "dataset")
+            for split in ("train", "valid", "test"):
+                for kind in ("images", "labels"):
+                    os.makedirs(os.path.join(dataset_dir, split, kind), exist_ok=True)
+
+            self._write_text_file(
+                os.path.join(dataset_dir, "data.yaml"),
+                "train: ../train/images\nval: valid/images\ntest: test/images\nnames: [person]\nnc: 1\n",
+            )
+            customizer = YoloDatasetCustomizer([dataset_dir])
+            output = StringIO()
+            with patch("builtins.input", side_effect=EOFError), redirect_stdout(output):
+                result = customizer.create_new_dataset_for_class_names(
+                    {"person"}, dst_path=tmp, data_set_name="output"
+                )
+
+        self.assertFalse(result)
+        self.assertIn("standard input is closed", output.getvalue())
+        self.assertIn("Dataset creation aborted.", output.getvalue())
+        self.assertNotIn("Aborted by user.", output.getvalue())
+
     def test_yolo_dataset_customizer_creates_filtered_dataset(self):
         customizer = YoloDatasetCustomizer([self.good_dataset_dir])
         success = customizer.create_new_dataset_for_class_names({"car"}, dst_path=self.workspace_root, data_set_name="filtered_car_dataset")

{yolodatasetcustomizer-0.4.1 → yolodatasetcustomizer-0.5.0}/uv.lock

@@ -1078,7 +1078,7 @@ wheels = [
 
 [[package]]
 name = "yolodatasetcustomizer"
-version = "0.4.1"
+version = "0.5.0"
 source = { editable = "." }
 dependencies = [
     { name = "pyyaml" },