YoloDatasetCustomizer 0.1.0__tar.gz → 0.2.0__tar.gz

{yolodatasetcustomizer-0.1.0 → yolodatasetcustomizer-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: YoloDatasetCustomizer
-Version: 0.1.0
+Version: 0.2.0
 Summary: Load one or multiple datasets. Reduce, mix or merge into a custom dataset.
 Project-URL: Homepage, https://github.com/mLuca/YoloDatasetCustomizer
 Project-URL: Repository, https://github.com/mLuca/YoloDatasetCustomizer

{yolodatasetcustomizer-0.1.0 → yolodatasetcustomizer-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "YoloDatasetCustomizer"
-version = "0.1.0"
+version = "0.2.0"
 description = "Load one or multiple datasets. Reduce, mix or merge into a custom dataset."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -40,4 +40,5 @@ Repository = "https://github.com/mLuca/YoloDatasetCustomizer"
 [dependency-groups]
 dev = [
     "pytest>=9.1.1",
+    "twine>=6.2.0",
 ]

yolodatasetcustomizer-0.2.0/resources/Improvements.txt

@@ -0,0 +1,33 @@
+Priority-ordered improvement list
+
+FIXED 1. Fix brittle path and OS handling
+   Why: The code constructs file paths with string concatenation and assumes Unix-style separators, which can break on Windows and in nested dataset layouts. Since this tool operates on filesystem paths, cross-platform reliability is a core requirement.
+
+FIXED 2. Remove mutable default arguments
+   Why: class_names: list[str] = [] creates shared state across instances. This is a Python bug pattern that can cause hard-to-trace behavior and incorrect dataset generation when objects are reused.
+
+3. Tighten validation and error handling
+   Why: The project often prints warnings and continues instead of surfacing a clear, actionable failure. For dataset processing, silently skipping broken YAML or missing directories can produce partially valid outputs and make debugging much harder.
+
+FIXED 4. Fix broken internal state and debug output
+   Why: __repr__ references attributes that do not exist, which signals stale or inconsistent internal state. This is a maintainability problem and a sign that code may be harder to trust during debugging.
+
+5. Standardize class-index typing and conversion
+   Why: Indices are handled as strings in some places and as numeric concepts in others. This inconsistency increases the chance of subtle label remapping bugs that are especially dangerous in object-detection datasets.
+
+6. Refactor the monolithic implementation into smaller modules
+   Why: The main file handles reading YAML, writing YAML, copying labels, scanning directories, and building datasets all at once. Splitting responsibilities would reduce complexity, improve testability, and make future features safer to add.
+
+7. Improve test coverage around edge cases
+   Why: The current tests cover the happy path well, but not enough realistic failure cases such as malformed YAML, duplicate file names, empty split folders, and unexpected dataset layouts. More regression tests would reduce the chance of silent breakage.
+
+8. Clarify and formally validate documented limitations
+   Why: The README already explains assumptions like fixed dataset naming and ignored YAML fields. Those constraints should be treated as explicit API requirements so users understand exactly what is supported and what is not.
+
+9. Clean up naming and spelling inconsistencies
+   Why: Terms like “indeces” and inconsistent variable naming make the code less readable. This is not just cosmetic; it slows down maintenance and increases the chance of mistakes during refactors.
+
+10. Add clearer public API contracts and examples
+   Why: The project is useful, but the usage patterns are not fully anchored in a stable interface. Clearer public contracts and examples would make the package easier to adopt and less fragile over time.
+
+Highest priority overall: fix robustness around paths, defaults, and validation. Those are the issues most likely to lead to real-world failures in a dataset utility.

yolodatasetcustomizer-0.2.0/src/YoloDatasetCustomizer/__init__.py

@@ -0,0 +1,17 @@
+"""YOLO Dataset Customizer - Load, mix, merge, and customize YOLO datasets."""
+
+from .customizer import YoloDatasetCustomizer
+from .labels import LabelFile
+from .reader import YoloDataFileReader
+from .writer import YoloDataFileWriter
+
+__version__ = "0.2.0"
+__author__ = "Luca Mazzon"
+__email__ = "mazzon.luca@gmail.com"
+
+__all__ = [
+    "YoloDatasetCustomizer",
+    "YoloDataFileWriter",
+    "YoloDataFileReader",
+    "LabelFile",
+]

yolodatasetcustomizer-0.2.0/src/YoloDatasetCustomizer/customizer.py

@@ -0,0 +1,166 @@
+import glob
+import os
+import shutil
+
+import yaml
+
+from .labels import LabelFile
+from .paths import _SPLIT_NAMES, _get_unique_path, _normalize_path
+from .reader import YoloDataFileReader
+from .writer import YoloDataFileWriter
+
+
+class YoloDatasetCustomizer:
+    """Find and customize YOLO datasets across one or more directory paths."""
+
+    def __init__(self, data_set_paths: list[str]):
+        """Initialize with YAML dataset paths or directories to scan for YOLO datasets."""
+        self.__found_data_file_paths: set[str] = set()
+        self.__data_sets: list[YoloDataFileReader] = []
+
+        self.add_data_sets(data_set_paths)
+        self.SUPPORTED_IMAGE_FORMATS = [
+            ".avif",
+            ".bmp",
+            ".dng",
+            ".heic",
+            ".jp2",
+            ".jpeg",
+            ".jpg",
+            ".mpo",
+            ".png",
+            ".tif",
+            ".tiff",
+            ".webp",
+        ]
+
+    def add_data_sets(self, data_set_paths: list[str]):
+        """Discover YOLO data.yaml files from explicit paths or directory trees."""
+        data_set_paths = [_normalize_path(path) for path in data_set_paths]
+
+        self.__found_data_file_paths.update(
+            [path for path in data_set_paths if os.path.basename(path).lower() == "data.yaml"]
+        )
+
+        paths_without_file_ending = [
+            path for path in data_set_paths if os.path.basename(path).lower() != "data.yaml"
+        ]
+        for path in paths_without_file_ending:
+            self.__found_data_file_paths.update(glob.glob(os.path.join(path, "**", "data.yaml"), recursive=True))
+
+        for path in self.__found_data_file_paths:
+            try:
+                ydfr = YoloDataFileReader(path)
+                self.__data_sets.append(ydfr)
+            except (yaml.YAMLError, TypeError, ValueError) as e:
+                print(f"ERROR: Issue loading YAML file at {path}: {e}")
+            except FileNotFoundError:
+                print(f"ERROR: Yaml file does not exist at {path}")
+
+    def get_found_data_file_paths(self) -> set[str]:
+        """Return all discovered YOLO dataset YAML file paths."""
+        return self.__found_data_file_paths
+
+    def get_found_class_names(self) -> set[str]:
+        """Return the union of all class names from discovered datasets."""
+        found_classes: set[str] = set()
+        for ydf in self.__data_sets:
+            found_classes.update(ydf.get_class_names())
+        return found_classes
+
+    def create_new_dataset_for_class_names(
+        self,
+        class_names: set[str],
+        dst_path: str = ".",
+        data_set_name: str = "new_dataset",
+        ignore_img_formats: bool = False,
+    ) -> bool:
+        """Create a filtered dataset containing only the requested classes."""
+        if len(class_names) == 0:
+            print("ERROR: No classes selected for new dataset.")
+            return False
+
+        new_dataset_path = os.path.join(_normalize_path(dst_path), data_set_name)
+        new_dataset_path = _get_unique_path(new_dataset_path)
+
+        new_class_indeces = self.__generate_new_class_indeces(class_names)
+        print(f"INFO: New class indeces: {new_class_indeces}")
+
+        for data_set in self.__data_sets:
+            common_class_names = set(data_set.get_class_names()) & class_names
+            if not common_class_names:
+                continue
+
+            old_to_new_index_match: dict[str, str] = {}
+            for class_name in common_class_names:
+                old_index = str(data_set.get_class_names().index(class_name))
+                new_index = new_class_indeces[class_name]
+                old_to_new_index_match[old_index] = new_index
+
+            for split_name in _SPLIT_NAMES:
+                for split_path in data_set.get_split_paths_for_split_name(split_name):
+                    old_images_path = os.path.join(_normalize_path(data_set.get_file_dir()), split_path)
+                    old_labels_path = self.__generate_label_path_from_img_path(old_images_path)
+
+                    for path in [old_labels_path, old_images_path]:
+                        if not os.path.exists(path):
+                            print(f"WARNING: Path '{path}' does not exist! Skipping it.")
+                            continue
+
+                    new_labels_path = os.path.join(new_dataset_path, split_name, "labels")
+                    new_images_path = os.path.join(new_dataset_path, split_name, "images")
+                    os.makedirs(new_labels_path, exist_ok=True)
+                    os.makedirs(new_images_path, exist_ok=True)
+
+                    for original_label_file_path in glob.glob(os.path.join(old_labels_path, "*.txt")):
+                        label_file = LabelFile(original_label_file_path)
+                        new_label_file_path = label_file.copy_by_class_indeces(old_to_new_index_match, new_labels_path)
+                        if new_label_file_path:
+                            file_name = os.path.basename(original_label_file_path)
+                            file_name_no_ext = file_name[: file_name.rfind(".")]
+                            corresponding_image_files = glob.glob(os.path.join(old_images_path, f"{file_name_no_ext}.*"))
+                            for img in corresponding_image_files:
+                                try:
+                                    _, ext = os.path.splitext(img)
+                                except Exception:
+                                    print(f"ERROR: Not a valid image, extension missing for: {img}")
+                                    return False
+
+                                if not ignore_img_formats and ext.lower() not in self.SUPPORTED_IMAGE_FORMATS:
+                                    print(
+                                        f"ERROR: Image format not supported by yolo: {img}\n"
+                                        f"Supported formats are: {self.SUPPORTED_IMAGE_FORMATS}"
+                                    )
+                                    return False
+
+                                new_image_name, _ = os.path.splitext(os.path.basename(new_label_file_path))
+                                new_image_name = new_image_name + ext
+                                destination = os.path.join(new_images_path, new_image_name)
+                                if not shutil.copyfile(img, destination):
+                                    print(
+                                        f"ERROR: Image could not be copied:\n"
+                                        f"Source: {img}\nDestination: {destination}"
+                                    )
+                                    return False
+
+        data_file_writer = YoloDataFileWriter(new_dataset_path, list(new_class_indeces.keys()))
+        try:
+            data_file_writer.write()
+        except Exception as e:
+            print(f"ERROR: Could not write new data.yaml to {new_dataset_path}. Generation of {data_set_name} is incomplete!")
+            print(f"Exception was: {e}")
+            return False
+
+        return True
+
+    def __generate_new_class_indeces(self, class_names: set[str]) -> dict[str, str]:
+        """Generate a fresh label index mapping for the selected class names."""
+        ret: dict[str, str] = {}
+        for i, name in enumerate(class_names):
+            ret[name] = str(i)
+        return ret
+
+    def __generate_label_path_from_img_path(self, img_path: str) -> str:
+        """Convert an images directory path into its corresponding labels directory path."""
+        img_path = _normalize_path(img_path)
+        return os.path.normpath(os.path.join(os.path.dirname(img_path), "labels"))

yolodatasetcustomizer-0.2.0/src/YoloDatasetCustomizer/labels.py

@@ -0,0 +1,71 @@
+import os
+import re
+
+from .paths import _get_unique_path
+
+
+class LabelFile:
+    """Read YOLO label files and copy label lines with remapped class indices."""
+
+    def __init__(self, file_path: str):
+        """
+        Args:
+            file_path: Valid path to the label file
+        Raises:
+            FileNotFoundError: When the label file does not exist
+        """
+        self.__class_indeces: set[str] = set()
+        self.file_path = _get_unique_path(file_path) if os.path.isdir(file_path) else os.path.abspath(file_path)
+        if not os.path.exists(self.file_path) or not os.path.isfile(self.file_path):
+            raise FileNotFoundError(f"No label file found at: {self.file_path}")
+
+        self.read_label_file()
+
+    def read_label_file(self):
+        """Parse the label file and collect all class indices referenced in it."""
+        with open(self.file_path, encoding="utf-8") as f:
+            for i, line in enumerate(f):
+                match = re.match(r"^(\d+) ", line)
+                if not match:
+                    print(f"WARNING: No class found on line {i} in label file {self.file_path}")
+                else:
+                    self.__class_indeces.add(match.group(1))
+
+    def get_class_indices(self) -> set[str]:
+        """Return the set of class indices in the loaded label file."""
+        return self.__class_indeces
+
+    def copy_by_class_indeces(self, old_to_new_index_match: dict[str, str], dst_path: str, file_name: str = "") -> str | None:
+        """Copy the label file to the destination with indices remapped for the new dataset."""
+        common_class_indeces = self.__class_indeces & set(old_to_new_index_match.keys())
+        if not common_class_indeces:
+            return None
+
+        dst_path = os.path.abspath(dst_path)
+        if os.path.basename(dst_path) != "labels":
+            dst_path = os.path.join(dst_path, "labels")
+
+        if os.path.exists(dst_path) and os.path.isdir(dst_path):
+            new_content: list[str] = []
+            with open(self.file_path, encoding="utf-8") as f:
+                for line in f:
+                    match = re.match(r"^(\d+) ", line)
+                    if not match:
+                        continue
+                    elif match.group(1) in common_class_indeces:
+                        old_index = match.group(1)
+                        new_index = old_to_new_index_match[old_index]
+                        if new_index != old_index:
+                            line = line.replace(old_index, new_index, 1)
+                        new_content.append(line)
+
+            if file_name == "":
+                file_name = os.path.basename(self.file_path)
+            dst_file_path = _get_unique_path(os.path.join(dst_path, file_name))
+
+            with open(dst_file_path, "w", encoding="utf-8") as f:
+                f.writelines(new_content)
+        else:
+            raise FileNotFoundError(f"Directory not found: {dst_path}")
+
+        return dst_file_path

yolodatasetcustomizer-0.2.0/src/YoloDatasetCustomizer/paths.py

@@ -0,0 +1,20 @@
+import os
+
+_SPLIT_NAMES = ["train", "val", "test"]
+
+
+def _normalize_path(path: str) -> str:
+    """Normalize path separators and return an absolute path for cross-platform use."""
+    return os.path.abspath(str(path).replace("\\", os.sep))
+
+
+def _get_unique_path(path: str) -> str:
+    """Return an unused file path by appending a numeric suffix when needed."""
+    path = _normalize_path(path)
+    base, ext = os.path.splitext(path)
+    unique_path = path
+    index = 1
+    while os.path.exists(unique_path):
+        unique_path = f"{base}_{index}{ext}"
+        index += 1
+    return unique_path

yolodatasetcustomizer-0.2.0/src/YoloDatasetCustomizer/reader.py

@@ -0,0 +1,90 @@
+import os
+from typing import Any
+
+import yaml
+
+from .paths import _SPLIT_NAMES, _normalize_path
+
+
+class YoloDataFileReader:
+    """Read a YOLO dataset YAML file and expose its split paths and class names."""
+
+    def __init__(self, file_path: str):
+        """
+        Args:
+            file_path: Valid path to the data set YAML file
+        Raises:
+            yaml.YAMLError: When there is an issue with reading the data-set YAML file
+            FileNotFoundError: When the data-set YAML file does not exist
+            ValueError: When a value in the YAML file is missing or is faulty
+            TypeError: When a value comes in an unexpected type
+        """
+        self.__file_path = _normalize_path(file_path)
+        self.__file_dir = os.path.dirname(self.__file_path)
+        self.__splits_paths: dict[str, list[str]] = {}
+        self.__class_names: list[str] = []
+
+        self.sync_content()
+
+    def get_split_paths_for_split_name(self, split_name: str) -> list[str]:
+        """Return a list of dataset split paths for the requested split name."""
+        return self.__splits_paths.get(split_name, [])
+
+    def get_class_names(self) -> list[str]:
+        """Return the class names defined in the dataset YAML."""
+        return self.__class_names
+
+    def get_file_path(self) -> str:
+        """Return the absolute path to the loaded YAML file."""
+        return self.__file_path
+
+    def get_file_dir(self) -> str:
+        """Return the directory containing the loaded dataset YAML file."""
+        return self.__file_dir
+
+    def get_indices_for_names(self, class_names: set[str]) -> set[str]:
+        """Map a set of class names to their corresponding numeric label indices."""
+        ret: set[str] = set()
+        for class_name in class_names:
+            try:
+                idx = self.__class_names.index(class_name)
+                ret.add(str(idx))
+            except ValueError:
+                pass
+        return ret
+
+    def sync_content(self):
+        """Reload YAML content and refresh split and name mappings."""
+        with open(self.__file_path, encoding="utf-8") as f:
+            content = yaml.safe_load(f)
+        for split_name in _SPLIT_NAMES:
+            self.__splits_paths[split_name] = self.__read_yaml_entry(content, split_name)
+        self.__class_names = self.__read_yaml_entry(content, "names")
+
+    def __read_yaml_entry(self, yaml_content: Any, entry_name: str, missing_ok: bool = True) -> list[str]:
+        """Normalize YAML entry values into a list of paths or names."""
+        entry = yaml_content.get(entry_name)
+        ret: list[str] = []
+        if entry:
+            if isinstance(entry, str):
+                ret.append(entry)
+            elif isinstance(entry, dict):
+                ret.extend(entry.values())
+            elif isinstance(entry, list):
+                ret.extend(entry)
+            else:
+                raise TypeError(f"Unexpected type for {entry_name} in: {self.__file_path}")
+        else:
+            if not missing_ok:
+                raise ValueError(f"{entry_name} not defined in: {self.__file_path}")
+
+        return ret
+
+    def __repr__(self) -> str:
+        return (
+            f"\nYoloDatasetFile(\n"
+            f"    path: {self.__file_path}\n"
+            f"    split_paths: {self.__splits_paths}\n"
+            f"    class_names: {self.__class_names}\n"
+            f")\n"
+        )

yolodatasetcustomizer-0.2.0/src/YoloDatasetCustomizer/writer.py

@@ -0,0 +1,41 @@
+import os
+
+import yaml
+
+from .paths import _normalize_path
+
+
+class YoloDataFileWriter:
+    """Create and write a YOLO dataset YAML file for a generated dataset."""
+
+    def __init__(
+        self,
+        data_set_dir: str = "./",
+        class_names: list[str] | None = None,
+        train_split: str = "./train",
+        val_split: str = "./valid",
+        test_split: str = "./test",
+    ) -> None:
+        """Initialize writer with dataset directory, class names, and split locations."""
+        self.data_set_dir = _normalize_path(data_set_dir)
+        self.class_names = list(class_names) if class_names is not None else []
+        self.train_split = train_split
+        self.val_split = val_split
+        self.test_split = test_split
+        self.FILE_NAME = "data.yaml"
+
+    def write(self):
+        """Write the YOLO data.yaml file with train/val/test paths and class names."""
+        classes = {}
+        for i, class_name in enumerate(self.class_names):
+            classes[i] = class_name
+        data = {
+            "train": f"{self.train_split}/images",
+            "val": f"{self.val_split}/images",
+            "test": f"{self.test_split}/images",
+            "names": classes,
+            "nc": len(self.class_names),
+        }
+
+        with open(os.path.join(self.data_set_dir, self.FILE_NAME), "w", encoding="utf-8") as f:
+            yaml.safe_dump(data, f, default_flow_style=False)

{yolodatasetcustomizer-0.1.0 → yolodatasetcustomizer-0.2.0}/test/YoloDatasetCustomizer_test.py

@@ -7,7 +7,7 @@ import yaml
 
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "src")))
 
-from YoloDatasetCustomizer.YoloDatasetCustomizer import (
+from YoloDatasetCustomizer import (
     YoloDataFileWriter,
     YoloDataFileReader,
     LabelFile,
@@ -130,6 +130,15 @@ nc: 1
         self.assertEqual(loaded["names"], {0: "person", 1: "car"})
         self.assertEqual(loaded["nc"], 2)
 
+    def test_yolo_data_file_writer_uses_independent_default_class_names(self):
+        writer_a = YoloDataFileWriter()
+        writer_b = YoloDataFileWriter()
+
+        writer_a.class_names.append("person")
+
+        self.assertEqual(writer_a.class_names, ["person"])
+        self.assertEqual(writer_b.class_names, [])
+
     def test_yolo_data_file_reader_reads_splits_names_and_indices(self):
         reader = YoloDataFileReader(os.path.join(self.good_dataset_dir, "data.yaml"))
         self.assertEqual(reader.get_file_path(), os.path.abspath(os.path.join(self.good_dataset_dir, "data.yaml")))
@@ -184,6 +193,12 @@ nc: 1
         self.assertIn(os.path.abspath(os.path.join(self.unsupported_image_dataset_dir, "data.yaml")), found_paths)
         self.assertEqual(customizer.get_found_class_names(), {"person", "car"})
 
+    def test_yolo_dataset_customizer_handles_windows_style_paths(self):
+        windows_style_root = self.workspace_root.replace(os.sep, "\\")
+        customizer = YoloDatasetCustomizer([windows_style_root])
+        self.assertIn(os.path.abspath(os.path.join(self.good_dataset_dir, "data.yaml")), customizer.get_found_data_file_paths())
+        self.assertEqual(customizer.get_found_class_names(), {"person", "car"})
+
     def test_yolo_dataset_customizer_creates_filtered_dataset(self):
         customizer = YoloDatasetCustomizer([self.good_dataset_dir])
         success = customizer.create_new_dataset_for_class_names({"car"}, dst_path=self.workspace_root, data_set_name="filtered_car_dataset")