deepvisiontools 1.0.0__tar.gz → 1.0.2__tar.gz

{deepvisiontools-1.0.0 → deepvisiontools-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepvisiontools
-Version: 1.0.0
+Version: 1.0.2
 Summary: deepvisiontools is a library built on torch for training and inference processes for deep learning computer vision tasks.
 Author-email: Lucas Bernigaud Samatan <lucas.bernigaud-samatan@inrae.fr>, Jordan Bernigaud Samatan <jordan.bernigaud-samatan@inrae.fr>
 Maintainer-email: Jordan Bernigaud Samatan <jordan.bernigaud-samatan@inrae.fr>, Lucas Bernigaud Samatan <lucas.bernigaud-samatan@inrae.fr>
@@ -688,7 +688,7 @@ Requires-Dist: torch-tb-profiler<1.0,>=0.4.0
 Requires-Dist: mlflow<4.0,>=3.1.0
 Requires-Dist: fastapi!=0.136.3,<1.0
 Requires-Dist: starlette<2.0,>=1.0.1
-Requires-Dist: ultralytics<9.0,>=8.3.28
+Requires-Dist: ultralytics<8.5.0,>=8.3.28
 Requires-Dist: transformers<6.0,>=5.0.0
 Requires-Dist: colorcet<4.0,>=3.1.0
 Requires-Dist: segmentation-models-pytorch<1.0,>=0.4.1
@@ -731,6 +731,11 @@ Dynamic: license-file
 
 - *release 1.0.0: detector / detector-seg / tabdet / tabdet-seg model wrappers are heavy but powerful models for detection and instance segmentation. However, they are not fully pretrained (only backbone on imagenet). Therefore, on small datasets (< 1000 images ish, but depend on the dataset ...) they can be slightly less competitive than yolo models (also included in the library) that were fully pretrained on coco. We are currently pretraining some of the models but it takes time. We are goind to update the weights once done, so keep tuned for new releases. We suggest you always train a yolo model as well for a baseline comparison. However, on larger datasets they can significantly outperform yolo11x, but again it depends on the dataset ...*
 
+### patchs
+
+- *1.0.2: added option to concat datasets for deepvisiondataset (DeepVisionDataset.concat)*
+- *1.0.1: fixed inconsistency for yolo wrappers due to modified behaviour in ultralytics between 8.3 and 8.4 versions. Fixed and compatible with both.*
+
 ## Scope
 
 deepvisiontools is suitable for supervised learning computer vision tasks. Tasks currently available are:

{deepvisiontools-1.0.0 → deepvisiontools-1.0.2}/README.md

@@ -14,6 +14,11 @@
 
 - *release 1.0.0: detector / detector-seg / tabdet / tabdet-seg model wrappers are heavy but powerful models for detection and instance segmentation. However, they are not fully pretrained (only backbone on imagenet). Therefore, on small datasets (< 1000 images ish, but depend on the dataset ...) they can be slightly less competitive than yolo models (also included in the library) that were fully pretrained on coco. We are currently pretraining some of the models but it takes time. We are goind to update the weights once done, so keep tuned for new releases. We suggest you always train a yolo model as well for a baseline comparison. However, on larger datasets they can significantly outperform yolo11x, but again it depends on the dataset ...*
 
+### patchs
+
+- *1.0.2: added option to concat datasets for deepvisiondataset (DeepVisionDataset.concat)*
+- *1.0.1: fixed inconsistency for yolo wrappers due to modified behaviour in ultralytics between 8.3 and 8.4 versions. Fixed and compatible with both.*
+
 ## Scope
 
 deepvisiontools is suitable for supervised learning computer vision tasks. Tasks currently available are:

{deepvisiontools-1.0.0 → deepvisiontools-1.0.2}/pyproject.toml

@@ -44,7 +44,7 @@ dependencies = [
   "fastapi!=0.136.3,<1.0",
   "starlette>=1.0.1,<2.0",
 
-  "ultralytics>=8.3.28,<9.0",
+  "ultralytics>=8.3.28,<8.5.0",
   "transformers>=5.0.0,<6.0",
   "colorcet>=3.1.0,<4.0",
   "segmentation-models-pytorch>=0.4.1,<1.0",

{deepvisiontools-1.0.0 → deepvisiontools-1.0.2}/src/deepvisiontools/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.0.0"
+__version__ = "1.0.2"
 
 from .config import current, set_current, use_config
 from .datasets import DeepVisionDataset, DeepVisionLoader

{deepvisiontools-1.0.0 → deepvisiontools-1.0.2}/src/deepvisiontools/datasets/base.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import warnings
 from collections.abc import Callable, Sequence
+from copy import deepcopy
 from pathlib import Path
 from typing import Any, Literal, Protocol, cast
 
@@ -334,6 +335,7 @@ class DeepVisionDataset(Dataset):
         - You can call the `keep_indices` method to keep only the selected indexes
         - You can modify the writer using the property `writer` so the `export` method returns the provided format. Note that you may need to modify the cast_to attribute to match writer requirements
         - You can `export` a (sub)dataset to save a split and (optionally) visualizations. Caution to set preprocessing/augmentation to None if you want original images before normalization.
+        - You can concat multiple DeepVisionDatasets (including subsets etc.) using the concat static method.
 
     ???+ example "Usage"
         ```Python
@@ -581,11 +583,11 @@ class DeepVisionDataset(Dataset):
     def set_writer(self, writer: AnnotationWriter) -> None:
         self.writer = writer
 
-    def to_data_type(self, dst_key: str) -> DeepVisionSubDataset:
+    def to_data_type(self, dst_key: str) -> DeepVisionDataset:
         """Return a view that converts every sample's target to `dst_key` lazily."""
         return DeepVisionSubDataset(self, cast_to=dst_key)
 
-    def split(self, proportions: Sequence[float]) -> list[DeepVisionSubDataset]:
+    def split(self, proportions: Sequence[float]) -> list[DeepVisionDataset]:
         """
         Randomly split the current dataset view into sub-views.
 
@@ -604,7 +606,7 @@ class DeepVisionDataset(Dataset):
         n = len(positions)
         cuts = [round(p * n) for p in proportions]
 
-        out: list[DeepVisionSubDataset] = []
+        out: list[DeepVisionDataset] = []
         start = 0
         for c in cuts[:-1]:
             out.append(DeepVisionSubDataset(self, indices=positions[start : start + c]))
@@ -616,7 +618,7 @@ class DeepVisionDataset(Dataset):
     def keep_indices(
         self,
         indices: list[int] | torch.Tensor | slice,
-    ) -> DeepVisionSubDataset:
+    ) -> DeepVisionDataset:
         """
         Keep a subset of samples from the current dataset view.
 
@@ -735,6 +737,26 @@ class DeepVisionDataset(Dataset):
 
         return image, target, img_name
 
+    @staticmethod
+    def concat(
+        datasets: Sequence[DeepVisionDataset],
+        writer: AnnotationWriter | str | None = None,
+        *,
+        prefix_names: bool = True,
+    ) -> DeepVisionConcatDataset:
+        """
+        Concatenate several DeepVisionDataset views into one DeepVision-compatible view.
+
+        The returned dataset can be indexed, split, loaded with DeepVisionLoader,
+        and exported with the configured writer. Input datasets are deep-copied so
+        later edits on the concatenated view do not mutate the original datasets.
+        """
+        return DeepVisionConcatDataset(
+            datasets,
+            writer=writer,
+            prefix_names=prefix_names,
+        )
+
 
 class DeepVisionSubDataset(DeepVisionDataset):
     """A thin view over a base dataset with its own index list and optional cast override."""
@@ -772,6 +794,354 @@ class DeepVisionSubDataset(DeepVisionDataset):
             self.cast_to = cast_to
 
 
+class DeepVisionConcatDataset(DeepVisionDataset):
+    """
+    Concatenation view over several DeepVisionDataset instances.
+    Modifying preprocessing, augmentation etc. property modifies internal
+    deepcopies of the composition datasets.
+
+    Args:
+        datasets: Dataset views to concatenate. DeepVisionSubDataset instances are
+            supported.
+        writer: Optional writer used by export. If None, a deep copy of the first
+            child dataset writer is used.
+        prefix_names: Prefix returned image names with the child dataset index.
+            This is recommended to avoid filename collisions during export.
+        indices: Optional global indices used internally for concat sub-views.
+    """
+
+    def __init__(
+        self,
+        datasets: Sequence[DeepVisionDataset],
+        writer: AnnotationWriter | str | None = None,
+        *,
+        prefix_names: bool = True,
+        indices: Sequence[int] | None = None,
+    ) -> None:
+        if len(datasets) == 0:
+            raise ValueError("DeepVisionConcatDataset requires at least one dataset.")
+
+        self._validate_compatible_datasets(datasets)
+        self._warn_if_runtime_transforms_differ(datasets)
+
+        self.datasets = [deepcopy(dataset) for dataset in datasets]
+        self.prefix_names = bool(prefix_names)
+
+        self._cumulative_sizes = self._build_cumulative_sizes(self.datasets)
+        self._cat_ids = dict(self.datasets[0].category_ids)
+        self._declared_reader_type = self.datasets[0]._effective_type()
+
+        # Compatibility attributes. They are not used for loading by this class,
+        # but keeping them available makes the object behave like a DeepVisionDataset.
+        self.root = self.datasets[0].root
+        self.image_dir = self.datasets[0].image_dir
+        self.reader = self.datasets[0].reader
+
+        self._scan = self._build_scan()
+
+        total_length = self._cumulative_sizes[-1]
+        if indices is None:
+            self._indices = list(range(total_length))
+        else:
+            self._indices = self._normalize_global_indices(indices, total_length)
+
+        self._writer: AnnotationWriter | None = None
+        if writer is None:
+            first_writer = self.datasets[0].writer
+            self.writer = deepcopy(first_writer) if first_writer is not None else None
+        else:
+            self.writer = writer
+
+    @staticmethod
+    def _build_cumulative_sizes(
+        datasets: Sequence[DeepVisionDataset],
+    ) -> list[int]:
+        cumulative_sizes: list[int] = []
+        total = 0
+
+        for dataset in datasets:
+            total += len(dataset)
+            cumulative_sizes.append(total)
+
+        return cumulative_sizes
+
+    @staticmethod
+    def _category_label_ids(dataset: DeepVisionDataset) -> tuple[int, ...]:
+        return tuple(sorted(int(label_id) for label_id in dataset.category_ids))
+
+    @classmethod
+    def _validate_compatible_datasets(
+        cls,
+        datasets: Sequence[DeepVisionDataset],
+    ) -> None:
+        first = datasets[0]
+        first_type = first._effective_type()
+        first_labels = cls._category_label_ids(first)
+        first_device = first.device
+
+        for dataset_index, dataset in enumerate(datasets[1:], start=1):
+            data_type = dataset._effective_type()
+            if data_type != first_type:
+                raise ValueError(
+                    "Cannot concatenate datasets with different effective data types: "
+                    f"dataset 0 has {first_type!r}, "
+                    f"dataset {dataset_index} has {data_type!r}."
+                )
+
+            if dataset.device != first_device:
+                raise ValueError(
+                    "Cannot concatenate datasets on different devices: "
+                    f"dataset 0 has {first_device!r}, "
+                    f"dataset {dataset_index} has {dataset.device!r}."
+                )
+
+            labels = cls._category_label_ids(dataset)
+            if labels != first_labels:
+                raise ValueError(
+                    "Cannot concatenate datasets with different category label ids: "
+                    f"dataset 0 has {first_labels}, "
+                    f"dataset {dataset_index} has {labels}. "
+                    "Only label ids are checked; category names are assumed to be "
+                    "semantically consistent by the user."
+                )
+
+    @staticmethod
+    def _preprocessing_signature(preprocessing: Callable | None) -> str:
+        if preprocessing is None:
+            return "None"
+
+        module = getattr(preprocessing, "__module__", "")
+        qualname = getattr(preprocessing, "__qualname__", "")
+        if module or qualname:
+            return f"{module}.{qualname}"
+
+        return repr(preprocessing)
+
+    @staticmethod
+    def _augmentation_signature(augmentation: Sequence[Transform]) -> tuple[str, ...]:
+        return tuple(repr(transform) for transform in augmentation)
+
+    @classmethod
+    def _warn_if_runtime_transforms_differ(
+        cls,
+        datasets: Sequence[DeepVisionDataset],
+    ) -> None:
+        first_preprocessing = cls._preprocessing_signature(datasets[0].preprocessing)
+        first_augmentation = cls._augmentation_signature(datasets[0].augmentation)
+
+        different_preprocessing = any(
+            cls._preprocessing_signature(dataset.preprocessing) != first_preprocessing
+            for dataset in datasets[1:]
+        )
+        different_augmentation = any(
+            cls._augmentation_signature(dataset.augmentation) != first_augmentation
+            for dataset in datasets[1:]
+        )
+
+        if different_preprocessing:
+            warnings.warn(
+                "DeepVisionConcatDataset received datasets with different "
+                "preprocessing callables. This is allowed, but samples from "
+                "different child datasets will be preprocessed differently unless "
+                "you reassign concat_dataset.preprocessing.",
+                UserWarning,
+                stacklevel=3,
+            )
+
+        if different_augmentation:
+            warnings.warn(
+                "DeepVisionConcatDataset received datasets with different "
+                "augmentation pipelines. This is allowed, but samples from "
+                "different child datasets will be augmented differently unless "
+                "you reassign concat_dataset.augmentation.",
+                UserWarning,
+                stacklevel=3,
+            )
+
+    def _build_scan(self) -> list[str]:
+        names: list[str] = []
+
+        for dataset_index, dataset in enumerate(self.datasets):
+            for local_index in range(len(dataset)):
+                base_index = dataset._indices[local_index]
+                image_name = dataset._scan[base_index]
+                names.append(self._format_image_name(dataset_index, image_name))
+
+        return names
+
+    @staticmethod
+    def _normalize_global_indices(
+        indices: Sequence[int],
+        total_length: int,
+    ) -> list[int]:
+        out: list[int] = []
+
+        for index in indices:
+            ii = int(index)
+            if ii < 0:
+                ii += total_length
+            if ii < 0 or ii >= total_length:
+                raise IndexError(
+                    f"Concat index {index} is out of range for dataset "
+                    f"of length {total_length}."
+                )
+            out.append(ii)
+
+        return out
+
+    @property
+    def preprocessing(self) -> Callable | None:
+        return self.datasets[0].preprocessing
+
+    @preprocessing.setter
+    def preprocessing(self, preprocessing: Callable | None) -> None:
+        for dataset in self.datasets:
+            dataset.preprocessing = preprocessing
+
+    @property
+    def augmentation(self) -> list[Transform]:
+        return self.datasets[0].augmentation
+
+    @augmentation.setter
+    def augmentation(self, augmentation: list[Transform] | None) -> None:
+        normalized = augmentation or []
+        for dataset in self.datasets:
+            dataset.augmentation = list(normalized)
+
+    @property
+    def device(self) -> str:
+        return self.datasets[0].device
+
+    @device.setter
+    def device(self, device: str) -> None:
+        for dataset in self.datasets:
+            dataset.device = device
+
+    @property
+    def cast_to(self) -> str | None:
+        return self.datasets[0].cast_to
+
+    @cast_to.setter
+    def cast_to(self, cast_to: str | None) -> None:
+        for dataset in self.datasets:
+            dataset.cast_to = cast_to
+
+    @property
+    def data_type(self) -> str:
+        return self._effective_type()
+
+    def _effective_type(self) -> str:
+        return self.cast_to or self._declared_reader_type or self.reader.out_type
+
+    def to_data_type(self, dst_key: str) -> DeepVisionConcatDataset:
+        """Return a concat view that converts every sample's target to dst_key lazily."""
+        out = self.keep_indices(slice(0, len(self)))
+        out.cast_to = dst_key
+        return out
+
+    def split(self, proportions: Sequence[float]) -> list[DeepVisionDataset]:
+        """
+        Randomly split the current concat view into concat sub-views.
+
+        Indices are interpreted relative to the current concat view.
+        """
+        s = sum(proportions)
+        assert round(s, 6) == 1.0, f"Split proportions must sum to 1, got {proportions}"
+
+        import random
+
+        positions = list(range(len(self)))
+        random.shuffle(positions)
+
+        n = len(positions)
+        cuts = [round(p * n) for p in proportions]
+
+        out: list[DeepVisionDataset] = []
+        start = 0
+
+        for c in cuts[:-1]:
+            out.append(self.keep_indices(positions[start : start + c]))
+            start += c
+
+        out.append(self.keep_indices(positions[start:]))
+        return out
+
+    def keep_indices(
+        self,
+        indices: list[int] | torch.Tensor | slice,
+    ) -> DeepVisionConcatDataset:
+        """
+        Keep a subset of samples from the current concat view.
+
+        The provided indices are positions relative to this concat view, not raw
+        child-dataset indices.
+        """
+        if isinstance(indices, torch.Tensor):
+            if indices.ndim != 1:
+                raise ValueError(f"indices must be a 1-D tensor, got {indices.shape}")
+            index_list = [int(i) for i in indices.tolist()]
+        elif isinstance(indices, slice):
+            index_list = list(range(len(self)))[indices]
+        else:
+            index_list = [int(i) for i in indices]
+
+        selected_global_indices: list[int] = []
+
+        for index in index_list:
+            ii = int(index)
+            if ii < 0:
+                ii += len(self)
+            if ii < 0 or ii >= len(self):
+                raise IndexError(
+                    f"Index {index} is out of range for concat view "
+                    f"of length {len(self)}."
+                )
+            selected_global_indices.append(self._indices[ii])
+
+        return DeepVisionConcatDataset(
+            self.datasets,
+            writer=deepcopy(self.writer),
+            prefix_names=self.prefix_names,
+            indices=selected_global_indices,
+        )
+
+    def _locate_global_index(self, global_index: int) -> tuple[int, int]:
+        previous_size = 0
+
+        for dataset_index, cumulative_size in enumerate(self._cumulative_sizes):
+            if global_index < cumulative_size:
+                local_index = global_index - previous_size
+                return dataset_index, local_index
+
+            previous_size = cumulative_size
+
+        raise RuntimeError("Failed to locate sample in DeepVisionConcatDataset.")
+
+    def _format_image_name(self, dataset_index: int, image_name: str) -> str:
+        if not self.prefix_names:
+            return image_name
+
+        return f"dataset_{dataset_index}__{image_name}"
+
+    def __len__(self) -> int:
+        return len(self._indices)
+
+    def __getitem__(self, index: int) -> tuple[torch.Tensor, BaseData, str]:
+        if index < 0:
+            index += len(self)
+
+        if index < 0 or index >= len(self):
+            raise IndexError(
+                f"Index {index} is out of range for concat view of length {len(self)}."
+            )
+
+        global_index = self._indices[index]
+        dataset_index, local_index = self._locate_global_index(global_index)
+
+        image, target, image_name = self.datasets[dataset_index][local_index]
+        return image, target, self._format_image_name(dataset_index, image_name)
+
+
 def _pad_to(img: torch.Tensor, target_shape: tuple[int, int]) -> torch.Tensor:
     tarh, tarw = target_shape
     h, w = img.shape[-2:]