deepvisiontools 0.0.4__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/PKG-INFO +21 -16
  2. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/README.md +4 -2
  3. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/pyproject.toml +14 -10
  4. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/__init__.py +3 -1
  5. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/config/config.py +3 -0
  6. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/data/dataset.py +5 -3
  7. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/formats/base_data.py +7 -1
  8. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/formats/base_formats.py +7 -2
  9. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/inference/__init__.py +2 -1
  10. deepvisiontools-0.2.0/src/deepvisiontools/inference/evaluator.py +256 -0
  11. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/inference/predictor.py +46 -9
  12. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/metrics/available_metrics.py +1 -1
  13. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/metrics/base_metric.py +268 -21
  14. deepvisiontools-0.2.0/src/deepvisiontools/metrics/func.py +36 -0
  15. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/models/__init__.py +6 -4
  16. deepvisiontools-0.2.0/src/deepvisiontools/models/timmyolo/__init__.py +4 -0
  17. deepvisiontools-0.2.0/src/deepvisiontools/models/timmyolo/errors.py +5 -0
  18. deepvisiontools-0.2.0/src/deepvisiontools/models/timmyolo/timmyolo.py +392 -0
  19. deepvisiontools-0.2.0/src/deepvisiontools/models/timmyolo/utils.py +230 -0
  20. {deepvisiontools-0.0.4/src/deepvisiontools/models/yolov8 → deepvisiontools-0.2.0/src/deepvisiontools/models/yolo}/__init__.py +3 -3
  21. {deepvisiontools-0.0.4/src/deepvisiontools/models/yolov8 → deepvisiontools-0.2.0/src/deepvisiontools/models/yolo}/errors.py +1 -1
  22. deepvisiontools-0.0.4/src/deepvisiontools/models/yolov8/yolov8.py → deepvisiontools-0.2.0/src/deepvisiontools/models/yolo/yolo.py +19 -7
  23. deepvisiontools-0.2.0/src/deepvisiontools/models/yoloseg/__init__.py +4 -0
  24. {deepvisiontools-0.0.4/src/deepvisiontools/models/yolov8seg → deepvisiontools-0.2.0/src/deepvisiontools/models/yoloseg}/utils.py +1 -1
  25. deepvisiontools-0.0.4/src/deepvisiontools/models/yolov8seg/yolov8seg.py → deepvisiontools-0.2.0/src/deepvisiontools/models/yoloseg/yoloseg.py +32 -11
  26. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/train/trainer.py +32 -12
  27. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools.egg-info/PKG-INFO +21 -16
  28. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools.egg-info/SOURCES.txt +13 -8
  29. deepvisiontools-0.2.0/src/deepvisiontools.egg-info/requires.txt +10 -0
  30. deepvisiontools-0.0.4/src/deepvisiontools/metrics/func.py +0 -34
  31. deepvisiontools-0.0.4/src/deepvisiontools/models/yolov8seg/__init__.py +0 -4
  32. deepvisiontools-0.0.4/src/deepvisiontools.egg-info/requires.txt +0 -9
  33. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/LICENSE.txt +0 -0
  34. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/setup.cfg +0 -0
  35. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/setup.py +0 -0
  36. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/config/__init__.py +0 -0
  37. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/config/errors.py +0 -0
  38. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/data/__init__.py +0 -0
  39. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/data/additional_augmentations.py +0 -0
  40. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/data/augmentation_class.py +0 -0
  41. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/data/batch_augmentations.py +0 -0
  42. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/data/data_reader/__init__.py +0 -0
  43. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/data/data_reader/readers.py +0 -0
  44. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/data/errors.py +0 -0
  45. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/formats/__init__.py +0 -0
  46. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/formats/errors.py +0 -0
  47. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/formats/formats.py +0 -0
  48. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/formats/utils.py +0 -0
  49. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/inference/patchifier.py +0 -0
  50. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/metrics/__init__.py +0 -0
  51. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/metrics/matcher.py +0 -0
  52. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/metrics/utils.py +0 -0
  53. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/models/basemodel.py +0 -0
  54. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/models/mask2former/__init__.py +0 -0
  55. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/models/mask2former/mask2former.py +0 -0
  56. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/models/smp/__init__.py +0 -0
  57. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/models/smp/errors.py +0 -0
  58. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/models/smp/smp.py +0 -0
  59. {deepvisiontools-0.0.4/src/deepvisiontools/models/yolov8 → deepvisiontools-0.2.0/src/deepvisiontools/models/yolo}/utils.py +0 -0
  60. {deepvisiontools-0.0.4/src/deepvisiontools/models/yolov8seg → deepvisiontools-0.2.0/src/deepvisiontools/models/yoloseg}/errors.py +0 -0
  61. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/preprocessing/__init__.py +0 -0
  62. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/preprocessing/image.py +0 -0
  63. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/preprocessing/preprocessing.py +0 -0
  64. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/train/__init__.py +0 -0
  65. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/utils/__init__.py +0 -0
  66. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/utils/errors.py +0 -0
  67. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools/utils/visualization.py +0 -0
  68. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools.egg-info/dependency_links.txt +0 -0
  69. {deepvisiontools-0.0.4 → deepvisiontools-0.2.0}/src/deepvisiontools.egg-info/top_level.txt +0 -0
@@ -1,9 +1,9 @@
1
- Metadata-Version: 2.2
1
+ Metadata-Version: 2.4
2
2
  Name: deepvisiontools
3
- Version: 0.0.4
4
- Summary: Overlay of PyTorch to generalize trainning and inference processes for detection & instance segmentation tasks.
5
- Author-email: Lucas Bernigaud Samatan <lucas.bernigaud-samatan@inrae.fr>, Jordan Bernigaud Samatan <jordan.bernigaud-samatan@inrae.fr>
6
- Maintainer-email: Jordan Bernigaud Samatan <jordan.bernigaud-samatan@inrae.fr>
3
+ Version: 0.2.0
4
+ Summary: Overlay of PyTorch to generalize trainning and inference processes for detection, instance segmentation and semantic segmentation tasks.
5
+ Author-email: Lucas Bernigaud Samatan <lucas.bernigaud-samatan@inrae.fr>, Jordan Bernigaud Samatan <jordan.bernigaud-samatan@inrae.fr>, Paul Faucher <paul.faucher@inrae.fr>
6
+ Maintainer-email: Jordan Bernigaud Samatan <jordan.bernigaud-samatan@inrae.fr>, Lucas Bernigaud Samatan <lucas.bernigaud-samatan@inrae.fr>
7
7
  License: GNU AFFERO GENERAL PUBLIC LICENSE
8
8
  Version 3, 19 November 2007
9
9
 
@@ -670,20 +670,23 @@ Project-URL: Homepage, https://forgemia.inra.fr/ue-apc/librairies/python/deepvis
670
670
  Project-URL: Issues, https://forgemia.inra.fr/ue-apc/librairies/python/deepvisiontools_v1/-/issues
671
671
  Project-URL: Documentation, https://deepvisiontools.readthedocs.io/en/latest/
672
672
  Project-URL: Repository, https://forgemia.inra.fr/ue-apc/librairies/python/deepvisiontools_v1
673
- Classifier: Development Status :: 3 - Alpha
673
+ Classifier: Development Status :: 4 - Beta
674
674
  Classifier: Programming Language :: Python :: 3.11
675
- Requires-Python: >=3.11.9
675
+ Classifier: Programming Language :: Python :: 3.12
676
+ Requires-Python: <=3.12.12,>=3.11.9
676
677
  Description-Content-Type: text/markdown
677
678
  License-File: LICENSE.txt
678
- Requires-Dist: torch==2.4.0
679
- Requires-Dist: torchvision==0.19.0
679
+ Requires-Dist: torch<=2.6.0,>=2.4.0
680
+ Requires-Dist: torchvision<=0.21.0,>=0.19.0
680
681
  Requires-Dist: pycocotools==2.0.8
681
- Requires-Dist: torchmetrics==1.4.0
682
- Requires-Dist: tensorboard==2.18.0
683
- Requires-Dist: ultralytics==8.3.28
684
- Requires-Dist: transformers==4.44.2
682
+ Requires-Dist: torchmetrics<=1.7.0,>=1.4.0
683
+ Requires-Dist: tensorboard<=2.19.0,>=2.18.0
684
+ Requires-Dist: ultralytics<=8.3.95,>=8.3.28
685
+ Requires-Dist: transformers<=4.50.0,>=4.44.2
685
686
  Requires-Dist: colorcet==3.1.0
686
- Requires-Dist: segmentation-models-pytorch==0.3.4
687
+ Requires-Dist: segmentation-models-pytorch<=0.5.0,>0.4.0
688
+ Requires-Dist: openpyxl==3.1.5
689
+ Dynamic: license-file
687
690
 
688
691
  <!--
689
692
  <p align="center">
@@ -704,20 +707,22 @@ deepvisiontools is developped by INRAE (french National Research Institute for A
704
707
 
705
708
  ## Citations
706
709
 
707
- deepvisiontools provide models wrapper that allow you to use differents models with all machinery available. It is the resposability of the user to cite properly the actual model developpers.
710
+ deepvisiontools provides model wrappers that allow you to use differents models with all machinery available. It is the responsability of the user to cite properly the actual model developpers.
708
711
  In particular, if using one of these models : Yolo, SMP please cite the actual models developpers (which are not deepvisiontools). For instance for Yolo, please cite ultralytics (https://www.ultralytics.com/) or for SMP cite https://github.com/qubvel-org/segmentation_models.pytorch
709
712
  Similar stands for other models.
713
+ TimmYolo model is a new model that combines timm encoders with a Yolo detection head. The strategy is based on patchification for the backbone (to avoid loosing resolution in resizing) then merge the features before running a Yolo detection heads. If you use this wrapper please cite both timm library, ultralytics library as well as deepvisiontools as model developper.
710
714
 
711
715
  ## Documentation
712
716
 
713
717
  All documentation about deepvisiontools, including tutorials, can be found at : https://deepvisiontools.readthedocs.io/en/latest/
714
718
 
715
719
  ## Installation
720
+ We have tested deepvisiontools for python versions between 3.11.9 and 3.12.11, we recommend to use python 3.12.11 to minimize the risk of bugs.
716
721
 
717
722
  ```shell
718
723
  pip install deepvisiontools
719
724
  ```
720
-
725
+ If you wish to use TimmYolo model, you may encounter an error saying that feature_only is not available for a given encoder name. Please upgrade your timm version (1.0.15 should have most of them). You may need to reinstall deepvisiontools as you could create conflicts with segmentation-models-pytorch.
721
726
  ## License
722
727
 
723
728
  GNU Affero General Public License v3 or later (AGPLv3+) (AGPL-3.0).
@@ -17,20 +17,22 @@ deepvisiontools is developped by INRAE (french National Research Institute for A
17
17
 
18
18
  ## Citations
19
19
 
20
- deepvisiontools provide models wrapper that allow you to use differents models with all machinery available. It is the resposability of the user to cite properly the actual model developpers.
20
+ deepvisiontools provides model wrappers that allow you to use differents models with all machinery available. It is the responsability of the user to cite properly the actual model developpers.
21
21
  In particular, if using one of these models : Yolo, SMP please cite the actual models developpers (which are not deepvisiontools). For instance for Yolo, please cite ultralytics (https://www.ultralytics.com/) or for SMP cite https://github.com/qubvel-org/segmentation_models.pytorch
22
22
  Similar stands for other models.
23
+ TimmYolo model is a new model that combines timm encoders with a Yolo detection head. The strategy is based on patchification for the backbone (to avoid loosing resolution in resizing) then merge the features before running a Yolo detection heads. If you use this wrapper please cite both timm library, ultralytics library as well as deepvisiontools as model developper.
23
24
 
24
25
  ## Documentation
25
26
 
26
27
  All documentation about deepvisiontools, including tutorials, can be found at : https://deepvisiontools.readthedocs.io/en/latest/
27
28
 
28
29
  ## Installation
30
+ We have tested deepvisiontools for python versions between 3.11.9 and 3.12.11, we recommend to use python 3.12.11 to minimize the risk of bugs.
29
31
 
30
32
  ```shell
31
33
  pip install deepvisiontools
32
34
  ```
33
-
35
+ If you wish to use TimmYolo model, you may encounter an error saying that feature_only is not available for a given encoder name. Please upgrade your timm version (1.0.15 should have most of them). You may need to reinstall deepvisiontools as you could create conflicts with segmentation-models-pytorch.
34
36
  ## License
35
37
 
36
38
  GNU Affero General Public License v3 or later (AGPLv3+) (AGPL-3.0).
@@ -7,28 +7,32 @@ name = "deepvisiontools"
7
7
  authors = [
8
8
  { name = "Lucas Bernigaud Samatan", email = "lucas.bernigaud-samatan@inrae.fr" },
9
9
  { name = "Jordan Bernigaud Samatan", email = "jordan.bernigaud-samatan@inrae.fr" },
10
+ { name = "Paul Faucher", email = "paul.faucher@inrae.fr" },
10
11
  ]
11
12
  maintainers = [
12
13
  { name = "Jordan Bernigaud Samatan", email = "jordan.bernigaud-samatan@inrae.fr" },
14
+ { name = "Lucas Bernigaud Samatan", email = "lucas.bernigaud-samatan@inrae.fr" },
13
15
  ]
14
- description = "Overlay of PyTorch to generalize trainning and inference processes for detection & instance segmentation tasks."
16
+ description = "Overlay of PyTorch to generalize trainning and inference processes for detection, instance segmentation and semantic segmentation tasks."
15
17
  readme = "README.md"
16
18
  classifiers = [
17
- "Development Status :: 3 - Alpha",
19
+ "Development Status :: 4 - Beta",
18
20
  "Programming Language :: Python :: 3.11",
21
+ "Programming Language :: Python :: 3.12"
19
22
  ]
20
23
  dynamic = ["version"]
21
- requires-python = ">=3.11.9"
24
+ requires-python = ">=3.11.9,<=3.12.12"
22
25
  dependencies = [
23
- "torch==2.4.0",
24
- "torchvision==0.19.0",
26
+ "torch>=2.4.0,<=2.6.0",
27
+ "torchvision>=0.19.0,<=0.21.0",
25
28
  "pycocotools==2.0.8",
26
- "torchmetrics==1.4.0",
27
- "tensorboard==2.18.0",
28
- "ultralytics==8.3.28",
29
- "transformers==4.44.2",
29
+ "torchmetrics>=1.4.0,<=1.7.0",
30
+ "tensorboard>=2.18.0,<=2.19.0",
31
+ "ultralytics>=8.3.28,<=8.3.95",
32
+ "transformers>=4.44.2,<=4.50.0",
30
33
  "colorcet==3.1.0",
31
- "segmentation-models-pytorch==0.3.4"
34
+ "segmentation-models-pytorch>0.4.0,<=0.5.0",
35
+ "openpyxl==3.1.5"
32
36
  ]
33
37
 
34
38
  license = { file = "LICENSE.txt" }
@@ -3,8 +3,9 @@ from deepvisiontools.train.trainer import Trainer
3
3
  from deepvisiontools.data.dataset import DeepVisionDataset, DeepVisionLoader
4
4
  from deepvisiontools.utils import visualization
5
5
  from deepvisiontools.inference.predictor import Predictor
6
+ from deepvisiontools.inference.evaluator import Evaluator
6
7
 
7
- __version__ = "0.0.4"
8
+ __version__ = "0.2.0"
8
9
 
9
10
  __all__ = (
10
11
  Configuration,
@@ -13,4 +14,5 @@ __all__ = (
13
14
  DeepVisionLoader,
14
15
  visualization,
15
16
  Predictor,
17
+ Evaluator,
16
18
  )
@@ -105,6 +105,7 @@ class Configuration(metaclass=_SingletonConfig):
105
105
  patchifier_mode: Literal["bbox", "instance_mask"] = "bbox",
106
106
  seed: Union[False, int] = False,
107
107
  deterministic: bool = False,
108
+ optimize=True,
108
109
  ):
109
110
 
110
111
  # generic
@@ -127,6 +128,8 @@ class Configuration(metaclass=_SingletonConfig):
127
128
  # reproducibility
128
129
  self.seed = seed
129
130
  self.deterministic = deterministic
131
+ # mixed precision (torch.amp + gradiant scaler)
132
+ self.optimize = optimize
130
133
 
131
134
  @property
132
135
  def device(self):
@@ -191,19 +191,21 @@ class DeepVisionDataset(Dataset):
191
191
  dataset3._indexes = idx[stop2:]
192
192
  return dataset1, dataset2, dataset3
193
193
 
194
- def keep_indexes(self, indexes: Union[list, slice, Tensor]):
194
+ def keep_indexes(self, indexes: Union[list, slice, Tensor]) -> DeepVisionDataset:
195
195
  """Filter dataset by keeping only indices given in arg.
196
196
 
197
197
  Args:
198
198
  indexes (``Union[list, slice, Tensor]``): can be slice, Tensor or list. To use slice please use : slice(i, j) with i, j desired slice indexes in arg.
199
199
  """
200
+ dataset = copy.deepcopy(self)
200
201
  if isinstance(indexes, Tensor):
201
202
  assert (
202
203
  indexes.dim() == 1
203
204
  ), f"Must use Tensor of dim 1 for indexes, got {indexes.shape}"
204
205
  indexes = indexes.tolist()
205
- indx = torch.tensor(self._indexes)[indexes].tolist()
206
- self._indexes = indx
206
+ indx = torch.tensor(dataset._indexes)[indexes].tolist()
207
+ dataset._indexes = indx
208
+ return dataset
207
209
 
208
210
  def export_dataset(
209
211
  self,
@@ -144,7 +144,13 @@ class SemanticMaskData(BaseData):
144
144
  mask = Mask(mask[0])
145
145
  self._value = copy.deepcopy(mask) # convert to long
146
146
  self._canvas_size = (mask.shape[-2], mask.shape[-1])
147
- self._nb_object = torch.unique(mask).shape[0] - 1
147
+ # compute nb of objects (unique cls ids) in mask that are not background (i.e 0)
148
+ nb_object = (
149
+ torch.unique(mask).shape[0] - 1
150
+ if 0 in torch.unique(mask).tolist()
151
+ else torch.unique(mask).shape[0]
152
+ )
153
+ self._nb_object = nb_object
148
154
  self._device = Configuration().device
149
155
  # we use here a trick : handling scores operation is complicated at format level.
150
156
  # We dynamically will modify it in the operationhandler class and implement the modification here
@@ -190,8 +190,13 @@ class BaseSemanticFormat(BaseFormat):
190
190
  scores.shape[-2:] == data.value.shape[-2:]
191
191
  ), f"scores do not have the same shape as data.value, got {scores.shape} and {data.value.shape}"
192
192
  self._data = data
193
- # labels are derived from semantic mask data value
194
- self._labels: Tensor = data.value.unique()[1:].long()
193
+ # labels are derived from semantic mask data value (careful : need to handle the case where 0 is not present in mask ...)
194
+ labs = (
195
+ data.value.unique()[1:].long()
196
+ if 0 in data.value.unique().tolist()
197
+ else data.value.unique().long()
198
+ )
199
+ self._labels: Tensor = labs
195
200
  # Check score
196
201
  if scores != None:
197
202
  scores = (
@@ -1,4 +1,5 @@
1
1
  from deepvisiontools.inference.patchifier import BasePatchifier, SemanticPatchifier, DetectPatchifier, PostProcesser
2
2
  from deepvisiontools.inference.predictor import Predictor, PredictorDataLoader
3
+ from deepvisiontools.inference.evaluator import Evaluator
3
4
 
4
- __all__ = ("Predictor", "BasePatchifier", "DetectPatchifier", "SemanticPatchifier", "PostProcesser", "PredictorDataLoader")
5
+ __all__ = ("Predictor", "BasePatchifier", "DetectPatchifier", "SemanticPatchifier", "PostProcesser", "PredictorDataLoader", "Evaluator")
@@ -0,0 +1,256 @@
1
+ from deepvisiontools import DeepVisionDataset, Configuration
2
+ from deepvisiontools.inference import Predictor
3
+ from deepvisiontools.utils import visualization
4
+ from tqdm import tqdm
5
+ from pathlib import Path
6
+ import pandas as pd
7
+ from typing import Literal, Union
8
+ from openpyxl import load_workbook
9
+ from openpyxl.styles import Font, PatternFill
10
+ import random
11
+
12
+
13
+ class Evaluator:
14
+ """Evaluator class : evaluate a given Predictor (model + patch_size + additional Configurations) on a dataset with given metrics.
15
+ The results are saved in a generated csv file (metrics at dataset level) and in a xlsx file (metrics at dataset and sample levels).
16
+ Highlighting samples that deviate from the mean or median (giving deviation_method) by nb_sigma sigma in the xlsx file.
17
+ Create visualizations of predictions giving number_visu.
18
+ Returns the dictionnary with metrics at dataset level and the dictionnary with metrics at sample level.
19
+ Prints a dataframe with metrics at dataset level (same content as in the generated csv file).
20
+
21
+ Args:
22
+ predictor (``Predictor``): Predictor class to evaluate
23
+ metrics (``list``): List of metrics to evaluate the Predictor on
24
+ deviation_method (``Literal[&quot;mean&quot;, &quot;median&quot;]``, **optional**): method to compute outlayers. Defaults to "mean".
25
+ nb_sigma (``Union[int, float]``, **optional**): number of standard deviations for outlayers. Defaults to 2.
26
+
27
+ Example:
28
+ ----------
29
+
30
+ .. highlight:: python
31
+ .. code-block:: python
32
+
33
+ >>> from deepvisiontools import Evaluator, Predictor
34
+ >>> from deepvisiontools.metrics import DetectF1Score
35
+ >>> predictor = Predictor(model=\path\to\model.pth)
36
+ >>> evaluator = Evaluator(predictor, metrics=[DetectF1Score()])
37
+ >>> evaluator.evaluate(mydataset, "results")
38
+
39
+ Attributes
40
+ ----------
41
+
42
+ Attributes:
43
+ predictor (``Predictor``)
44
+ metrics (``List[BaseMetric]``)
45
+ nb_sigma (``int``)
46
+ data_type (``Literal[&quot;instance_mask&quot;, &quot;bbox&quot;, &quot;keypoint&quot;, &quot;semantic_mask&quot;]``)
47
+ deviation_method (```Literal[&quot;mean&quot;, &quot;median&quot;]``)
48
+
49
+
50
+ **Methods**
51
+
52
+ """
53
+
54
+ def __init__(
55
+ self,
56
+ predictor: Predictor,
57
+ metrics: list,
58
+ deviation_method: Literal["mean", "median"] = "mean",
59
+ nb_sigma: Union[int, float] = 2,
60
+ ):
61
+
62
+ self.predictor = predictor
63
+ self.metrics = metrics
64
+ self.nb_sigma = nb_sigma
65
+ self.data_type = Configuration().data_type
66
+ self.deviation_method = deviation_method
67
+ assert deviation_method in [
68
+ "mean",
69
+ "median",
70
+ ], "deviation_method must be 'mean' or 'median'"
71
+
72
+ def evaluate(
73
+ self,
74
+ dataset: DeepVisionDataset,
75
+ result_folder: Union[str, Path],
76
+ number_visu: Union[Literal["all"], int] = "all",
77
+ ):
78
+ """Run evaluation on dataset. Compute metrics for dataset and for each sample of the dataset."""
79
+ result_folder = (
80
+ result_folder if isinstance(result_folder, Path) else Path(result_folder)
81
+ )
82
+ result_folder.mkdir(parents=True, exist_ok=True)
83
+ export_csv_path = result_folder / "results.csv"
84
+ export_xlsx_path = result_folder / "results.xlsx"
85
+ visu_folder_path = result_folder / "visu"
86
+
87
+ assert (
88
+ isinstance(number_visu, int) or number_visu == "all"
89
+ ), "number_visu must be integer or equal to 'all'"
90
+
91
+ dataset.preprocessing = None # because already preprocess in Predictor
92
+
93
+ # Creation and initialisation of the dictionary containing the metrics for each image (one dictionary/metric containing one dictionary/image)
94
+ metric_sample_dict = {}
95
+ for metric in self.metrics:
96
+ metric.to(Configuration().device)
97
+ metric_sample_dict.update({metric.name: {}})
98
+
99
+ if (
100
+ number_visu != 0 and number_visu != "all"
101
+ ): # randomly select index of the number_visu images to visualize
102
+ list_index_img = [i for i in range(len(dataset))]
103
+ random.shuffle(list_index_img)
104
+ list_index_img_visu = list_index_img[:number_visu]
105
+
106
+ # For each sample in the dataset : prediction then calculation and storage of tp, fp, tn, fn (for each metric)
107
+ iterator = tqdm(dataset, total=len(dataset), desc=f"Evaluation progress ")
108
+
109
+ for index_img, (image, target, img_name) in enumerate(iterator):
110
+ prediction = self.predictor.predict(image)
111
+
112
+ for metric in self.metrics:
113
+ metric.update(
114
+ prediction, target
115
+ ) # update 'stats' attribute for each metric
116
+
117
+ metric_sample_dict[metric.name].update(
118
+ {img_name: {"img_index_in_dataset": index_img}}
119
+ ) # creation of the image dictionary (for the given metric) with its index information
120
+ # compute metric for the image (all classes combined and for each class)
121
+ metric_sample = metric.compute_last_sample()
122
+ metric_sample_dict[metric.name][img_name].update(metric_sample)
123
+
124
+ # prediction visualization
125
+ if number_visu != 0 and number_visu != "all":
126
+ if index_img in list_index_img_visu:
127
+ visualization(
128
+ image,
129
+ prediction,
130
+ save_path=visu_folder_path
131
+ / "visu_img{0}_{1}".format(index_img, img_name),
132
+ )
133
+ elif number_visu == "all":
134
+ visualization(
135
+ image,
136
+ prediction,
137
+ save_path=visu_folder_path
138
+ / "visu_img{0}_{1}".format(index_img, img_name),
139
+ )
140
+
141
+ # after all updates recompute to get averaged values of metric (global, samplewise, macro, micro), save in metric_dict
142
+ metric_dict = {}
143
+ for metric in self.metrics:
144
+ results = metric.compute()
145
+ metric_dict.update({metric.name: results})
146
+
147
+ # Converts dictionary metric values from tensor to float :
148
+ for metric_name in metric_dict:
149
+ for sous_metric_name in metric_dict[metric_name]:
150
+ metric_dict[metric_name][sous_metric_name] = round(
151
+ metric_dict[metric_name][sous_metric_name].item(), 3
152
+ )
153
+
154
+ # Converts the dict into a dataframe, which is saved in a csv file:
155
+ df_metrics = pd.DataFrame(metric_dict)
156
+ df_metrics.to_csv(export_csv_path)
157
+
158
+ # Create the Excel file with metrics values for each image
159
+ sheet_names = self._create_xls(export_xlsx_path, df_metrics, metric_sample_dict)
160
+
161
+ # Modifies the Excel file with metrics : highlights extreme values and adds mean, std, median
162
+ self._formate_xls(export_xlsx_path, sheet_names, metric_sample_dict)
163
+
164
+ print(df_metrics)
165
+ return (metric_dict, metric_sample_dict)
166
+
167
+ def _create_xls(self, export_xlsx_path, df_metrics, metric_sample_dict):
168
+ """Create an Excel file with metrics values for each image"""
169
+ sheet_names = []
170
+ with pd.ExcelWriter(export_xlsx_path) as writer:
171
+ df_metrics.to_excel(
172
+ writer, sheet_name="Dataset_Metrics", index=True
173
+ ) # creates a first sheet with the 'sub-metrics' for each metric (equivalent to the contents of the csv generated earlier)
174
+ for metric in metric_sample_dict:
175
+ df_sample = pd.DataFrame(metric_sample_dict[metric]).T
176
+ df_sample["Outlier?"] = None
177
+ sheet_name = "{0}_by_samples".format(metric)
178
+ if len(sheet_name) > 31: # the length of excel sheets name is 31
179
+ sheet_name = sheet_name[: len(sheet_name) - (len(sheet_name) - 31)]
180
+ df_sample.to_excel(
181
+ writer, sheet_name=sheet_name, index=True
182
+ ) # creates a sheet by metric with details by image
183
+ sheet_names.append(sheet_name)
184
+ return sheet_names
185
+
186
+ def _highlight_cells(self, sheet, colonne_verif, min_val, max_val):
187
+ """Applies a red background and bold text if the value is outside the range"""
188
+ # Style for formatting (red fill + bold text)
189
+ red_fill = PatternFill(
190
+ start_color="FFC7CE", end_color="FFC7CE", fill_type="solid"
191
+ )
192
+ bold_font = Font(bold=True, color="9C0006")
193
+
194
+ for row in sheet.iter_rows(
195
+ min_row=2, min_col=colonne_verif, max_col=colonne_verif
196
+ ):
197
+ for cell in row:
198
+ if cell.value is not None and (
199
+ cell.value < min_val or cell.value > max_val
200
+ ):
201
+ cell.fill = red_fill
202
+ sheet.cell(row=cell.row, column=1).font = (
203
+ bold_font # Make the index bold and red
204
+ )
205
+ sheet.cell(cell.row, sheet.max_column).value = (
206
+ "True" # True in "Outlier?" column
207
+ )
208
+
209
+ def _formate_xls(self, export_xlsx_path, sheet_names, metric_sample_dict):
210
+ """Modifies the Excel file with metrics : highlights extreme values and adds mean, std, median"""
211
+
212
+ wb = load_workbook(export_xlsx_path)
213
+
214
+ for sh_name, metric in zip(sheet_names, metric_sample_dict):
215
+
216
+ df_sample = pd.DataFrame(metric_sample_dict[metric]).T
217
+ columns_list = df_sample.columns.tolist()
218
+ sheet_maw_row = wb[sh_name].max_row
219
+
220
+ # add mean, std, median labels in last rows
221
+ wb[sh_name].cell(sheet_maw_row + 2, 1).value = "Mean : "
222
+ wb[sh_name].cell(sheet_maw_row + 3, 1).value = "Std : "
223
+ wb[sh_name].cell(sheet_maw_row + 4, 1).value = "Median : "
224
+
225
+ # add mean, std and median values in last rows for each class and "all_class"
226
+ for num_col, col_name in enumerate(columns_list[1:]):
227
+ mean = df_sample.mean()[col_name]
228
+ std = df_sample.std()[col_name]
229
+ median = df_sample.median()[col_name]
230
+
231
+ num_col_xsl = num_col + 3
232
+ # highlights extreme values
233
+ if self.deviation_method == "median":
234
+ self._highlight_cells(
235
+ wb[sh_name],
236
+ colonne_verif=num_col_xsl,
237
+ min_val=median - (self.nb_sigma * std),
238
+ max_val=median + (self.nb_sigma * std),
239
+ )
240
+ else:
241
+ self._highlight_cells(
242
+ wb[sh_name],
243
+ colonne_verif=num_col_xsl,
244
+ min_val=mean - (self.nb_sigma * std),
245
+ max_val=mean + (self.nb_sigma * std),
246
+ )
247
+
248
+ # add mean, std and median in last rows
249
+ wb[sh_name].cell(sheet_maw_row + 2, num_col_xsl).value = round(mean, 3)
250
+ wb[sh_name].cell(sheet_maw_row + 3, num_col_xsl).value = round(std, 3)
251
+ wb[sh_name].cell(sheet_maw_row + 4, num_col_xsl).value = round(
252
+ median, 3
253
+ )
254
+
255
+ # Save file after modifications
256
+ wb.save(export_xlsx_path)
@@ -16,6 +16,7 @@ from deepvisiontools.inference.patchifier import (
16
16
  )
17
17
  from torchvision.transforms.v2 import Pad, CenterCrop
18
18
  from tqdm import tqdm
19
+ from deepvisiontools.formats import SemanticMaskData, SemanticMaskFormat
19
20
 
20
21
 
21
22
  def build_patchifier(*args, **kwargs):
@@ -39,7 +40,8 @@ class Predictor:
39
40
  nms_iou_threshold (``float``, **optional**): nms threshold to be used when upatchifying. Defaults to 0.45.
40
41
  final_score_threshold (``float``, **optional**): Apply a score thresholding after penalty and after nms. Defaults to 0.4.
41
42
  categories (``Dict[int, str]``, **optional**): To rename your categories in the visualization.
42
- patchifier (``Union[BasePatchifier, None], **optional``): If None use default SemanticPatchifier or DetectPatchifier according to Configuration().data_type. Default to None.
43
+ patchifier (``Union[BasePatchifier, None], **optional**): If None use default SemanticPatchifier or DetectPatchifier according to Configuration().data_type. Default to None.
44
+ verbose (``bool``, **optional**) : if set to True will display progress state in patchs predictions. Default to True.
43
45
 
44
46
  Example:
45
47
  ----------
@@ -64,6 +66,7 @@ class Predictor:
64
66
  cropper (``Transform``)
65
67
  patchifier (``BasePatchifier``)
66
68
  categories (``Dict[int, str]``)
69
+ verbose (``bool``)
67
70
 
68
71
 
69
72
  **Methods**
@@ -79,6 +82,7 @@ class Predictor:
79
82
  batch_size: int = 1,
80
83
  categories: Dict[int, str] = None,
81
84
  patchifier: Union[BasePatchifier, None] = None,
85
+ verbose: bool = True,
82
86
  ):
83
87
  assert any(
84
88
  [
@@ -103,24 +107,36 @@ class Predictor:
103
107
  else:
104
108
  self.patchifier = patchifier
105
109
  self.categories = categories
110
+ self.verbose = verbose
106
111
 
107
112
  def forward_pass(self, batch_patchs: Tensor) -> BatchedFormat:
108
113
  """Run predictions on image / batch of patches"""
109
114
  with torch.no_grad():
110
115
  loader = PredictorDataLoader(batch_patchs, self.batch_size)
111
116
  # wrap in tqdm if more than one batch
112
- loader = (
113
- tqdm(loader, desc="Predict on batch of patches : ", total=len(loader))
114
- if len(loader) > 1
115
- else loader
116
- )
117
+ if self.verbose:
118
+ loader = (
119
+ tqdm(
120
+ loader, desc="Predict on batch of patches : ", total=len(loader)
121
+ )
122
+ if len(loader) > 1
123
+ else loader
124
+ )
117
125
  predictions = BatchedFormat([])
126
+
118
127
  for patch in loader:
119
- predictions += self.model.get_predictions(patch)
128
+ with torch.autocast(
129
+ device_type=Configuration().device,
130
+ dtype=torch.float16,
131
+ enabled=Configuration().optimize,
132
+ ):
133
+ predictions += self.model.get_predictions(patch)
120
134
  return predictions
121
135
 
122
136
  def predict(
123
- self, image: Union[str, Path, Tensor], visu_path: Union[str, Path] = ""
137
+ self,
138
+ image: Union[str, Path, Tensor],
139
+ visu_path: Union[str, Path] = "",
124
140
  ) -> BaseFormat:
125
141
  """Main function of ```Predictor``` : call everything needed for prediction.
126
142
 
@@ -143,12 +159,33 @@ class Predictor:
143
159
  h_original, w_original = image.shape[-2:]
144
160
  image = image.to(Configuration().device)
145
161
  # preprocess
146
- preprocessed_image = self.preprocessing(image[None, :])[0]
162
+ if self.preprocessing != None:
163
+ preprocessed_image = self.preprocessing(image[None, :])[0]
164
+ else:
165
+ preprocessed_image = image
147
166
  # handles patch if needed
148
167
  if self.patch_size != None and self.patch_size != (
149
168
  image.shape[-2],
150
169
  image.shape[-1],
151
170
  ):
171
+ _h_pad, _w_pad = 0, 0
172
+ if image.shape[-2] < self.patch_size[0]:
173
+ _h_pad = self.patch_size[0] - image.shape[-2]
174
+ _t_pad = int(_h_pad // 2)
175
+ _t_pad = _t_pad if _t_pad > 0 else 0
176
+ _b_pad = _h_pad - _t_pad
177
+ _b_pad = _b_pad if _b_pad > 0 else 0
178
+ if image.shape[-2] < self.patch_size[0]:
179
+ _w_pad = self.patch_size[1] - image.shape[-1]
180
+ _l_pad = int(_w_pad // 2)
181
+ _l_pad = _l_pad if _l_pad > 0 else 0
182
+ _r_pad = _w_pad - _l_pad
183
+ _r_pad = _r_pad if _r_pad > 0 else 0
184
+ if _h_pad > 0 or _w_pad > 0:
185
+ preprocessed_image = Pad((_l_pad, _t_pad, _r_pad, _b_pad))(
186
+ preprocessed_image
187
+ )
188
+
152
189
  batch_patch, pad_origins, padded_image, pad_coord = (
153
190
  self.patchifier.patchify(preprocessed_image)
154
191
  )
@@ -118,7 +118,7 @@ class ClassifRecall(ClassifMetric):
118
118
  super().__init__(func=F.recall, name="ClassifRecall", *args, **kwargs)
119
119
 
120
120
 
121
- class ClassifAccuracy(ClassifMetric):
121
+ class ClassifPrecision(ClassifMetric):
122
122
  """Classification precision score"""
123
123
 
124
124
  def __init__(self, *args, **kwargs):