konfai 1.2.3__tar.gz → 1.2.5__tar.gz

{konfai-1.2.3 → konfai-1.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: konfai
-Version: 1.2.3
+Version: 1.2.5
 Summary: Modular and configurable Deep Learning framework with YAML and PyTorch
 Author-email: Valentin Boussot <boussot.v@gmail.com>
 License-Expression: Apache-2.0

{konfai-1.2.3 → konfai-1.2.5}/konfai/data/data_manager.py

@@ -317,7 +317,6 @@ class Subset:
                 else:
                     index_set = index_set.intersection(set(self._get_index(s, names)))
                 index = list(index_set)
-                print(index)
         else:
             index = self._get_index(self.subset, names)
         if self.shuffle:
@@ -380,8 +379,8 @@ class Data(ABC):
             use_cache=use_cache,
         )
         self.dataLoader_args = {
-            "num_workers": int(os.environ["KONFAI_WORKERS"]) if use_cache else 0,
-            "pin_memory": True,
+            "num_workers": 0,
+            "pin_memory": False,
         }
         self.data: list[list[dict[str, list[DatasetManager]]]] = []
         self.mapping: list[list[list[tuple[int, int, int]]]] = []

{konfai-1.2.3 → konfai-1.2.5}/konfai/data/transform.py

@@ -48,12 +48,13 @@ class Clip(Transform):
 
     def __init__(
         self,
-        min_value: float = -1024,
-        max_value: float = 1024,
+        min_value: float | str = -1024,
+        max_value: float | str = 1024,
         save_clip_min: bool = False,
         save_clip_max: bool = False,
+        mask: str | None = None
     ) -> None:
-        if max_value <= min_value:
+        if isinstance(min_value, float) and isinstance(max_value, float) and max_value <= min_value:
             raise ValueError(
                 f"[Clip] Invalid clipping range: max_value ({max_value}) must be greater than min_value ({min_value})"
             )
@@ -61,14 +62,56 @@ class Clip(Transform):
         self.max_value = max_value
         self.save_clip_min = save_clip_min
         self.save_clip_max = save_clip_max
+        self.mask = mask
 
     def __call__(self, name: str, tensor: torch.Tensor, cache_attribute: Attribute) -> torch.Tensor:
-        tensor[torch.where(tensor < self.min_value)] = self.min_value
-        tensor[torch.where(tensor > self.max_value)] = self.max_value
+        mask = None
+        if self.mask is not None:
+            for dataset in self.datasets:
+                if dataset.is_dataset_exist(self.mask, name):
+                    mask, _ = dataset.read_data(self.mask, name)
+                    break
+        if mask is None and self.mask is not None:
+            raise ValueError(f"Requested mask '{self.mask}' is not present in any dataset. Check your dataset group names or configuration.")
+        if mask is None:
+            tensor_masked = tensor
+        else:
+            tensor_masked = tensor[mask == 1]
+
+        if isinstance(self.min_value, str):
+            if self.min_value == "min":
+                min_value = torch.min(tensor_masked)
+            elif self.min_value.startswith("percentile:"):
+                try:
+                    percentile = float(self.min_value.split(":")[1])
+                    min_value = np.percentile(tensor_masked, percentile)
+                except (IndexError, ValueError):
+                    raise ValueError(f"Invalid format for min_value: '{self.min_value}'. Expected 'percentile:<float>'")
+            else:
+                raise TypeError(f"Unsupported string for min_value: '{self.min_value}'. Must be a float, 'min', or 'percentile:<float>'.")
+        else:
+            min_value = self.min_value
+
+        if isinstance(self.max_value, str):
+            if self.max_value == "max":
+                max_value = torch.max(tensor_masked)
+            elif self.max_value.startswith("percentile:"):
+                try:
+                    percentile = float(self.max_value.split(":")[1])
+                    max_value = np.percentile(tensor_masked, percentile)
+                except (IndexError, ValueError):
+                    raise ValueError(f"Invalid format for max_value: '{self.max_value}'. Expected 'percentile:<float>'")
+            else:
+                raise TypeError(f"Unsupported string for max_value: '{self.max_value}'. Must be a float, 'max', or 'percentile:<float>'.")
+        else:
+            max_value = self.max_value
+        
+        tensor[torch.where(tensor < min_value)] = min_value
+        tensor[torch.where(tensor > max_value)] = max_value
         if self.save_clip_min:
-            cache_attribute["Min"] = self.min_value
+            cache_attribute["Min"] = min_value
         if self.save_clip_max:
-            cache_attribute["Max"] = self.max_value
+            cache_attribute["Max"] = max_value
         return tensor
 
     def inverse(self, name: str, tensor: torch.Tensor, cache_attribute: Attribute) -> torch.Tensor:
@@ -143,36 +186,42 @@ class Standardize(Transform):
         lazy: bool = False,
         mean: list[float] | None = None,
         std: list[float] | None = None,
+        mask: str | None = None
     ) -> None:
         self.lazy = lazy
         self.mean = mean
         self.std = std
+        self.mask = mask
 
     def __call__(self, name: str, tensor: torch.Tensor, cache_attribute: Attribute) -> torch.Tensor:
+        mask = None
+        if self.mask is not None:
+            for dataset in self.datasets:
+                if dataset.is_dataset_exist(self.mask, name):
+                    mask, _ = dataset.read_data(self.mask, name)
+                    break
+        if mask is None and self.mask is not None:
+            raise ValueError(f"Requested mask '{self.mask}' is not present in any dataset. Check your dataset group names or configuration.")
+        if mask is None:
+            tensor_masked = tensor
+        else:
+            tensor_masked = tensor[mask == 1]
+
         if "Mean" not in cache_attribute:
-            cache_attribute["Mean"] = (
-                torch.mean(
-                    tensor.type(torch.float32),
-                    dim=[i + 1 for i in range(len(tensor.shape) - 1)],
-                )
-                if self.mean is None
-                else torch.tensor([self.mean])
-            )
+            cache_attribute["Mean"] = torch.tensor([torch.mean(tensor_masked.type(torch.float32))]) if self.mean is None else torch.tensor([self.mean])
+            
         if "Std" not in cache_attribute:
             cache_attribute["Std"] = (
-                torch.std(
-                    tensor.type(torch.float32),
-                    dim=[i + 1 for i in range(len(tensor.shape) - 1)],
-                )
+                torch.tensor([torch.std(
+                    tensor_masked.type(torch.float32))])
                 if self.std is None
                 else torch.tensor([self.std])
             )
-
         if self.lazy:
             return tensor
         else:
-            mean = cache_attribute.get_tensor("Mean").view(-1, *[1 for _ in range(len(tensor.shape) - 1)])
-            std = cache_attribute.get_tensor("Std").view(-1, *[1 for _ in range(len(tensor.shape) - 1)])
+            mean = cache_attribute.get_tensor("Mean")
+            std = cache_attribute.get_tensor("Std")
             return (tensor - mean) / std
 
     def inverse(self, name: str, tensor: torch.Tensor, cache_attribute: Attribute) -> torch.Tensor:

{konfai-1.2.3 → konfai-1.2.5}/konfai/network/network.py

@@ -6,8 +6,7 @@ from collections import OrderedDict
 from collections.abc import Callable, Iterable, Iterator, Sequence
 from enum import Enum
 from functools import partial
-from typing import Any
-from typing_extensions import Self
+from typing import Any, Self
 
 import numpy as np
 import torch

{konfai-1.2.3 → konfai-1.2.5}/konfai/predictor.py

@@ -407,7 +407,6 @@ class _Predictor:
         """
 
         self.model_composite.eval()
-        self.model_composite.to(torch.float32)
         self.model_composite.module.set_state(NetState.PREDICTION)
         self.dataloader_prediction.dataset.load("Prediction")
         with tqdm.tqdm(
@@ -716,7 +715,9 @@ class Predictor(DistributedObject):
                         path = models_directory() + self.name + "/StateDict/"
                         name = sorted(os.listdir(path))[-1]
                 if os.path.exists(path + name):
-                    state_dicts.append(torch.load(path + name, map_location=torch.device('cpu'), weights_only=False))
+                    state_dicts.append(
+                        torch.load(path + name, map_location=torch.device("cpu"), weights_only=False)  # nosec B614
+                    )  # nosec B614
                 else:
                     raise Exception(f"Model : {path + name} does not exist !")
         return state_dicts

{konfai-1.2.3 → konfai-1.2.5}/konfai/trainer.py

@@ -632,7 +632,6 @@ class Trainer(DistributedObject):
             world_size (int): Total number of distributed processes.
         """
         state = State[konfai_state()]
-        print(checkpoints_directory() + self.name + "/")
         if state != State.RESUME and os.path.exists(checkpoints_directory() + self.name + "/"):
             if os.environ["KONFAI_OVERWRITE"] != "True":
                 accept = input(f"The model {self.name} already exists ! Do you want to overwrite it (yes,no) : ")

{konfai-1.2.3 → konfai-1.2.5}/konfai.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: konfai
-Version: 1.2.3
+Version: 1.2.5
 Summary: Modular and configurable Deep Learning framework with YAML and PyTorch
 Author-email: Valentin Boussot <boussot.v@gmail.com>
 License-Expression: Apache-2.0

{konfai-1.2.3 → konfai-1.2.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "konfai"
-version = "1.2.3"
+version = "1.2.5"
 description = "Modular and configurable Deep Learning framework with YAML and PyTorch"
 readme = "README.md"
 requires-python = ">=3.8"