konfai 1.0.8__tar.gz → 1.0.9__tar.gz

{konfai-1.0.8 → konfai-1.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: konfai
-Version: 1.0.8
+Version: 1.0.9
 Summary: Modular and configurable Deep Learning framework with YAML and PyTorch
 Author-email: Valentin Boussot <boussot.v@gmail.com>
 License-Expression: Apache-2.0

konfai-1.0.9/konfai/__init__.py

@@ -0,0 +1,16 @@
+import os
+import datetime
+
+MODELS_DIRECTORY = lambda : os.environ["KONFAI_MODELS_DIRECTORY"]
+CHECKPOINTS_DIRECTORY =lambda : os.environ["KONFAI_CHECKPOINTS_DIRECTORY"]
+MODEL = lambda : os.environ["KONFAI_MODEL"]
+PREDICTIONS_DIRECTORY =lambda : os.environ["KONFAI_PREDICTIONS_DIRECTORY"]
+EVALUATIONS_DIRECTORY =lambda : os.environ["KONFAI_EVALUATIONS_DIRECTORY"]
+STATISTICS_DIRECTORY = lambda : os.environ["KONFAI_STATISTICS_DIRECTORY"]
+SETUPS_DIRECTORY = lambda : os.environ["KONFAI_SETUPS_DIRECTORY"]
+CONFIG_FILE = lambda : os.environ["KONFAI_CONFIG_FILE"]
+KONFAI_STATE = lambda : os.environ["KONFAI_STATE"]
+KONFAI_ROOT = lambda : os.environ["KONFAI_ROOT"]
+CUDA_VISIBLE_DEVICES = lambda : os.environ["CUDA_VISIBLE_DEVICES"]
+
+DATE = lambda : datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")

{konfai-1.0.8 → konfai-1.0.9}/konfai/data/augmentation.py

@@ -6,7 +6,7 @@ import SimpleITK as sitk
 import torch.nn.functional as F
 from typing import Union
 import os
-from konfai import DEEP_LEARNING_API_ROOT
+from konfai import KONFAI_ROOT
 from konfai.utils.config import config
 from konfai.utils.utils import _getModule
 from konfai.utils.dataset import Attribute, data_to_image
@@ -51,7 +51,7 @@ class Prob():
 class DataAugmentationsList():
 
     @config()
-    def __init__(self, nb : int = 10, dataAugmentations: dict[str, Prob] = {"default:RandomElastixTransform" : Prob(1)}) -> None:
+    def __init__(self, nb : int = 10, dataAugmentations: dict[str, Prob] = {"default:Flip" : Prob(1)}) -> None:
         self.nb = nb
         self.dataAugmentations : list[DataAugmentation] = []
         self.dataAugmentationsLoader = dataAugmentations
@@ -59,7 +59,7 @@ class DataAugmentationsList():
     def load(self, key: str):
         for augmentation, prob in self.dataAugmentationsLoader.items():
             module, name = _getModule(augmentation, "data.augmentation")
-            dataAugmentation: DataAugmentation = getattr(importlib.import_module(module), name)(config = None, DL_args="{}.Dataset.augmentations.{}.dataAugmentations".format(DEEP_LEARNING_API_ROOT(), key))
+            dataAugmentation: DataAugmentation = getattr(importlib.import_module(module), name)(config = None, DL_args="{}.Dataset.augmentations.{}.dataAugmentations".format(KONFAI_ROOT(), key))
             dataAugmentation.load(prob.prob)
             self.dataAugmentations.append(dataAugmentation)
     
@@ -525,7 +525,7 @@ class Elastix(DataAugmentation):
 class Permute(DataAugmentation):
 
     @config("Permute")
-    def __init__(self, prob_permute: Union[list[float], None] = [0.33 ,0.33]) -> None:
+    def __init__(self, prob_permute: Union[list[float], None] = [0.5 ,0.5]) -> None:
         super().__init__()
         self._permute_dims = torch.tensor([[0, 2, 1, 3], [0, 3, 1, 2]])
         self.prob_permute = prob_permute

konfai-1.0.8/konfai/data/dataset.py → konfai-1.0.9/konfai/data/data_manager.py

@@ -12,8 +12,8 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 import threading
 from torch.cuda import device_count
 
-from konfai import DL_API_STATE, DEEP_LEARNING_API_ROOT
-from konfai.data.HDF5 import DatasetPatch, DatasetManager
+from konfai import KONFAI_STATE, KONFAI_ROOT
+from konfai.data.patching import DatasetPatch, DatasetManager
 from konfai.utils.config import config
 from konfai.utils.utils import memoryInfo, cpuInfo, memoryForecast, getMemory, State
 from konfai.utils.dataset import Dataset, Attribute
@@ -36,7 +36,7 @@ class GroupTransform:
         if self._pre_transforms is not None:
             if isinstance(self._pre_transforms, dict):
                 for classpath, transform in self._pre_transforms.items():
-                    transform = transform.getTransform(classpath, DL_args =  "{}.Dataset.groups_src.{}.groups_dest.{}.pre_transforms".format(DEEP_LEARNING_API_ROOT(), group_src, group_dest))
+                    transform = transform.getTransform(classpath, DL_args =  "{}.Dataset.groups_src.{}.groups_dest.{}.pre_transforms".format(KONFAI_ROOT(), group_src, group_dest))
                     transform.setDatasets(datasets)
                     self.pre_transforms.append(transform)
             else:
@@ -47,7 +47,7 @@ class GroupTransform:
         if self._post_transforms is not None:
             if isinstance(self._post_transforms, dict):
                 for classpath, transform in self._post_transforms.items():
-                    transform = transform.getTransform(classpath, DL_args = "{}.Dataset.groups_src.{}.groups_dest.{}.post_transforms".format(DEEP_LEARNING_API_ROOT(), group_src, group_dest))
+                    transform = transform.getTransform(classpath, DL_args = "{}.Dataset.groups_src.{}.groups_dest.{}.post_transforms".format(KONFAI_ROOT(), group_src, group_dest))
                     transform.setDatasets(datasets)
                     self.post_transforms.append(transform)
             else:
@@ -64,7 +64,7 @@ class GroupTransform:
 class Group(dict[str, GroupTransform]):
 
     @config()
-    def __init__(self, groups_dest: dict[str, GroupTransform] = {"default": GroupTransform()}):
+    def __init__(self, groups_dest: dict[str, GroupTransform] = {"default:group_dest": GroupTransform()}):
         super().__init__(groups_dest)
 
 class CustomSampler(Sampler[int]):
@@ -127,7 +127,7 @@ class DatasetIter(data.Dataset):
                     total=len(indexs),
                     desc="Caching : init | {} | {}".format(memoryForecast(memory_init, 0, self.nb_dataset), cpuInfo()),
                     leave=False,
-                    disable=self.rank != 0 and "DL_API_CLUSTER" not in os.environ
+                    disable=self.rank != 0 and "KONFAI_CLUSTER" not in os.environ
                 )
 
                 def process(index):
@@ -135,7 +135,7 @@ class DatasetIter(data.Dataset):
                     with memory_lock:
                         pbar.set_description("Caching : {} | {} | {}".format(memoryInfo(), memoryForecast(memory_init, index, self.nb_dataset), cpuInfo()))
                         pbar.update(1)
-                with ThreadPoolExecutor(max_workers=os.cpu_count()//device_count()) as executor:
+                with ThreadPoolExecutor(max_workers=os.cpu_count()//(device_count() if device_count() > 0 else 1)) as executor:
                     futures = [executor.submit(process, index) for index in indexs]
                     for _ in as_completed(futures):
                         pass
@@ -295,7 +295,7 @@ class Data(ABC):
             return [[] for _ in range(world_size)]
         
         maps = []
-        if DL_API_STATE() == str(State.PREDICTION) or DL_API_STATE() == str(State.EVALUATION):
+        if KONFAI_STATE() == str(State.PREDICTION) or KONFAI_STATE() == str(State.EVALUATION):
             np_map = np.asarray(map)
             unique_index = np.unique(np_map[:, 0])
             offset = int(np.ceil(len(unique_index)/world_size))
@@ -314,7 +314,11 @@ class Data(ABC):
     def getData(self, world_size: int) -> list[list[DataLoader]]:
         datasets: dict[str, list[(str, bool)]] = {}
         for dataset_filename in self.dataset_filenames:
-            if len(dataset_filename.split(":")) == 2:
+            if len(dataset_filename.split(":")) == 1:
+                filename = dataset_filename
+                format = "mha"
+                append = True
+            elif len(dataset_filename.split(":")) == 2:
                 filename, format = dataset_filename.split(":")
                 append = True
             else:
@@ -331,8 +335,9 @@ class Data(ABC):
                     else:
                         datasets[group] = [(filename, append)]
         for group_src in self.groups_src:
-            assert group_src in datasets, "Error group source {} not found".format(group_src)
-
+            if group_src not in datasets:
+                raise ValueError("[DatasetManager] Error: group source {} not found. Available groups: {}".format(group_src, list(datasets.keys())))
+                
             for group_dest in self.groups_src[group_src]:
                 self.groups_src[group_src][group_dest].load(group_src, group_dest, [self.datasets[filename] for filename, _ in datasets[group_src]])
         for key, dataAugmentations in self.dataAugmentationsList.items():
@@ -422,8 +427,8 @@ class Data(ABC):
 class DataTrain(Data):
 
     @config("Dataset")
-    def __init__(self,  dataset_filenames : list[str] = ["default:Dataset.h5"], 
-                        groups_src : dict[str, Group] = {"default" : Group()},
+    def __init__(self,  dataset_filenames : list[str] = ["default:./Dataset"], 
+                        groups_src : dict[str, Group] = {"default:group_src" : Group()},
                         augmentations : Union[dict[str, DataAugmentationsList], None] = {"DataAugmentation_0" : DataAugmentationsList()},
                         inlineAugmentations: bool = False,
                         patch : Union[DatasetPatch, None] = DatasetPatch(),
@@ -437,7 +442,7 @@ class DataTrain(Data):
 class DataPrediction(Data):
 
     @config("Dataset")
-    def __init__(self,  dataset_filenames : list[str] = ["default:Dataset.h5"], 
+    def __init__(self,  dataset_filenames : list[str] = ["default:./Dataset"], 
                         groups_src : dict[str, Group] = {"default" : Group()},
                         augmentations : Union[dict[str, DataAugmentationsList], None] = {"DataAugmentation_0" : DataAugmentationsList()},
                         inlineAugmentations: bool = False,
@@ -452,19 +457,10 @@ class DataPrediction(Data):
 class DataMetric(Data):
 
     @config("Dataset")
-    def __init__(self,  dataset_filenames : list[str] = ["default:Dataset.h5"], 
+    def __init__(self,  dataset_filenames : list[str] = ["default:./Dataset"], 
                         groups_src : dict[str, Group] = {"default" : Group()},
                         subset : Union[PredictionSubset, dict[str, PredictionSubset]] = PredictionSubset(),
                         validation: Union[str, None] = None,
                         num_workers : int = 4) -> None:
 
-        super().__init__(dataset_filenames=dataset_filenames, groups_src=groups_src, patch=None, use_cache=False, subset=subset, num_workers=num_workers, batch_size=1, train_size=1 if validation is None else validation)
-
-class DataHyperparameter(Data):
-
-    @config("Dataset")
-    def __init__(self,  dataset_filenames : list[str] = ["default:Dataset.h5"], 
-                        groups_src : dict[str, Group] = {"default" : Group()},
-                        patch : Union[DatasetPatch, None] = DatasetPatch()) -> None:
-
-        super().__init__(dataset_filenames, groups_src, patch, False, PredictionSubset(), 0, False, 1)
+        super().__init__(dataset_filenames=dataset_filenames, groups_src=groups_src, patch=None, use_cache=False, subset=subset, num_workers=num_workers, batch_size=1, train_size=1 if validation is None else validation)

konfai-1.0.8/konfai/data/HDF5.py → konfai-1.0.9/konfai/data/patching.py

@@ -37,13 +37,13 @@ class PathCombine(ABC):
     3D : 
         AAA+AAB+ABA+ABB+BAA+BAB+BBA+BBB
         
-        AAC+ABC+BAC+BBC
-        ACA+ACB+BCA+BCB
         CAA+CAB+CBA+CBB
+        ACA+ACB+BCA+BCB
+        AAC+ABC+BAC+BBC
 
-        ACC+BCC
-        CAC+CBC
         CCA+CCB
+        CAC+CBC
+        ACC+BCC
     
     """
     def setPatchConfig(self, patch_size: list[int], overlap: int):
@@ -214,13 +214,13 @@ class Patch(ABC):
 class DatasetPatch(Patch):
 
     @config("Patch")
-    def __init__(self, patch_size : list[int] = [128, 256, 256], overlap : Union[int, None] = None, mask: Union[str, None] = None, padValue: float = 0, extend_slice: int = 0) -> None:
+    def __init__(self, patch_size : list[int] = [128, 128, 128], overlap : Union[int, None] = None, mask: Union[str, None] = None, padValue: float = 0, extend_slice: int = 0) -> None:
         super().__init__(patch_size, overlap, mask, padValue, extend_slice)
 
 class ModelPatch(Patch):
 
     @config("Patch")
-    def __init__(self, patch_size : list[int] = [128, 256, 256], overlap : Union[int, None] = None, patchCombine: Union[str, None] = None, mask: Union[str, None] = None, padValue: float = 0, extend_slice: int = 0) -> None:
+    def __init__(self, patch_size : list[int] = [128, 128, 128], overlap : Union[int, None] = None, patchCombine: Union[str, None] = None, mask: Union[str, None] = None, padValue: float = 0, extend_slice: int = 0) -> None:
         super().__init__(patch_size, overlap, mask, padValue, extend_slice)
         self.patchCombine = patchCombine
 

{konfai-1.0.8 → konfai-1.0.9}/konfai/data/transform.py

@@ -130,7 +130,7 @@ class Standardize(Transform):
         
 class TensorCast(Transform):
 
-    def __init__(self, dtype : str = "default:float32,int64,int16") -> None:
+    def __init__(self, dtype : str = "float32") -> None:
         self.dtype : torch.dtype = getattr(torch, dtype)
 
     def __call__(self, name: str, input : torch.Tensor, cache_attribute: Attribute) -> torch.Tensor:
@@ -142,7 +142,7 @@ class TensorCast(Transform):
 
 class Padding(Transform):
 
-    def __init__(self, padding : list[int] = [0,0,0,0,0,0], mode : str = "default:constant,reflect,replicate,circular") -> None:
+    def __init__(self, padding : list[int] = [0,0,0,0,0,0], mode : str = "constant") -> None:
         self.padding = padding
         self.mode = mode
 
@@ -332,7 +332,7 @@ class ResampleTransform(Transform):
 
 class Mask(Transform):
 
-    def __init__(self, path : str = "default:./default.mha", value_outside: int = 0) -> None:
+    def __init__(self, path : str = "./default.mha", value_outside: int = 0) -> None:
         self.path = path
         self.value_outside = value_outside
         

{konfai-1.0.8 → konfai-1.0.9}/konfai/evaluator.py

@@ -7,10 +7,10 @@ import json
 import shutil
 import builtins
 import importlib
-from konfai import EVALUATIONS_DIRECTORY, PREDICTIONS_DIRECTORY, DEEP_LEARNING_API_ROOT, CONFIG_FILE
+from konfai import EVALUATIONS_DIRECTORY, PREDICTIONS_DIRECTORY, KONFAI_ROOT, CONFIG_FILE
 from konfai.utils.config import config
 from konfai.utils.utils import _getModule, DistributedObject, synchronize_data
-from konfai.data.dataset import DataMetric
+from konfai.data.data_manager import DataMetric
 
 class CriterionsAttr():
 
@@ -28,7 +28,7 @@ class CriterionsLoader():
         criterions = {}
         for module_classpath, criterionsAttr in self.criterionsLoader.items():
             module, name = _getModule(module_classpath, "metric.measure")
-            criterions[config("{}.metrics.{}.targetsCriterions.{}.criterionsLoader.{}".format(DEEP_LEARNING_API_ROOT(), output_group, target_group, module_classpath))(getattr(importlib.import_module(module), name))(config = None)] = criterionsAttr
+            criterions[config("{}.metrics.{}.targetsCriterions.{}.criterionsLoader.{}".format(KONFAI_ROOT(), output_group, target_group, module_classpath))(getattr(importlib.import_module(module), name))(config = None)] = criterionsAttr
         return criterions
 
 class TargetCriterionsLoader():
@@ -86,8 +86,8 @@ class Statistics():
 class Evaluator(DistributedObject):
 
     @config("Evaluator")
-    def __init__(self, train_name: str = "default:name", metrics: dict[str, TargetCriterionsLoader] = {"default": TargetCriterionsLoader()}, dataset : DataMetric = DataMetric(),) -> None:
-        if os.environ["DEEP_LEANING_API_CONFIG_MODE"] != "Done":
+    def __init__(self, train_name: str = "default:TRAIN_01", metrics: dict[str, TargetCriterionsLoader] = {"default": TargetCriterionsLoader()}, dataset : DataMetric = DataMetric(),) -> None:
+        if os.environ["KONFAI_CONFIG_MODE"] != "Done":
             exit(0)
         super().__init__(train_name)
         self.metric_path = EVALUATIONS_DIRECTORY()+self.name+"/"
@@ -111,7 +111,7 @@ class Evaluator(DistributedObject):
     
     def setup(self, world_size: int):
         if os.path.exists(self.metric_path):
-            if os.environ["DL_API_OVERWRITE"] != "True":
+            if os.environ["KONFAI_OVERWRITE"] != "True":
                 accept = builtins.input("The metric {} already exists ! Do you want to overwrite it (yes,no) : ".format(self.name))
                 if accept != "yes":
                     return

konfai-1.0.9/konfai/main.py

@@ -0,0 +1,52 @@
+import argparse
+import os
+from torch.cuda import device_count
+import torch.multiprocessing as mp
+from konfai.utils.utils import setup, TensorBoard, Log
+
+import sys
+sys.path.insert(0, os.getcwd())
+
+def main():
+    parser = argparse.ArgumentParser(description="KonfAI", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    try:
+        with setup(parser) as distributedObject:
+            with Log(distributedObject.name):
+                world_size = device_count()
+                if world_size == 0:
+                    world_size = 1
+                distributedObject.setup(world_size)
+                with TensorBoard(distributedObject.name):
+                    mp.spawn(distributedObject, nprocs=world_size)
+    except Exception as e:
+        print(e)
+        exit(1)
+
+
+def cluster():
+    parser = argparse.ArgumentParser(description="KonfAI", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+    # Cluster manager arguments
+    cluster_args = parser.add_argument_group('Cluster manager arguments')
+    cluster_args.add_argument('--name', type=str, help='Task name', required=True)
+    cluster_args.add_argument('--num-nodes', '--num_nodes', default=1, type=int, help='Number of nodes')
+    cluster_args.add_argument('--memory', type=int, default=16, help='Amount of memory per node')
+    cluster_args.add_argument('--time-limit', '--time_limit', type=int, default=1440, help='Job time limit in minute')
+    cluster_args.add_argument('--resubmit', action='store_true', help='Automatically resubmit job just before timout')
+    try:
+        with setup(parser) as distributedObject:
+            args = parser.parse_args()
+            config = vars(args)
+            os.environ["KONFAI_OVERWRITE"] = "True"
+            os.environ["KONFAI_CLUSTER"] = "True"
+
+            n_gpu = len(config["gpu"].split(","))
+            distributedObject.setup(n_gpu*int(config["num_nodes"]))
+            import submitit
+            executor = submitit.AutoExecutor(folder="./Cluster/")
+            executor.update_parameters(name=config["name"], mem_gb=config["memory"], gpus_per_node=n_gpu, tasks_per_node=n_gpu//distributedObject.size, cpus_per_task=config["num_workers"], nodes=config["num_nodes"], timeout_min=config["time_limit"])
+            with TensorBoard(distributedObject.name):
+                executor.submit(distributedObject)
+    except Exception as e:
+        print(e)
+        exit(1)

{konfai-1.0.8 → konfai-1.0.9}/konfai/metric/measure.py

@@ -17,7 +17,7 @@ from abc import abstractmethod
 
 from konfai.utils.config import config
 from konfai.utils.utils import _getModule
-from konfai.data.HDF5 import ModelPatch
+from konfai.data.patching import ModelPatch
 from konfai.network.blocks import LatentDistribution
 from konfai.network.network import ModelLoader, Network
 
@@ -92,7 +92,7 @@ class PSNR(MaskedLoss):
         return psnr 
 
     def __init__(self, dynamic_range: Union[float, None] = None) -> None:
-        dynamic_range = dynamic_range if dynamic_range else 1024+3000
+        dynamic_range = dynamic_range if dynamic_range else 1024+3071
         super().__init__(partial(PSNR._loss, dynamic_range), False)
     
 class SSIM(MaskedLoss):
@@ -143,8 +143,11 @@ class Dice(Criterion):
         target = self.flatten(target)
         return (2.*(input * target).sum() + self.smooth)/(input.sum() + target.sum() + self.smooth)
 
+
     def forward(self, output: torch.Tensor, *targets : list[torch.Tensor]) -> torch.Tensor:
         target = targets[0]
+        if output.shape[1] == 1:
+            output = F.one_hot(output.type(torch.int64), num_classes=torch.max(output).item()+1).permute(0, len(target.shape), *[i+1 for i in range(len(target.shape)-1)]).float()
         target = F.one_hot(target.type(torch.int64), num_classes=output.shape[1]).permute(0, len(target.shape), *[i+1 for i in range(len(target.shape)-1)]).float().squeeze(2)
         return 1-torch.mean(self.dice_per_channel(output, target))
 
@@ -239,7 +242,7 @@ class PerceptualLoss(Criterion):
         @config(None)
         def __init__(self, losses: dict[str, float] = {"Gram": 1, "torch_nn_L1Loss": 1}) -> None:
             self.losses = losses
-            self.DL_args = os.environ['DEEP_LEARNING_API_CONFIG_PATH'] if "DEEP_LEARNING_API_CONFIG_PATH" in os.environ else ""
+            self.DL_args = os.environ['KONFAI_CONFIG_PATH'] if "KONFAI_CONFIG_PATH" in os.environ else ""
 
         def getLoss(self) -> dict[torch.nn.Module, float]:
             result: dict[torch.nn.Module, float] = {}
@@ -252,7 +255,7 @@ class PerceptualLoss(Criterion):
         super().__init__()
         self.path_model = path_model
         if self.path_model not in modelsRegister:
-            self.model = modelLoader.getModel(train=False, DL_args=os.environ['DEEP_LEARNING_API_CONFIG_PATH'].split("PerceptualLoss")[0]+"PerceptualLoss.Model", DL_without=["optimizer", "schedulers", "nb_batch_per_step", "init_type", "init_gain", "outputsCriterions", "drop_p"])
+            self.model = modelLoader.getModel(train=False, DL_args=os.environ['KONFAI_CONFIG_PATH'].split("PerceptualLoss")[0]+"PerceptualLoss.Model", DL_without=["optimizer", "schedulers", "nb_batch_per_step", "init_type", "init_gain", "outputsCriterions", "drop_p"])
             if path_model.startswith("https"):
                 state_dict = torch.hub.load_state_dict_from_url(path_model)
                 state_dict = {"Model": {self.model.getName() : state_dict["model"]}}

{konfai-1.0.8 → konfai-1.0.9}/konfai/models/classification/convNeXt.py

@@ -2,7 +2,7 @@ import torch
 import torch.nn.functional as F
 from konfai.network import network, blocks
 from konfai.utils.config import config
-from konfai.data.HDF5 import ModelPatch
+from konfai.data.patching import ModelPatch
 
 """
 "convnext_tiny_1k": "https://dl.fbaipublicfiles.com/convnext/convnext_tiny_1k_224_ema.pth", depths=[3, 3, 9, 3], dims=[96, 192, 384, 768]

{konfai-1.0.8 → konfai-1.0.9}/konfai/models/classification/resnet.py

@@ -3,7 +3,7 @@ from typing import Type
 import torch
 from konfai.network import network, blocks
 from konfai.utils.config import config
-from konfai.data.HDF5 import ModelPatch
+from konfai.data.patching import ModelPatch
 
 """
 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', dim = 2, in_channels = 3, depths=[2, 2, 2, 2], widths = [64, 64, 128, 256, 512],  num_classes=1000, useBottleneck=False

{konfai-1.0.8 → konfai-1.0.9}/konfai/models/generation/cStyleGan.py

@@ -3,7 +3,7 @@ import torch
 
 from konfai.network import network, blocks
 from konfai.utils.config import config
-from konfai.data.HDF5 import ModelPatch    
+from konfai.data.patching import ModelPatch    
 
 class MappingNetwork(network.ModuleArgsDict):
     def __init__(self, z_dim: int, c_dim: int, w_dim: int, num_layers: int, embed_features: int, layer_features: int):

{konfai-1.0.8 → konfai-1.0.9}/konfai/models/generation/ddpm.py

@@ -8,7 +8,7 @@ import numpy as np
 
 from konfai.network import network, blocks
 from konfai.utils.config import config
-from konfai.data.HDF5 import ModelPatch
+from konfai.data.patching import ModelPatch
 from konfai.utils.utils import gpuInfo
 from konfai.metric.measure import Criterion
 

{konfai-1.0.8 → konfai-1.0.9}/konfai/models/generation/diffusionGan.py

@@ -5,7 +5,7 @@ import numpy as np
 
 from konfai.network import network, blocks
 from konfai.utils.config import config
-from konfai.data.HDF5 import ModelPatch, Attribute
+from konfai.data.patching import ModelPatch, Attribute
 from konfai.data import augmentation
 from konfai.models.segmentation import UNet, NestedUNet
 from konfai.models.generation.ddpm import DDPM

{konfai-1.0.8 → konfai-1.0.9}/konfai/models/generation/gan.py

@@ -3,7 +3,7 @@ import torch
 
 from konfai.network import network, blocks
 from konfai.utils.config import config
-from konfai.data.HDF5 import ModelPatch
+from konfai.data.patching import ModelPatch
 
 class Discriminator(network.Network):
 

{konfai-1.0.8 → konfai-1.0.9}/konfai/models/segmentation/NestedUNet.py

@@ -3,7 +3,7 @@ import torch
 
 from konfai.network import network, blocks
 from konfai.utils.config import config
-from konfai.data.HDF5 import ModelPatch
+from konfai.data.patching import ModelPatch
 
      
 class NestedUNetBlock(network.ModuleArgsDict):

{konfai-1.0.8 → konfai-1.0.9}/konfai/models/segmentation/UNet.py

@@ -3,7 +3,7 @@ from typing import Union
 
 from konfai.network import network, blocks
 from konfai.utils.config import config
-from konfai.data.HDF5 import ModelPatch
+from konfai.data.patching import ModelPatch
 
 class UNetHead(network.ModuleArgsDict):
 

{konfai-1.0.8 → konfai-1.0.9}/konfai/network/network.py

@@ -13,11 +13,11 @@ from torch.utils.checkpoint import checkpoint
 from typing import Union
 from enum import Enum
 
-from konfai import DEEP_LEARNING_API_ROOT
+from konfai import KONFAI_ROOT
 from konfai.metric.schedulers import Scheduler
 from konfai.utils.config import config
 from konfai.utils.utils import State, _getModule, getDevice, getGPUMemory
-from konfai.data.HDF5 import Accumulator, ModelPatch
+from konfai.data.patching import Accumulator, ModelPatch
 
 class NetState(Enum):
     TRAIN = 0,
@@ -40,7 +40,7 @@ class OptimizerLoader():
     
     def getOptimizer(self, key: str, parameter: Iterator[torch.nn.parameter.Parameter]) -> torch.optim.Optimizer:
         torch.optim.AdamW
-        return config("{}.Model.{}.Optimizer".format(DEEP_LEARNING_API_ROOT(), key))(getattr(importlib.import_module('torch.optim'), self.name))(parameter, config = None)
+        return config("{}.Model.{}.Optimizer".format(KONFAI_ROOT(), key))(getattr(importlib.import_module('torch.optim'), self.name))(parameter, config = None)
         
 class SchedulerStep():
     
@@ -98,8 +98,8 @@ class CriterionsLoader():
         for module_classpath, criterionsAttr in self.criterionsLoader.items():
             module, name = _getModule(module_classpath, "metric.measure")
             criterionsAttr.isTorchCriterion = module.startswith("torch")
-            criterionsAttr.sheduler = criterionsAttr.l.getShedulers("{}.Model.{}.outputsCriterions.{}.targetsCriterions.{}.criterionsLoader.{}".format(DEEP_LEARNING_API_ROOT(), model_classname, output_group, target_group, module_classpath))
-            criterions[config("{}.Model.{}.outputsCriterions.{}.targetsCriterions.{}.criterionsLoader.{}".format(DEEP_LEARNING_API_ROOT(), model_classname, output_group, target_group, module_classpath))(getattr(importlib.import_module(module), name))(config = None)] = criterionsAttr
+            criterionsAttr.sheduler = criterionsAttr.l.getShedulers("{}.Model.{}.outputsCriterions.{}.targetsCriterions.{}.criterionsLoader.{}".format(KONFAI_ROOT(), model_classname, output_group, target_group, module_classpath))
+            criterions[config("{}.Model.{}.outputsCriterions.{}.targetsCriterions.{}.criterionsLoader.{}".format(KONFAI_ROOT(), model_classname, output_group, target_group, module_classpath))(getattr(importlib.import_module(module), name))(config = None)] = criterionsAttr
         return criterions
 
 class TargetCriterionsLoader():
@@ -753,14 +753,14 @@ class Network(ModuleArgsDict, ABC):
         output_layer_accumulator : dict[str, Accumulator] = {}
         output_layer_patch_indexed : dict[str, Patch_Indexed] = {}
         it = 0
-        debug = "DL_API_DEBUG" in os.environ
+        debug = "KONFAI_DEBUG" in os.environ
         for (nameTmp, output_layer) in self.named_forward(*inputs):
             name = nameTmp.replace(";accu;", "")
             if debug:
-                if "DL_API_DEBUG_LAST_LAYER" in os.environ:
-                    os.environ["DL_API_DEBUG_LAST_LAYER"] = "{}|{}:{}:{}".format(os.environ["DL_API_DEBUG_LAST_LAYER"], name, getGPUMemory(output_layer.device), str(output_layer.device).replace("cuda:", ""))
+                if "KONFAI_DEBUG_LAST_LAYER" in os.environ:
+                    os.environ["KONFAI_DEBUG_LAST_LAYER"] = "{}|{}:{}:{}".format(os.environ["KONFAI_DEBUG_LAST_LAYER"], name, getGPUMemory(output_layer.device), str(output_layer.device).replace("cuda:", ""))
                 else:
-                    os.environ["DL_API_DEBUG_LAST_LAYER"] = "{}:{}:{}".format(name, getGPUMemory(output_layer.device), str(output_layer.device).replace("cuda:", ""))
+                    os.environ["KONFAI_DEBUG_LAST_LAYER"] = "{}:{}:{}".format(name, getGPUMemory(output_layer.device), str(output_layer.device).replace("cuda:", ""))
             it += 1
             if name in layers_name or nameTmp in layers_name:
                 if ";accu;" in nameTmp:
@@ -918,12 +918,12 @@ class Network(ModuleArgsDict, ABC):
 class ModelLoader():
 
     @config("Model")
-    def __init__(self, classpath : str = "default:segmentation.UNet") -> None:
-        self.module, self.name = _getModule(classpath.split(".")[-1] if len(classpath.split(".")) > 1 else classpath, ".".join(classpath.split(".")[:-1]) if len(classpath.split(".")) > 1 else "")
+    def __init__(self, classpath : str = "default:segmentation.UNet.UNet") -> None:
+        self.module, self.name = _getModule(classpath, "models")
         
     def getModel(self, train : bool = True, DL_args: Union[str, None] = None, DL_without=["optimizer", "schedulers", "nb_batch_per_step", "init_type", "init_gain"]) -> Network:
         if not DL_args:
-            DL_args="{}.Model".format(DEEP_LEARNING_API_ROOT())
+            DL_args="{}.Model".format(KONFAI_ROOT())
         model = partial(getattr(importlib.import_module(self.module), self.name), config = None, DL_args=DL_args)
         if not train: 
             model = partial(model, DL_without = DL_without)

{konfai-1.0.8 → konfai-1.0.9}/konfai/predictor.py

@@ -6,12 +6,12 @@ import torch
 import tqdm
 import os
 
-from konfai import MODELS_DIRECTORY, PREDICTIONS_DIRECTORY, CONFIG_FILE, MODEL, DEEP_LEARNING_API_ROOT
+from konfai import MODELS_DIRECTORY, PREDICTIONS_DIRECTORY, CONFIG_FILE, MODEL, KONFAI_ROOT
 from konfai.utils.config import config
 from konfai.utils.utils import State, get_patch_slices_from_nb_patch_per_dim, NeedDevice, _getModule, DistributedObject, DataLog, description
 from konfai.utils.dataset import Dataset, Attribute
-from konfai.data.dataset import DataPrediction, DatasetIter
-from konfai.data.HDF5 import Accumulator, PathCombine
+from konfai.data.data_manager import DataPrediction, DatasetIter
+from konfai.data.patching import Accumulator, PathCombine
 from konfai.network.network import ModelLoader, Network, NetState, CPU_Model
 from konfai.data.transform import Transform, TransformLoader
 
@@ -52,13 +52,13 @@ class OutDataset(Dataset, NeedDevice, ABC):
             
             if _transform_type is not None:
                 for classpath, transform in _transform_type.items():
-                    transform = transform.getTransform(classpath, DL_args =  "{}.outsDataset.{}.OutDataset.{}".format(DEEP_LEARNING_API_ROOT(), name_layer, name))
+                    transform = transform.getTransform(classpath, DL_args =  "{}.outsDataset.{}.OutDataset.{}".format(KONFAI_ROOT(), name_layer, name))
                     transform.setDatasets(datasets)
                     transform_type.append(transform)
 
         if self._patchCombine is not None:
-            module, name = _getModule(self._patchCombine, "data.HDF5")
-            self.patchCombine = getattr(importlib.import_module(module), name)(config = None, DL_args =  "{}.outsDataset.{}.OutDataset".format(DEEP_LEARNING_API_ROOT(), name_layer))
+            module, name = _getModule(self._patchCombine, "data.patching")
+            self.patchCombine = getattr(importlib.import_module(module), name)(config = None, DL_args =  "{}.outsDataset.{}.OutDataset".format(KONFAI_ROOT(), name_layer))
     
     def setPatchConfig(self, patchSize: Union[list[int], None], overlap: Union[int, None], nb_data_augmentation: int) -> None:
         if patchSize is not None and overlap is not None:
@@ -94,7 +94,7 @@ class OutDataset(Dataset, NeedDevice, ABC):
 class OutSameAsGroupDataset(OutDataset):
 
     @config("OutDataset")
-    def __init__(self, dataset_filename: str = "Dataset:h5", group: str = "default", sameAsGroup: str = "default", pre_transforms : dict[str, TransformLoader] = {"default:Normalize": TransformLoader()}, post_transforms : dict[str, TransformLoader] = {"default:Normalize": TransformLoader()}, final_transforms : dict[str, TransformLoader] = {"default:Normalize": TransformLoader()}, patchCombine: Union[str, None] = None, redution: str = "mean", inverse_transform: bool = True) -> None:
+    def __init__(self, dataset_filename: str = "./Dataset:mha", group: str = "default", sameAsGroup: str = "default", pre_transforms : dict[str, TransformLoader] = {"default:Normalize": TransformLoader()}, post_transforms : dict[str, TransformLoader] = {"default:Normalize": TransformLoader()}, final_transforms : dict[str, TransformLoader] = {"default:Normalize": TransformLoader()}, patchCombine: Union[str, None] = None, redution: str = "mean", inverse_transform: bool = True) -> None:
         super().__init__(dataset_filename, group, pre_transforms, post_transforms, final_transforms, patchCombine)
         self.group_src, self.group_dest = sameAsGroup.split(":")
         self.redution = redution
@@ -240,7 +240,7 @@ class _Predictor():
         self.modelComposite.module.setState(NetState.PREDICTION)
         desc = lambda : "Prediction : {}".format(description(self.modelComposite))
         self.dataloader_prediction.dataset.load()
-        with tqdm.tqdm(iterable = enumerate(self.dataloader_prediction), leave=False, desc = desc(), total=len(self.dataloader_prediction), disable=self.global_rank != 0 and "DL_API_CLUSTER" not in os.environ) as batch_iter:
+        with tqdm.tqdm(iterable = enumerate(self.dataloader_prediction), leave=False, desc = desc(), total=len(self.dataloader_prediction), disable=self.global_rank != 0 and "KONFAI_CLUSTER" not in os.environ) as batch_iter:
             dist.barrier()
             for it, data_dict in batch_iter:
                 input = self.getInput(data_dict)
@@ -322,7 +322,7 @@ class Predictor(DistributedObject):
                     gpu_checkpoints: Union[list[str], None] = None,
                     outsDataset: Union[dict[str, OutDatasetLoader], None] = {"default:Default" : OutDatasetLoader()},
                     images_log: list[str] = []) -> None:
-        if os.environ["DEEP_LEANING_API_CONFIG_MODE"] != "Done":
+        if os.environ["KONFAI_CONFIG_MODE"] != "Done":
             exit(0)
         super().__init__(train_name)
         self.manual_seed = manual_seed
@@ -374,7 +374,7 @@ class Predictor(DistributedObject):
         for dataset_filename in self.datasets_filename:
             path = self.predict_path +dataset_filename
             if os.path.exists(path):
-                if os.environ["DL_API_OVERWRITE"] != "True":
+                if os.environ["KONFAI_OVERWRITE"] != "True":
                     accept = builtins.input("The prediction {} already exists ! Do you want to overwrite it (yes,no) : ".format(path))
                     if accept != "yes":
                         return

{konfai-1.0.8 → konfai-1.0.9}/konfai/trainer.py

@@ -12,8 +12,8 @@ from torch.utils.tensorboard.writer import SummaryWriter
 from torch.optim.swa_utils import AveragedModel
 import torch.distributed as dist
 
-from konfai import MODELS_DIRECTORY, CHECKPOINTS_DIRECTORY, STATISTICS_DIRECTORY, SETUPS_DIRECTORY, CONFIG_FILE, MODEL, DATE, DL_API_STATE
-from konfai.data.dataset import DataTrain
+from konfai import MODELS_DIRECTORY, CHECKPOINTS_DIRECTORY, STATISTICS_DIRECTORY, SETUPS_DIRECTORY, CONFIG_FILE, MODEL, DATE, KONFAI_STATE
+from konfai.data.data_manager import DataTrain
 from konfai.utils.config import config
 from konfai.utils.utils import State, DataLog, DistributedObject, description
 from konfai.network.network import Network, ModelLoader, NetState, CPU_Model
@@ -72,7 +72,7 @@ class _Trainer():
             self.modelEMA.module.setState(NetState.TRAIN)
 
         desc = lambda : "Training : {}".format(description(self.model, self.modelEMA))
-        with tqdm.tqdm(iterable = enumerate(self.dataloader_training), desc = desc(), total=len(self.dataloader_training), leave=False, disable=self.global_rank != 0 and "DL_API_CLUSTER" not in os.environ) as batch_iter:
+        with tqdm.tqdm(iterable = enumerate(self.dataloader_training), desc = desc(), total=len(self.dataloader_training), leave=False, disable=self.global_rank != 0 and "KONFAI_CLUSTER" not in os.environ) as batch_iter:
             for _, data_dict in batch_iter:
                 with torch.amp.autocast('cuda', enabled=self.autocast):
                     input = self.getInput(data_dict)
@@ -103,7 +103,7 @@ class _Trainer():
         desc = lambda : "Validation : {}".format(description(self.model, self.modelEMA))
         data_dict = None
         self.dataloader_validation.dataset.load()
-        with tqdm.tqdm(iterable = enumerate(self.dataloader_validation), desc = desc(), total=len(self.dataloader_validation), leave=False, disable=self.global_rank != 0 and "DL_API_CLUSTER" not in os.environ) as batch_iter:
+        with tqdm.tqdm(iterable = enumerate(self.dataloader_validation), desc = desc(), total=len(self.dataloader_validation), leave=False, disable=self.global_rank != 0 and "KONFAI_CLUSTER" not in os.environ) as batch_iter:
             for _, data_dict in batch_iter:
                 input = self.getInput(data_dict)
                 self.model(input)
@@ -205,7 +205,7 @@ class Trainer(DistributedObject):
     def __init__(   self,
                     model : ModelLoader = ModelLoader(),
                     dataset : DataTrain = DataTrain(),
-                    train_name : str = "default:name",
+                    train_name : str = "default:TRAIN_01",
                     manual_seed : Union[int, None] = None,
                     epochs: int = 100,
                     it_validation : Union[int, None] = None,
@@ -215,7 +215,7 @@ class Trainer(DistributedObject):
                     ema_decay : float = 0,
                     data_log: Union[list[str], None] = None,
                     save_checkpoint_mode: str= "BEST") -> None:
-        if os.environ["DEEP_LEANING_API_CONFIG_MODE"] != "Done":
+        if os.environ["KONFAI_CONFIG_MODE"] != "Done":
             exit(0)
         super().__init__(train_name)
         self.manual_seed = manual_seed        
@@ -292,9 +292,9 @@ class Trainer(DistributedObject):
         return (1-self.ema_decay) * averaged_model_parameter + self.ema_decay * model_parameter
     
     def setup(self, world_size: int):
-        state = State._member_map_[DL_API_STATE()]
-        if state != State.RESUME and os.path.exists(STATISTICS_DIRECTORY()+self.name+"/"):
-            if os.environ["DL_API_OVERWRITE"] != "True":
+        state = State._member_map_[KONFAI_STATE()]
+        if state != State.RESUME and os.path.exists(CHECKPOINTS_DIRECTORY()+self.name+"/"):
+            if os.environ["KONFAI_OVERWRITE"] != "True":
                 accept = input("The model {} already exists ! Do you want to overwrite it (yes,no) : ".format(self.name))
                 if accept != "yes":
                     return

{konfai-1.0.8 → konfai-1.0.9}/konfai/utils/config.py

@@ -3,9 +3,8 @@ import ruamel.yaml
 import inspect 
 import collections
 from copy import deepcopy
-from typing import Union
+from typing import Union, Literal, get_origin, get_args
 import torch
-
 from konfai import CONFIG_FILE
 
 yaml = ruamel.yaml.YAML()
@@ -26,7 +25,7 @@ class Config():
         if not os.path.exists(self.filename):
             result = input("Create a new config file ? [no,yes,interactive] : ")
             if result in ["yes", "interactive"]:
-                os.environ["DEEP_LEANING_API_CONFIG_MODE"] = "interactive" if result == "interactive" else "default"
+                os.environ["KONFAI_CONFIG_MODE"] = "interactive" if result == "interactive" else "default"
             else:
                 exit(0)
             with open(self.filename, "w") as f:
@@ -69,7 +68,7 @@ class Config():
 
     def __exit__(self, type, value, traceback) -> None:
         self.yml.close()
-        if os.environ["DEEP_LEANING_API_CONFIG_MODE"] == "remove":
+        if os.environ["KONFAI_CONFIG_MODE"] == "remove":
             if os.path.exists(CONFIG_FILE()):
                 os.remove(CONFIG_FILE())
             return
@@ -87,22 +86,22 @@ class Config():
         except:
             result = input("\nKeep a default configuration file ? (yes,no) : ")
             if result == "yes":
-                os.environ["DEEP_LEANING_API_CONFIG_MODE"] = "default"
+                os.environ["KONFAI_CONFIG_MODE"] = "default"
             else:
-                os.environ["DEEP_LEANING_API_CONFIG_MODE"] = "remove"
+                os.environ["KONFAI_CONFIG_MODE"] = "remove"
                 exit(0)
         return default.split(":")[1] if len(default.split(":")) > 1 else default
 
     @staticmethod
     def _getInputDefault(name : str, default : Union[str, None], isList : bool = False) -> Union[list[Union[str, None]], str, None]:
         if isinstance(default, str) and (default == "default" or (len(default.split(":")) > 1 and default.split(":")[0] == "default")):
-            if os.environ["DEEP_LEANING_API_CONFIG_MODE"] == "interactive":
+            if os.environ["KONFAI_CONFIG_MODE"] == "interactive":
                 if isList:
                     list_tmp = []
                     key_tmp = "OK"
-                    while key_tmp != "!" and os.environ["DEEP_LEANING_API_CONFIG_MODE"] == "interactive":
+                    while (key_tmp != "!" and key_tmp != " ") and os.environ["KONFAI_CONFIG_MODE"] == "interactive":
                         key_tmp = Config._getInput(name, default)
-                        if key_tmp != "!":
+                        if (key_tmp != "!" and key_tmp != " "):
                             if key_tmp == "":
                                 key_tmp = default.split(":")[1] if len(default.split(":")) > 1 else default
                             list_tmp.append(key_tmp)
@@ -134,6 +133,7 @@ class Config():
                 list_tmp = [] 
                 for key in value_config:
                     list_tmp.extend(Config._getInputDefault(name, key, isList=True))
+                
 
                 value = list_tmp
                 value_config = list_tmp
@@ -155,7 +155,7 @@ class Config():
                     dict_value[key] = value_tmp
                 value = dict_value
         if isinstance(self.config, str):
-            os.environ['DEEP_LEARNING_API_CONFIG_VARIABLE'] = "True"
+            os.environ['KONFAI_CONFIG_VARIABLE'] = "True"
             return None
 
         self.config[name] = value_config if value_config is not None else "None"
@@ -169,17 +169,30 @@ def config(key : Union[str, None] = None):
             if "config" in kwargs:
                 filename =  kwargs["config"]
                 if filename == None:
-                    filename = os.environ['DEEP_LEARNING_API_CONFIG_FILE']
+                    filename = os.environ['KONFAI_CONFIG_FILE']
                 else:
-                    os.environ['DEEP_LEARNING_API_CONFIG_FILE'] = filename
+                    os.environ['KONFAI_CONFIG_FILE'] = filename
                 key_tmp =  kwargs["DL_args"]+("."+key if key is not None else "") if "DL_args" in kwargs else key
                 without =  kwargs["DL_without"] if "DL_without" in kwargs else []
-                os.environ['DEEP_LEARNING_API_CONFIG_PATH'] = key_tmp
+                os.environ['KONFAI_CONFIG_PATH'] = key_tmp
                 with Config(filename, key_tmp) as config:
-                    os.environ['DEEP_LEARNING_API_CONFIG_VARIABLE'] = "False"
+                    os.environ['KONFAI_CONFIG_VARIABLE'] = "False"
                     kwargs = {} 
                     for param in list(inspect.signature(function).parameters.values())[len(args):]:
+
                         annotation = param.annotation
+                        # --- support Literal ---
+                        if get_origin(annotation) is Literal:
+                            allowed_values = get_args(annotation)
+                            default_value = param.default if param.default != inspect._empty else allowed_values[0]
+                            value = config.getValue(param.name, f"default:{default_value}")
+                            if value not in allowed_values:
+                                raise ValueError(
+                                    f"[Config] Invalid value '{value}' for parameter '{param.name}'. "
+                                    f"Expected one of: {allowed_values}."
+                                )
+                            kwargs[param.name] = value
+                            continue
                         if str(annotation).startswith("typing.Union") or str(annotation).startswith("typing.Optional"):
                             for i in annotation.__args__:
                                 annotation = i
@@ -188,8 +201,24 @@ def config(key : Union[str, None] = None):
                             continue
                         if not annotation == inspect._empty:
                             if annotation not in [int, str, bool, float, torch.Tensor]:
-                                if str(annotation).startswith("list") or str(annotation).startswith("tuple") or str(annotation).startswith("typing.Tuple"):
-                                    if annotation.__args__[0] in [int, str, bool, float]:
+                                if str(annotation).startswith("list") or str(annotation).startswith("tuple") or str(annotation).startswith("typing.Tuple") or str(annotation).startswith("typing.List"):
+                                    elem_type = annotation.__args__[0]
+                                    values = config.getValue(param.name, param.default)
+                                    if getattr(elem_type, '__origin__', None) is Union:
+                                        valid_types = elem_type.__args__
+                                        result = []
+                                        for v in values:
+                                            for t in valid_types:
+                                                try:
+                                                    if t == torch.Tensor and not isinstance(v, torch.Tensor):
+                                                        v = torch.tensor(v)
+                                                    result.append(t(v) if t != torch.Tensor else v)
+                                                    break
+                                                except Exception:
+                                                    continue
+                                        kwargs[param.name] = result
+
+                                    elif annotation.__args__[0] in [int, str, bool, float]:
                                         values = config.getValue(param.name, param.default)
                                         kwargs[param.name] = values
                                     else:
@@ -204,9 +233,13 @@ def config(key : Union[str, None] = None):
                                     else: 
                                         raise ConfigError()
                                 else:
-                                    kwargs[param.name] = annotation(config = filename, DL_args = key_tmp)
-                                    if os.environ['DEEP_LEARNING_API_CONFIG_VARIABLE'] == "True":
-                                        os.environ['DEEP_LEARNING_API_CONFIG_VARIABLE'] = "False"
+                                    try:
+                                        kwargs[param.name] = annotation(config = filename, DL_args = key_tmp)
+                                    except Exception as e:
+                                        raise ValueError("[Config] Failed to instantiate {} with type {}".format(param.name, annotation.__name__))
+                                    
+                                    if os.environ['KONFAI_CONFIG_VARIABLE'] == "True":
+                                        os.environ['KONFAI_CONFIG_VARIABLE'] = "False"
                                         kwargs[param.name] = None
                             else:
                                 kwargs[param.name] = config.getValue(param.name, param.default)

{konfai-1.0.8 → konfai-1.0.9}/konfai/utils/dataset.py

@@ -750,7 +750,7 @@ class Dataset():
         else:
             with Dataset.File(self.filename, True, self.format) as file:
                 names = file.getNames(groups)
-        return [name for i, name in enumerate(names) if index is None or i in index]
+        return [name for i, name in enumerate(sorted(names)) if index is None or i in index]
     
     def getInfos(self, groups: str, name: str) -> tuple[list[int], Attribute]:
         if self.is_directory:

{konfai-1.0.8 → konfai-1.0.9}/konfai/utils/utils.py

@@ -10,7 +10,7 @@ from abc import ABC, abstractmethod
 from enum import Enum
 from typing import Any, Union
 
-from konfai import CONFIG_FILE, STATISTICS_DIRECTORY, PREDICTIONS_DIRECTORY, DL_API_STATE, CUDA_VISIBLE_DEVICES
+from konfai import CONFIG_FILE, EVALUATIONS_DIRECTORY, STATISTICS_DIRECTORY, PREDICTIONS_DIRECTORY, KONFAI_STATE, CUDA_VISIBLE_DEVICES
 import torch.distributed as dist
 import argparse
 import subprocess
@@ -35,8 +35,8 @@ def _getModule(classpath : str, type : str) -> tuple[str, str]:
         module = ".".join(classpath.split("_")[:-1])
         name = classpath.split("_")[-1] 
     else:
-        module = "konfai."+type
-        name = classpath
+        module = "konfai."+type+("." if len(classpath.split(".")) > 2 else "")+".".join(classpath.split(".")[:-1])
+        name = classpath.split(".")[-1]
     return module, name
 
 def cpuInfo() -> str:
@@ -236,36 +236,51 @@ class DataLog(Enum):
     VIDEO   = lambda tb, name, layer, it : tb.add_video(name, _logVideoFormat(layer), it),
     AUDIO   = lambda tb, name, layer, it : tb.add_audio(name, _logImageFormat(layer), it)
 
-class Log():
-
+class Log:
     def __init__(self, name: str) -> None:
-        path = PREDICTIONS_DIRECTORY() if DL_API_STATE() == "PREDICTION" else STATISTICS_DIRECTORY()
-        if not os.path.exists("{}{}".format(path, name)):
-            os.makedirs("{}{}".format(path, name))
-        self.file = open("{}{}/log.txt".format(path, name), 'w')
+        if KONFAI_STATE() == "PREDICTION":
+            path = PREDICTIONS_DIRECTORY()
+        elif KONFAI_STATE() == "EVALUATION":
+            path = EVALUATIONS_DIRECTORY()
+        else:
+            path = STATISTICS_DIRECTORY()
+        
+        self.verbose = os.environ.get("KONFAI_VERBOSE", "True") == "True"
+        self.log_path = os.path.join(path, name)
+        os.makedirs(self.log_path, exist_ok=True)
+        
+        self.file = open(os.path.join(self.log_path, "log.txt"), "w", buffering=1) 
         self.stdout_bak = sys.stdout
         self.stderr_bak = sys.stderr
-        self.verbose = os.environ["DEEP_LEARNING_VERBOSE"] == "True"
 
     def __enter__(self):
         self.file.__enter__()
         sys.stdout = self
         sys.stderr = self
         return self
-    
-    def __exit__(self, type, value, traceback):
-        self.file.__exit__(type, value, traceback)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.file.__exit__(exc_type, exc_val, exc_tb)
         sys.stdout = self.stdout_bak
         sys.stderr = self.stderr_bak
-         
+
     def write(self, msg):
-        if msg.strip() != "":
-            self.file.write(msg)
-            if self.verbose:
-                print(msg, file=sys.__stdout__)
+        if not msg:
+            return
+        self.file.write(msg)
+        self.file.flush()
+        if self.verbose:
+            sys.__stdout__.write(msg)
+            sys.__stdout__.flush()
 
     def flush(self):
-        pass
+        self.file.flush()
+
+    def isatty(self):
+        return False
+
+    def fileno(self):
+        return sys.__stdout__.fileno()
     
 class TensorBoard():
     
@@ -274,8 +289,8 @@ class TensorBoard():
         self.name = name
 
     def __enter__(self):
-        if "DEEP_LEARNING_TENSORBOARD_PORT" in os.environ:
-            command = ["tensorboard", "--logdir", PREDICTIONS_DIRECTORY() if DL_API_STATE() == "PREDICTION" else STATISTICS_DIRECTORY() + self.name + "/", "--port", os.environ["DEEP_LEARNING_TENSORBOARD_PORT"], "--bind_all"]
+        if "KONFAI_TENSORBOARD_PORT" in os.environ:
+            command = ["tensorboard", "--logdir", PREDICTIONS_DIRECTORY() if KONFAI_STATE() == "PREDICTION" else STATISTICS_DIRECTORY() + self.name + "/", "--port", os.environ["KONFAI_TENSORBOARD_PORT"], "--bind_all"]
             self.process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
             try:
                 s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
@@ -285,7 +300,7 @@ class TensorBoard():
                 IP = '127.0.0.1'
             finally:
                 s.close()
-            print("Tensorboard : http://{}:{}/".format(IP, os.environ["DEEP_LEARNING_TENSORBOARD_PORT"]))
+            print("Tensorboard : http://{}:{}/".format(IP, os.environ["KONFAI_TENSORBOARD_PORT"]))
         return self
     
     def __exit__(self, type, value, traceback):
@@ -361,72 +376,72 @@ class DistributedObject():
                 pynvml.nvmlShutdown()
             cleanup()
 
-def setupAPI(parser: argparse.ArgumentParser) -> DistributedObject:
-    # API arguments
-    api_args = parser.add_argument_group('API arguments')
-    api_args.add_argument("type", type=State, choices=list(State))
-    api_args.add_argument('-y', action='store_true', help="Accept overwrite")
-    api_args.add_argument('-tb', action='store_true', help='Start TensorBoard')
-    api_args.add_argument("-c", "--config", type=str, default="None", help="Configuration file location")
-    api_args.add_argument("-g", "--gpu", type=str, default=os.environ["CUDA_VISIBLE_DEVICES"] if "CUDA_VISIBLE_DEVICES" in os.environ else "", help="List of GPU")
-    api_args.add_argument('--num-workers', '--num_workers', default=4, type=int, help='No. of workers per DataLoader & GPU')
-    api_args.add_argument("-models_dir", "--MODELS_DIRECTORY", type=str, default="./Models/", help="Models location")
-    api_args.add_argument("-checkpoints_dir", "--CHECKPOINTS_DIRECTORY", type=str, default="./Checkpoints/", help="Checkpoints location")
-    api_args.add_argument("-model", "--MODEL", type=str, default="", help="URL Model")
-    api_args.add_argument("-predictions_dir", "--PREDICTIONS_DIRECTORY", type=str, default="./Predictions/", help="Predictions location")
-    api_args.add_argument("-evaluation_dir", "--EVALUATIONS_DIRECTORY", type=str, default="./Evaluations/", help="Evaluations location")
-    api_args.add_argument("-statistics_dir", "--STATISTICS_DIRECTORY", type=str, default="./Statistics/", help="Statistics location")
-    api_args.add_argument("-setups_dir", "--SETUPS_DIRECTORY", type=str, default="./Setups/", help="Setups location")
-    api_args.add_argument('-log', action='store_true', help='Save log')
-    api_args.add_argument('-quiet', action='store_false', help='')
+def setup(parser: argparse.ArgumentParser) -> DistributedObject:
+    # KONFAI arguments
+    KONFAI_args = parser.add_argument_group('KONFAI arguments')
+    KONFAI_args.add_argument("type", type=State, choices=list(State))
+    KONFAI_args.add_argument('-y', action='store_true', help="Accept overwrite")
+    KONFAI_args.add_argument('-tb', action='store_true', help='Start TensorBoard')
+    KONFAI_args.add_argument("-c", "--config", type=str, default="None", help="Configuration file location")
+    KONFAI_args.add_argument("-g", "--gpu", type=str, default=os.environ["CUDA_VISIBLE_DEVICES"] if "CUDA_VISIBLE_DEVICES" in os.environ else "", help="List of GPU")
+    KONFAI_args.add_argument('--num-workers', '--num_workers', default=4, type=int, help='No. of workers per DataLoader & GPU')
+    KONFAI_args.add_argument("-models_dir", "--MODELS_DIRECTORY", type=str, default="./Models/", help="Models location")
+    KONFAI_args.add_argument("-checkpoints_dir", "--CHECKPOINTS_DIRECTORY", type=str, default="./Checkpoints/", help="Checkpoints location")
+    KONFAI_args.add_argument("-model", "--MODEL", type=str, default="", help="URL Model")
+    KONFAI_args.add_argument("-predictions_dir", "--PREDICTIONS_DIRECTORY", type=str, default="./Predictions/", help="Predictions location")
+    KONFAI_args.add_argument("-evaluation_dir", "--EVALUATIONS_DIRECTORY", type=str, default="./Evaluations/", help="Evaluations location")
+    KONFAI_args.add_argument("-statistics_dir", "--STATISTICS_DIRECTORY", type=str, default="./Statistics/", help="Statistics location")
+    KONFAI_args.add_argument("-setups_dir", "--SETUPS_DIRECTORY", type=str, default="./Setups/", help="Setups location")
+    KONFAI_args.add_argument('-log', action='store_true', help='Save log')
+    KONFAI_args.add_argument('-quiet', action='store_false', help='')
     
     
     args = parser.parse_args()
     config = vars(args)
 
     os.environ["CUDA_VISIBLE_DEVICES"] = config["gpu"]
-    os.environ["DL_API_MODELS_DIRECTORY"] = config["MODELS_DIRECTORY"]
-    os.environ["DL_API_CHECKPOINTS_DIRECTORY"] = config["CHECKPOINTS_DIRECTORY"]
-    os.environ["DL_API_PREDICTIONS_DIRECTORY"] = config["PREDICTIONS_DIRECTORY"]
-    os.environ["DL_API_EVALUATIONS_DIRECTORY"] = config["EVALUATIONS_DIRECTORY"]
-    os.environ["DL_API_STATISTICS_DIRECTORY"] = config["STATISTICS_DIRECTORY"]
+    os.environ["KONFAI_MODELS_DIRECTORY"] = config["MODELS_DIRECTORY"]
+    os.environ["KONFAI_CHECKPOINTS_DIRECTORY"] = config["CHECKPOINTS_DIRECTORY"]
+    os.environ["KONFAI_PREDICTIONS_DIRECTORY"] = config["PREDICTIONS_DIRECTORY"]
+    os.environ["KONFAI_EVALUATIONS_DIRECTORY"] = config["EVALUATIONS_DIRECTORY"]
+    os.environ["KONFAI_STATISTICS_DIRECTORY"] = config["STATISTICS_DIRECTORY"]
     
-    os.environ["DL_API_STATE"] = str(config["type"])
+    os.environ["KONFAI_STATE"] = str(config["type"])
     
-    os.environ["DL_API_MODEL"] = config["MODEL"]
+    os.environ["KONFAI_MODEL"] = config["MODEL"]
 
-    os.environ["DL_API_SETUPS_DIRECTORY"] = config["SETUPS_DIRECTORY"]
+    os.environ["KONFAI_SETUPS_DIRECTORY"] = config["SETUPS_DIRECTORY"]
 
-    os.environ["DL_API_OVERWRITE"] = "{}".format(config["y"])
-    os.environ["DEEP_LEANING_API_CONFIG_MODE"] = "Done"
+    os.environ["KONFAI_OVERWRITE"] = "{}".format(config["y"])
+    os.environ["KONFAI_CONFIG_MODE"] = "Done"
     if config["tb"]:
-        os.environ["DEEP_LEARNING_TENSORBOARD_PORT"] = str(find_free_port())
+        os.environ["KONFAI_TENSORBOARD_PORT"] = str(find_free_port())
 
-    os.environ["DEEP_LEARNING_VERBOSE"] = str(config["quiet"])
+    os.environ["KONFAI_VERBOSE"] = str(config["quiet"])
 
     if config["config"] == "None":
         if config["type"] is State.PREDICTION:
-             os.environ["DEEP_LEARNING_API_CONFIG_FILE"] = "Prediction.yml"
+             os.environ["KONFAI_CONFIG_FILE"] = "Prediction.yml"
         elif config["type"] is State.EVALUATION:
-            os.environ["DEEP_LEARNING_API_CONFIG_FILE"] = "Evaluation.yml"
+            os.environ["KONFAI_CONFIG_FILE"] = "Evaluation.yml"
         else:
-            os.environ["DEEP_LEARNING_API_CONFIG_FILE"] = "Config.yml"
+            os.environ["KONFAI_CONFIG_FILE"] = "Config.yml"
     else:
-        os.environ["DEEP_LEARNING_API_CONFIG_FILE"] = config["config"]
+        os.environ["KONFAI_CONFIG_FILE"] = config["config"]
     torch.autograd.set_detect_anomaly(True)
     os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
     
     if config["type"] is State.PREDICTION:    
         from konfai.predictor import Predictor
-        os.environ["DEEP_LEARNING_API_ROOT"] = "Predictor"
+        os.environ["KONFAI_ROOT"] = "Predictor"
         return Predictor(config=CONFIG_FILE())    
     elif config["type"] is State.EVALUATION:
         from konfai.evaluator import Evaluator
-        os.environ["DEEP_LEARNING_API_ROOT"] = "Evaluator"
+        os.environ["KONFAI_ROOT"] = "Evaluator"
         return Evaluator(config=CONFIG_FILE())
     else:
         from konfai.trainer import Trainer
-        os.environ["DEEP_LEARNING_API_ROOT"] = "Trainer"
+        os.environ["KONFAI_ROOT"] = "Trainer"
         return Trainer(config=CONFIG_FILE())
 
 

{konfai-1.0.8 → konfai-1.0.9}/konfai.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: konfai
-Version: 1.0.8
+Version: 1.0.9
 Summary: Modular and configurable Deep Learning framework with YAML and PyTorch
 Author-email: Valentin Boussot <boussot.v@gmail.com>
 License-Expression: Apache-2.0

{konfai-1.0.8 → konfai-1.0.9}/konfai.egg-info/SOURCES.txt

@@ -12,10 +12,10 @@ konfai.egg-info/dependency_links.txt
 konfai.egg-info/entry_points.txt
 konfai.egg-info/requires.txt
 konfai.egg-info/top_level.txt
-konfai/data/HDF5.py
 konfai/data/__init__.py
 konfai/data/augmentation.py
-konfai/data/dataset.py
+konfai/data/data_manager.py
+konfai/data/patching.py
 konfai/data/transform.py
 konfai/metric/__init__.py
 konfai/metric/measure.py

{konfai-1.0.8 → konfai-1.0.9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "konfai"
-version = "1.0.8"
+version = "1.0.9"
 description = "Modular and configurable Deep Learning framework with YAML and PyTorch"
 readme = "README.md"
 requires-python = ">=3.8"

konfai-1.0.8/konfai/__init__.py

@@ -1,16 +0,0 @@
-import os
-import datetime
-
-MODELS_DIRECTORY = lambda : os.environ["DL_API_MODELS_DIRECTORY"]
-CHECKPOINTS_DIRECTORY =lambda : os.environ["DL_API_CHECKPOINTS_DIRECTORY"]
-MODEL = lambda : os.environ["DL_API_MODEL"]
-PREDICTIONS_DIRECTORY =lambda : os.environ["DL_API_PREDICTIONS_DIRECTORY"]
-EVALUATIONS_DIRECTORY =lambda : os.environ["DL_API_EVALUATIONS_DIRECTORY"]
-STATISTICS_DIRECTORY = lambda : os.environ["DL_API_STATISTICS_DIRECTORY"]
-SETUPS_DIRECTORY = lambda : os.environ["DL_API_SETUPS_DIRECTORY"]
-CONFIG_FILE = lambda : os.environ["DEEP_LEARNING_API_CONFIG_FILE"]
-DL_API_STATE = lambda : os.environ["DL_API_STATE"]
-DEEP_LEARNING_API_ROOT = lambda : os.environ["DEEP_LEARNING_API_ROOT"]
-CUDA_VISIBLE_DEVICES = lambda : os.environ["CUDA_VISIBLE_DEVICES"]
-
-DATE = lambda : datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")

konfai-1.0.8/konfai/main.py

@@ -1,45 +0,0 @@
-import argparse
-import os
-from torch.cuda import device_count
-import torch.multiprocessing as mp
-from konfai.utils.utils import setupAPI, TensorBoard, Log
-
-import sys
-sys.path.insert(0, os.getcwd())
-
-def main():
-    parser = argparse.ArgumentParser(description="KonfAI", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    with setupAPI(parser) as distributedObject:
-        with Log(distributedObject.name):
-            world_size = device_count()
-            if world_size == 0:
-                world_size = 1
-            distributedObject.setup(world_size)
-            with TensorBoard(distributedObject.name):
-                mp.spawn(distributedObject, nprocs=world_size)
-
-
-def cluster():
-    parser = argparse.ArgumentParser(description="KonfAI", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-
-    # Cluster manager arguments
-    cluster_args = parser.add_argument_group('Cluster manager arguments')
-    cluster_args.add_argument('--name', type=str, help='Task name', required=True)
-    cluster_args.add_argument('--num-nodes', '--num_nodes', default=1, type=int, help='Number of nodes')
-    cluster_args.add_argument('--memory', type=int, default=16, help='Amount of memory per node')
-    cluster_args.add_argument('--time-limit', '--time_limit', type=int, default=1440, help='Job time limit in minute')
-    cluster_args.add_argument('--resubmit', action='store_true', help='Automatically resubmit job just before timout')
-
-    with setupAPI(parser) as distributedObject:
-        args = parser.parse_args()
-        config = vars(args)
-        os.environ["DL_API_OVERWRITE"] = "True"
-        os.environ["DL_API_CLUSTER"] = "True"
-
-        n_gpu = len(config["gpu"].split(","))
-        distributedObject.setup(n_gpu*int(config["num_nodes"]))
-        import submitit
-        executor = submitit.AutoExecutor(folder="./Cluster/")
-        executor.update_parameters(name=config["name"], mem_gb=config["memory"], gpus_per_node=n_gpu, tasks_per_node=n_gpu//distributedObject.size, cpus_per_task=config["num_workers"], nodes=config["num_nodes"], timeout_min=config["time_limit"])
-        with TensorBoard(distributedObject.name):
-            executor.submit(distributedObject)