konfai 1.1.1__py3-none-any.whl → 1.1.3__py3-none-any.whl

konfai/utils/utils.py

@@ -18,13 +18,15 @@ import random
 from torch.utils.data import DataLoader
 import torch.nn.functional as F
 import sys
+import re
+
 
 def description(model, modelEMA = None, showMemory: bool = True) -> str:
     values_desc = lambda weights, values: " ".join(["{}({:.2f}) : {:.6f}".format(name.split(":")[-1], weight, value) for (name, value), weight in zip(values.items(), weights.values())])
     model_desc = lambda model : "("+" ".join(["{}({:.6f}) : {}".format(name, network.optimizer.param_groups[0]['lr'] if network.optimizer is not None else 0, values_desc(network.measure.getLastWeights(), network.measure.getLastValues())) for name, network in model.module.getNetworks().items() if network.measure is not None])+")"
     result = "Loss {}".format(model_desc(model))
     if modelEMA is not None:
-        result += "Loss EMA {}".format(model_desc(modelEMA))
+        result += " Loss EMA {}".format(model_desc(modelEMA))
     result += " "+gpuInfo()
     if showMemory:
         result +=" | {}".format(memoryInfo())
@@ -237,7 +239,7 @@ class DataLog(Enum):
     AUDIO   = lambda tb, name, layer, it : tb.add_audio(name, _logImageFormat(layer), it)
 
 class Log:
-    def __init__(self, name: str) -> None:
+    def __init__(self, name: str, rank: int) -> None:
         if KONFAI_STATE() == "PREDICTION":
             path = PREDICTIONS_DIRECTORY()
         elif KONFAI_STATE() == "EVALUATION":
@@ -248,11 +250,12 @@ class Log:
         self.verbose = os.environ.get("KONFAI_VERBOSE", "True") == "True"
         self.log_path = os.path.join(path, name)
         os.makedirs(self.log_path, exist_ok=True)
-        
-        self.file = open(os.path.join(self.log_path, "log.txt"), "w", buffering=1) 
+        self.rank = rank
+        self.file = open(os.path.join(self.log_path, "log_{}.txt".format(rank)), "w", buffering=1) 
         self.stdout_bak = sys.stdout
         self.stderr_bak = sys.stderr
-
+        self._buffered_line = ""
+    
     def __enter__(self):
         self.file.__enter__()
         sys.stdout = self
@@ -264,12 +267,26 @@ class Log:
         sys.stdout = self.stdout_bak
         sys.stderr = self.stderr_bak
 
-    def write(self, msg):
+    def write(self, msg: str):
         if not msg:
             return
-        self.file.write(msg)
-        self.file.flush()
-        if self.verbose:
+        
+        
+        ANSI_ESCAPE = re.compile(r'\x1b\[[0-9;]*[a-zA-Z]')
+        CARRIAGE_RETURN = re.compile(r'(?:\r|\x1b\[A).*')
+        msg_clean = ANSI_ESCAPE.sub('', msg)
+        if '\r' in msg_clean or '' in msg:
+            # On garde seulement le contenu après le dernier retour chariot
+            msg_clean = msg_clean.split('\r')[-1].strip()
+            self._buffered_line = msg_clean
+        else:
+            self._buffered_line = msg_clean.strip()
+
+        if self._buffered_line:
+            # Écrit dans le fichier
+            self.file.write(self._buffered_line + "\n")
+            self.file.flush()
+        if self.verbose and (self.rank == 0 or "KONFAI_CLUSTER" in os.environ):
             sys.__stdout__.write(msg)
             sys.__stdout__.flush()
 
@@ -352,7 +369,7 @@ class DistributedObject():
         return result
 
     def __call__(self, rank: Union[int, None] = None) -> None:
-        with Log(self.name):
+        with Log(self.name, rank):
             world_size = len(self.dataloader)
             global_rank, local_rank = setupGPU(world_size, self.port, rank)
             if global_rank is None:
@@ -385,6 +402,7 @@ def setup(parser: argparse.ArgumentParser) -> DistributedObject:
     KONFAI_args.add_argument('-tb', action='store_true', help='Start TensorBoard')
     KONFAI_args.add_argument("-c", "--config", type=str, default="None", help="Configuration file location")
     KONFAI_args.add_argument("-g", "--gpu", type=str, default=os.environ["CUDA_VISIBLE_DEVICES"] if "CUDA_VISIBLE_DEVICES" in os.environ else "", help="List of GPU")
+    KONFAI_args.add_argument("-cpu", "--cpu", type=str, default="1" , help="List of GPU")
     KONFAI_args.add_argument('--num-workers', '--num_workers', default=4, type=int, help='No. of workers per DataLoader & GPU')
     KONFAI_args.add_argument("-models_dir", "--MODELS_DIRECTORY", type=str, default="./Models/", help="Models location")
     KONFAI_args.add_argument("-checkpoints_dir", "--CHECKPOINTS_DIRECTORY", type=str, default="./Checkpoints/", help="Checkpoints location")
@@ -401,6 +419,7 @@ def setup(parser: argparse.ArgumentParser) -> DistributedObject:
     config = vars(args)
 
     os.environ["CUDA_VISIBLE_DEVICES"] = config["gpu"]
+    os.environ["KONFAI_NB_CORES"] = config["cpu"]
     os.environ["KONFAI_MODELS_DIRECTORY"] = config["MODELS_DIRECTORY"]
     os.environ["KONFAI_CHECKPOINTS_DIRECTORY"] = config["CHECKPOINTS_DIRECTORY"]
     os.environ["KONFAI_PREDICTIONS_DIRECTORY"] = config["PREDICTIONS_DIRECTORY"]
@@ -465,6 +484,14 @@ def setupGPU(world_size: int, port: int, rank: Union[int, None] = None) -> tuple
     if torch.cuda.is_available():
         torch.cuda.empty_cache()
         dist.init_process_group("nccl", rank=global_rank, init_method="tcp://{}:{}".format(host_name, port), world_size=world_size)
+    else:
+        if not dist.is_initialized():
+            dist.init_process_group(
+                backend="gloo",
+                init_method=f"tcp://{host_name}:{port}",
+                rank=global_rank,
+                world_size=world_size
+            )
     return global_rank, local_rank
 
 import socket
@@ -524,8 +551,8 @@ SUPPORTED_EXTENSIONS = [
 
 class KonfAIError(Exception):
 
-    def __init__(self, typeError: str, message: list[str]) -> None:
-        super().__init__("\n[{}] {}".format(typeError, message[0])+"\n→\t".join(message[1:]))
+    def __init__(self, typeError: str, messages: list[str]) -> None:
+        super().__init__("\n[{}] {}".format(typeError, messages[0])+("\n" if len(messages) > 0 else "")+"\n→\t".join(messages[1:]))
 
 
 class ConfigError(KonfAIError):
@@ -553,3 +580,13 @@ class AugmentationError(KonfAIError):
 
     def __init__(self, *message) -> None:
         super().__init__("Augmentation", message)
+
+class EvaluatorError(KonfAIError):
+
+    def __init__(self, *message) -> None:
+        super().__init__("Evaluator", message)
+
+class TransformError(KonfAIError):
+
+    def __init__(self, *message) -> None:
+        super().__init__("Transform", message)

{konfai-1.1.1.dist-info → konfai-1.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: konfai
-Version: 1.1.1
+Version: 1.1.3
 Summary: Modular and configurable Deep Learning framework with YAML and PyTorch
 Author-email: Valentin Boussot <boussot.v@gmail.com>
 License-Expression: Apache-2.0

{konfai-1.1.1.dist-info → konfai-1.1.3.dist-info}/RECORD

@@ -1,13 +1,13 @@
-konfai/__init__.py,sha256=fZM9EjrMcdJZE-jQAsNu3ZcWfa9Ylbly2q4l9STfpBI,795
-konfai/evaluator.py,sha256=TsHQNN4DhP33Sal9-exbOouZAHe7gNEk7RsBaasgkEQ,7373
-konfai/main.py,sha256=OPAUWees1j6NNmggyI7lM2abZctCqk1OsvQuc9FpePY,2515
-konfai/predictor.py,sha256=FGn6_Nt4PCdxNFoIOt8nbBjrAMTwc55uZqy6OFXUjhY,22320
-konfai/trainer.py,sha256=22IKMiPeHNJMGcJdVq9BH-Hsopp6xF4QSYyPabz4LGs,19604
+konfai/__init__.py,sha256=YXG-wpSEXWs6Jt3BDI77V4r89gEUNX-6lxW9btj5VYI,851
+konfai/evaluator.py,sha256=rAhfdRemMjzC3VoaqyQKJR0SBekuLDiLT1nhblH8RQk,8293
+konfai/main.py,sha256=xhCIs3VKXxNCgyyN616K-VANEFCsZZRR-l8LIMHlmPw,2597
+konfai/predictor.py,sha256=SgHs_gylsyVtQ41DBRBRmcpHIMg2ghLbxELbQBxjjFY,22747
+konfai/trainer.py,sha256=GCi8oRgTlbZxwKfmJlS2S4VU1yj1pCl6QeAWNwdSFPs,20223
 konfai/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-konfai/data/augmentation.py,sha256=ASmKWBpykLBHDB_YeTgoBTlqvJ06v5OUG7n7ugPN6NU,31718
-konfai/data/data_manager.py,sha256=AfZpQaHEd4QmA8Sxd6SK2LK3Nv658wlZVbivETq4Y4k,28439
-konfai/data/patching.py,sha256=JFgb4qgS_gqlcnFV_Rm0JvCzKQleXNZyOWxqqM8H01M,14499
-konfai/data/transform.py,sha256=ttALCEvvqIcwcYJTp4KCnf87e2hzvkXlwrtksfpWRUU,25064
+konfai/data/augmentation.py,sha256=kp4HyMSGflbuHbOsNiXBxszLFSuOZ4zRRQT0cHNIVUI,32620
+konfai/data/data_manager.py,sha256=oG73iHT5qV52UTWQPC3MKjsjaQHJUOAPF1WA-XeTmUE,29700
+konfai/data/patching.py,sha256=oq43r3JlS9hw-pnzpjT3epvXJxEKqiQEtFVZPjY9Jic,15749
+konfai/data/transform.py,sha256=ZZoxft0V8t-DaKjF3OsIFezz41B8_xwPqBvYCYwF0dk,26471
 konfai/metric/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konfai/metric/measure.py,sha256=K7s15wNJm1_Iaav9y7Oe_UhSnbLUiybhLTVumHZ8ZaY,22001
 konfai/metric/schedulers.py,sha256=UoSr1TW_hrus3DhvOEbDefxCSUGz7lJS_8vbz0GEye8,1370
@@ -21,19 +21,19 @@ konfai/models/generation/vae.py,sha256=_3JYVT2ojZ0P98tYcD2ny7a-gWVUmnByLDhY7i-n_
 konfai/models/registration/registration.py,sha256=18EiWt4RJIXLyFtqU-kHjV1sMnQRm9mxAA6_-2B1YqI,6313
 konfai/models/representation/representation.py,sha256=RwQYoxtdph440-t_ZLelykl0hkUAD1zdspQaLkgxb-0,2677
 konfai/models/segmentation/NestedUNet.py,sha256=6XGizAIc4bDL8vx4AHW8BBFjUvovRYcjdMBHsN4ViNo,4301
-konfai/models/segmentation/UNet.py,sha256=9-g63oNqaxSlGmrD1-IVzJ-kac9QphmM7i-3rEsH23I,4218
+konfai/models/segmentation/UNet.py,sha256=JhKyGuaFXY7thJcbQ2mWGuIBtNdUhh5FoacpWDZ0N1k,4065
 konfai/network/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konfai/network/blocks.py,sha256=RaTI0Lrvq1V-GIFei-WTUB6wlg4LydZksAyJ8DMk40M,13502
-konfai/network/network.py,sha256=DTROmwwPJZQllJx32GwAixfy1E23F1mtYy1bpF23TjU,46538
+konfai/network/network.py,sha256=SnwQUKFTZZU8zuix5sm0vW8s0G3h6Fa8pHoIcwC984Q,46512
 konfai/utils/ITK.py,sha256=OxTieDNNYHGkn7zxJsAG-6ecRG1VYMvn1dlBbBe1DOs,13955
 konfai/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konfai/utils/config.py,sha256=OsCUqgNaPX56nD_MpJ2WRpiQ7xxqHonfPYaS9VGtsl8,12413
-konfai/utils/dataset.py,sha256=RNULLVauhYiyeQkpVhOVq4VRwI4KRyoQA2VCmOh8VbE,35525
+konfai/utils/dataset.py,sha256=6ZzevdhJ7e5zlXATAVwSh9O6acKXM7gYNxkMAa5DrmM,36351
 konfai/utils/registration.py,sha256=v1srEBOcgDnHrx0YtsK6bcj0yCMH7wNeaQ3wC7gEvOw,8898
-konfai/utils/utils.py,sha256=GPzQQ7kBmraHytsUdPOqwW0Qu-kAHGA-wwMd4obdVLE,22018
-konfai-1.1.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-konfai-1.1.1.dist-info/METADATA,sha256=mMT6fvoAbwoMkChW6G_Y2vDhrtiMss86xezEnylzLrE,2515
-konfai-1.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-konfai-1.1.1.dist-info/entry_points.txt,sha256=fG82HRN5-g39ACSOCtij_I3N6EHxfYnMR0D7TI_8pW8,81
-konfai-1.1.1.dist-info/top_level.txt,sha256=xF470dkIlFoFqTZEOlRehKJr4WU_8OKGXrJqYm9vWKs,7
-konfai-1.1.1.dist-info/RECORD,,
+konfai/utils/utils.py,sha256=Laq8bGc5mGKFZlJIkHxa-BrC9uR2F7MTRuL4YFAIxQY,23439
+konfai-1.1.3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+konfai-1.1.3.dist-info/METADATA,sha256=Fj6snnVsKrMZjmdZZkp4u1YoonXYd_VWQV_B3hxd7qQ,2515
+konfai-1.1.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+konfai-1.1.3.dist-info/entry_points.txt,sha256=fG82HRN5-g39ACSOCtij_I3N6EHxfYnMR0D7TI_8pW8,81
+konfai-1.1.3.dist-info/top_level.txt,sha256=xF470dkIlFoFqTZEOlRehKJr4WU_8OKGXrJqYm9vWKs,7
+konfai-1.1.3.dist-info/RECORD,,