radnn 0.0.6__py3-none-any.whl → 0.0.7__py3-none-any.whl

radnn/experiment/ml_experiment.py

@@ -0,0 +1,389 @@
+import os
+import numpy as np
+from datetime import datetime
+from tensorflow import keras
+from radnn import MLSystem, FileSystem, FileStore, Errors
+from radnn.data import DataSetBase
+from radnn.learn import LearningAlgorithm
+
+from radnn.plots import PlotLearningCurve
+from radnn.evaluation import EvaluateClassification
+
+from .ml_experiment_env import MLExperimentEnv
+from .ml_experiment_config import MLExperimentConfig, model_code_mllib
+from .ml_experiment_store import MLExperimentStore
+
+
+# --------------------------------------------------------------------------------------------------------------------
+# Define a custom sorting function
+def _sort_by_last_path_element(folder):
+  # Split the path into its components
+  components = folder.split(os.pathsep)
+  # Extract the last path element
+  last_element = components[-1]
+  # Extract the numeric part of the last element
+  numeric_part = ''.join(filter(str.isdigit, last_element))
+  # Convert the numeric part to an integer
+  try:
+    return int(numeric_part)
+  except ValueError:
+    # If the numeric part is not convertible to an integer, return a large number
+    return float('inf')
+# --------------------------------------------------------------------------------------------------------------------
+
+
+
+
+
+
+# ======================================================================================================================
+class MLExperiment:
+  # --------------------------------------------------------------------------------------------------------------------
+  def __init__(self, setup, model=None, learning_algorithm=None, cost_function=None, metrics=[], is_retraining=False):
+    # ...........................................................................
+    self.model = model
+    self.learning_algorithm: LearningAlgorithm = learning_algorithm
+    self.cost_function = cost_function
+    self.metrics = metrics
+    self.is_retraining = is_retraining
+    self.is_showing_step_progress = False
+    self.environment = None
+    self.config = None
+    self.model_name = None
+    self.model_fs = None
+
+    self.generator = None
+    self._dataset: DataSetBase = None
+    self._ts_feed = None
+    self._vs_feed = None
+
+    self.process_log = None
+    self.learning_history = None
+    self._evaluation = None
+
+    self._is_graph_built = False
+    self._has_loaded_state = False
+
+    self._start_train_time = None
+    self._end_train_time = None
+
+    self._currentModelFolder = None
+    self._currentModelStateFolder = None
+    self._currentModelLogFileStore = None
+    # ...........................................................................
+
+    if isinstance(setup, MLExperimentEnv):
+      self.environment = setup
+      self.config = self.environment.config
+      self.model_fs = self.environment.model_fs
+    elif isinstance(setup, MLExperimentConfig):
+      self.config = setup
+    elif isinstance(setup, dict):
+      self.config = MLExperimentConfig().assign(setup)
+    elif hasattr(setup, "config"):
+      self.config = setup.config
+    else:
+      raise Exception("Incompatible machine learning experiment setup object")
+
+    self.model_name = model_code_mllib(self.config)
+
+    #if self.environment is None:
+    #  raise Exception("Not supported yet: Creating a machine learning experiment environment from a config or dictionary object")
+
+    if self.model_fs is None:
+      if MLSystem.Instance().filesys is None:
+        raise Exception(Errors.MLSYS_NO_FILESYS)
+      oFileSys: FileSystem = MLSystem.Instance().filesys
+      self.model_fs = MLExperimentStore(oFileSys.models.folder(self.model_name))
+
+    assert isinstance(self.model_fs, MLExperimentStore), f"Unsupported model store object of class {self.model_fs.__class__}"
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def evaluation(self):
+    if self._evaluation is None:
+      raise Exception("Must run evaluation for the metrics object to become available")
+    return self._evaluation
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def model_state_folder(self):
+    return self.model_fs.state_fs.base_folder
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def checkpoint_paths(self):
+    oCheckpointFS = self.model_fs.checkpoint_fs
+    if self._currentModelFolder is not None:
+      if self._currentModelFolder != self.model_fs.base_folder:
+        oCheckpointFS = FileStore(self._currentModelFolder).subfs("checkpoints")
+
+    sCheckPointPaths = oCheckpointFS.Files("*.index", True, p_tSortFilenamesAs=int)
+    sCheckPointPaths = sorted(sCheckPointPaths, key=_sort_by_last_path_element)
+    return sCheckPointPaths
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def is_pretrained(self):
+    return self.model_fs.state_fs.exists("saved_model.pb")
+  # --------------------------------------------------------------------------------------------------------------------
+  @classmethod
+  def restore(cls, model_name, model=None):
+    oConfig = {"ModelName": model_name}
+    oExperiment = MLExperiment(oConfig, model)
+    return oExperiment.load()
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def dataset(self):
+    return self._dataset
+  # ......................................
+  @dataset.setter
+  def dataset(self, value):
+    self._dataset = value
+    if isinstance(self._dataset, DataSetBase):
+      if self._ts_feed is None:
+        self._ts_feed = self._dataset.ts.feed
+      if self._vs_feed is None:
+        self._vs_feed = self._dataset.vs.feed
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def training_set(self):
+    return self._ts_feed
+  # ......................................
+  @training_set.setter
+  def training_set(self, value):
+    self._ts_feed = value
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def validation_set(self):
+    return self._vs_feed
+  # ......................................
+  @validation_set.setter
+  def validation_set(self, value):
+    self._vs_feed = value
+
+  # --------------------------------------------------------------------------------------------------------------------
+  def print_info(self):
+    if not self._is_graph_built:
+      for oSamples, oTarget in self._ts_feed.take(1):
+        # We recall one batch/sample to create the graph and initialize the model parameters
+        y = self.model(oSamples)
+        break
+
+      self._is_graph_built = True
+
+    self.model.summary()
+  # --------------------------------------------------------------------------------------------------------------------
+  def _timing_info(self):
+    # Timings
+    nEpochs = self.config["Training.MaxEpoch"]
+    dDiff = self._end_train_time - self._start_train_time
+    nElapsedHours = dDiff.total_seconds() / 3600
+    nSecondsPerEpoch = dDiff.total_seconds() / nEpochs
+    dTiming = {"StartTime": self._start_train_time, "EndTime": self._end_train_time,
+               "ElapsedHours": nElapsedHours, "SecondsPerEpoch": nSecondsPerEpoch}
+    return dTiming
+  # --------------------------------------------------------------------------------------------------------------------
+  def save(self):
+
+    self.model.save(self.model_fs.state_fs.base_folder)
+    if self.learning_history is not None:
+      self.model_fs.log_fs.obj.save(self.learning_history, "keras_learning_history.pkl")
+      dTiming = self._timing_info()
+      self.model_fs.log_fs.obj.save(dTiming, "timing_info.pkl")
+      dTiming["StartTime"] = dTiming["StartTime"].strftime('%Y-%m-%dT%H:%M:%S')
+      dTiming["EndTime"] = dTiming["EndTime"].strftime('%Y-%m-%dT%H:%M:%S')
+      self.model_fs.log_fs.json.save(dTiming, f"timing_info_{self._end_train_time.strftime('%Y-%m-%dT%H%M%S')}.json",
+                             is_sorted_keys=False)
+
+    # //TODO: Keep cost function names and other learning parameters for evaluation
+  # --------------------------------------------------------------------------------------------------------------------
+  def load(self, use_last_checkpoint=False, model_root_folder=None):
+    self._currentModelFolder = self.model_fs.base_folder
+    self._currentModelStateFolder = self.model_fs.state_fs.base_folder
+    self._currentModelLogFileStore = self.model_fs.log_fs
+    if model_root_folder is not None:
+      self._currentModelFolder = model_root_folder
+      self._currentModelStateFolder = os.path.join(model_root_folder, "state")
+      self._currentModelLogFileStore = FileStore(model_root_folder).subfs("logs")
+
+    print("Loading saved state from ", self._currentModelStateFolder)
+    self.model = keras.models.load_model(self._currentModelStateFolder)
+    self.learning_history = (self._currentModelLogFileStore.obj.load("keras_learning_history.pkl"))
+    self._has_loaded_state = True
+
+    if use_last_checkpoint:
+      self.load_model_params(use_last_checkpoint=True)
+
+    return self.model
+  # --------------------------------------------------------------------------------------------------------------------
+  def unload_model(self):
+    del self.model
+    self.model = None
+  # --------------------------------------------------------------------------------------------------------------------
+  def save_model_params(self):
+    if self.model is not None:
+      self.model.save_weights(self.model_fs.param_fs.folder("model_params"))
+  # --------------------------------------------------------------------------------------------------------------------
+  def load_model_params(self, checkpoint_path=None, use_last_checkpoint=False, is_ignoring_not_found=False):
+    sTargetPath = None
+    sCheckPointPaths = self.checkpoint_paths
+    if checkpoint_path is not None:
+      if checkpoint_path in sCheckPointPaths:
+        sTargetPath = checkpoint_path
+      else:
+        raise Exception(f"Model params not found in checkpoint path {checkpoint_path}")
+    elif use_last_checkpoint:
+      if len(sCheckPointPaths) > 0:
+        sTargetPath = sCheckPointPaths[-1]
+      else:
+        raise Exception(f"No checkpoints are saved in {self.model_fs.checkpoint_fs.base_folder}")
+    else:
+      if self.model_fs.param_fs.is_empty:
+        raise Exception(f"Model params not found in {self.model_fs.param_fs.base_folder}")
+      else:
+        sTargetPath = self.model_fs.param_fs.subpath("model_params")
+
+    if sTargetPath is not None:
+      if is_ignoring_not_found:
+        self.model.load_weights(sTargetPath)
+        '''
+        nLogLevel = tf.get_logger().getEffectiveLevel()
+        try:
+          tf.get_logger().setLevel(logging.CRITICAL) #This does not suppress warning during loading of models
+
+        finally:
+          tf.get_logger().setLevel(nLogLevel)
+        '''
+      else:
+        self.model.load_weights(sTargetPath)
+      print("Loaded weights from %s" % sTargetPath)
+      self._has_loaded_state = True
+  # --------------------------------------------------------------------------------------------------------------------
+  def transfer_model_params_to(self, new_model, input_shape, metircs=[]):
+    self.save_model_params()
+    del self.model
+    self.model = new_model
+    self.model.build(input_shape=input_shape)
+    self.load_model_params()
+    self.model.compile(metrics=metircs)  # , run_eagerly = True)
+    return self.model
+  # --------------------------------------------------------------------------------------------------------------------
+  def train(self):
+    # //TODO: Decouple from Keras training
+
+    if (not self.is_pretrained) or self.is_retraining:
+      oOptimizer = self.learning_algorithm.optimizer
+      self._start_train_time = datetime.now()
+      self.model.compile(loss=self.cost_function, optimizer=oOptimizer, metrics=self.metrics)
+      if MLSystem.Instance().switches["IsDebuggable"]:
+        self.model.run_eagerly = True
+
+      if self.is_showing_step_progress:
+        nVerbose = 1
+      else:
+        nVerbose = 2
+
+      nEpochs = self.config["Training.MaxEpoch"]
+
+      if self.generator is not None:
+        self.process_log = self.model.fit_generator(generator=self.generator,
+                                                    epochs=nEpochs,
+                                                    callbacks=self.learning_algorithm.callbacks,
+                                                    validation_data=self._vs_feed,
+                                                    verbose=nVerbose)
+      else:
+        if "Training.StepsPerEpoch" in self.config:
+          self.process_log = self.model.fit(self._ts_feed
+                                            , batch_size=self.config["Training.BatchSize"]
+                                            , epochs=nEpochs
+                                            , validation_data=self._vs_feed
+                                            , callbacks=self.learning_algorithm.callbacks
+                                            , steps_per_epoch=self.config["Training.StepsPerEpoch"]
+                                            , verbose=nVerbose)
+        else:
+          self.process_log = self.model.fit(self._ts_feed
+                                            , batch_size=self.config["Training.BatchSize"]
+                                            , epochs=nEpochs
+                                            , validation_data=self._vs_feed
+                                            , callbacks=self.learning_algorithm.callbacks
+                                            , verbose=nVerbose)
+      self._end_train_time = datetime.now()
+      self.learning_history = self.process_log.history
+      self.save()
+      self._is_graph_built = True
+    else:
+      self.load()
+
+    return self.model
+  # --------------------------------------------------------------------------------------------------------------------
+
+
+  # // Evaluation \\
+  # --------------------------------------------------------------------------------------------------------------------
+  def plot_learning_curve(self):
+    oTrainingLogPlot = PlotLearningCurve(self.learning_history, self.model_name)
+    oTrainingLogPlot.prepare(metric_key=self.metrics[0]).show()
+    oTrainingLogPlot.prepare_cost(self.cost_function).show()
+  # --------------------------------------------------------------------------------------------------------------------
+  def _evaluation_metrics(self, true_class_labels, predicted_class_labels):
+    true_class_labels = true_class_labels.reshape(-1)
+    predicted_class_labels = predicted_class_labels.reshape(-1)
+    self._evaluation = EvaluateClassification(true_class_labels, predicted_class_labels)
+
+    return self._evaluation
+  # --------------------------------------------------------------------------------------------------------------------
+  def _evaluation_report(self, true_class_labels, predicted_class_labels, p_nID=None):
+    true_class_labels = true_class_labels.reshape(-1)
+    predicted_class_labels = predicted_class_labels.reshape(-1)
+
+    self._evaluation = EvaluateClassification(true_class_labels, predicted_class_labels)
+    self._evaluation.print_confusion_matrix()
+    self._evaluation.print_overall()
+    self._evaluation.print_per_class()
+
+    if p_nID is not None:
+      bMissclassifiedFlags = (true_class_labels != predicted_class_labels)
+      print(f"Missclassified Samples: {np.sum(bMissclassifiedFlags)}/{true_class_labels.shape[0]}")
+
+      nMissTrue = true_class_labels[bMissclassifiedFlags]
+      nMissPredicted = predicted_class_labels[bMissclassifiedFlags]
+      nMissIDs = p_nID[bMissclassifiedFlags]
+      for i, nID in enumerate(nMissIDs):
+        print(f"  |__ Sample#{int(nID):07d} True:{int(nMissTrue[i])} != {int(nMissPredicted[i])}")
+
+    return self._evaluation
+  # --------------------------------------------------------------------------------------------------------------------
+  def evaluate_classifier(self, true_class_labels=None, sample_feed=None, is_evaluating_using_keras=False, is_printing=True):
+    if sample_feed is None:
+      sample_feed = self._vs_feed
+
+    nLoss, nAccuracy = (None, None)
+    if is_evaluating_using_keras:
+      oTestResults = self.model.evaluate(sample_feed, verbose=1)
+      nLoss, nAccuracy = oTestResults
+      print("Evaluation: Loss:%.6f - Accuracy:%.6f" % (nLoss, nAccuracy))
+
+    # ... // Evaluate \\ ...
+    nVerbose = 0
+    if is_printing:
+      nVerbose = 1
+
+    nPrediction = self.model.predict(sample_feed, verbose=nVerbose)
+    if isinstance(nPrediction, tuple):
+      nPredictedClassProbabilities, nPredictedClassLabels, nIDs = nPrediction
+    else:
+      nPredictedClassProbabilities = nPrediction
+      nPredictedClassLabels = np.argmax(nPredictedClassProbabilities, axis=1)
+      nIDs = None
+
+    if true_class_labels is None:
+      true_class_labels = self._dataset.vs_labels
+
+    if is_printing:
+      return self._evaluation_report(true_class_labels, nPredictedClassLabels, nIDs), nLoss, nAccuracy
+    else:
+      return self._evaluation_metrics(true_class_labels, nPredictedClassLabels), nLoss, nAccuracy
+  # --------------------------------------------------------------------------------------------------------------------
+
+
+
+
+
+

radnn/experiment/ml_experiment_config.py

@@ -30,6 +30,35 @@ from datetime import datetime
 
 from radnn.system import FileSystem
 
+# --------------------------------------------------------------------------------------
+def model_code_mllib(p_oDict):
+  if "Experiment.BaseName" in p_oDict:
+    sBaseName = p_oDict["Experiment.BaseName"]
+    nNumber = int(p_oDict["Experiment.Number"])
+    sVariation = None
+    if "Experiment.Variation" in p_oDict:
+      sVariation = p_oDict["Experiment.Variation"]
+    nFoldNumber = None
+    if "Experiment.FoldNumber" in p_oDict:
+      nFoldNumber = p_oDict["Experiment.FoldNumber"]
+
+    sCode = "%s_%02d" % (sBaseName, nNumber)
+    if sVariation is not None:
+      sCode += ".%s" % str(sVariation)
+    if nFoldNumber is not None:
+      sCode += "-%02d" % int(nFoldNumber)
+
+  elif "ModelName" in p_oDict:
+    sCode = p_oDict["ModelName"]
+    if "ModelVariation" in p_oDict:
+      sCode += "_" + p_oDict["ModelVariation"]
+    if "ExperimentNumber" in p_oDict:
+      sCode = sCode + "_%02d" % p_oDict["ExperimentNumber"]
+
+  return sCode
+# --------------------------------------------------------------------------------------
+
+
 # --------------------------------------------------------------------------------------
 def legacy_model_code(config_dict):
   if "ModelName" in config_dict:
@@ -163,19 +192,18 @@ class MLExperimentConfig(dict):
       self["Experiment.BaseName"] += "_" + config_dict["DatasetName"]
     return self
   # --------------------------------------------------------------------------------------
-  def save(self, filename):
+  def save_to_json(self, filename=None):
     if filename is not None:
       self.filename = filename
 
-    sJSON = json.dumps(self, default=lambda o: o.__dict__, sort_keys=False, indent=4)
+    sJSON = json.dumps(self, sort_keys=False, indent=4)
     with open(self.filename, "w") as oFile:
       oFile.write(sJSON)
       oFile.close()
 
     return self
-
   # --------------------------------------------------------------------------------------
-  def save_config(self, fs, filename_only=None):
+  def save(self, fs, filename_only=None):
     if isinstance(fs, FileSystem):
       fs = fs.configs
 
@@ -183,8 +211,11 @@ class MLExperimentConfig(dict):
       filename_only = get_experiment_code(self)
 
     sFileName = fs.file(filename_only + ".json")
-    return self.save(sFileName)
-
+    return self.save_to_json(sFileName)
+  # --------------------------------------------------------------------------------------
+  def save_config(self, fs, filename_only):
+    # Backwards compatibility 0.6.0
+    return self.save()
   # --------------------------------------------------------------------------------------
   def load_config(self, fs, filename_only):
     if isinstance(fs, FileSystem):

radnn/experiment/ml_experiment_env.py

@@ -36,6 +36,11 @@ from radnn.system.tee_logger import TeeLogger
 
 
 
+
+
+
+
+
 class MLExperimentEnv(dict):
 
   # --------------------------------------------------------------------------------------------------------------------
@@ -98,7 +103,7 @@ class MLExperimentEnv(dict):
       sExperimentFileName = experiment_filename
     else:
       sExperimentFileName = oConfigFS.file(f"{experiment_base_name}_{experiment_variation}.json")
-    oConfig = MLExperimentConfig(sExperimentFileName,p_nExperimentNumber=experiment_variation)
+    oConfig = MLExperimentConfig(filename=sExperimentFileName, variation=experiment_variation)
     oPrintOutput.append(f" |__ {'configuration file':<24}: {sExperimentFileName}")
 
     return oConfig, oPrintOutput, dExperimentSpec
@@ -108,6 +113,7 @@ class MLExperimentEnv(dict):
   def __init__(self, config_fs, model_fs=None, base_name=None, number=None, variation=None, fold_number=None,
                experiment_filename=None, experiment_code=None, experiment_config=None, model_filestore=None):
 
+    super(MLExperimentEnv, self).__init__()
     if isinstance(config_fs, FileSystem):
       oConfigFS = config_fs.configs
       oModelFS  = config_fs.models

radnn/experiment/ml_experiment_store.py

@@ -0,0 +1,10 @@
+from radnn import FileStore
+
+class MLExperimentStore(FileStore):
+  def __init__(self, base_folder, is_verbose=False, must_exist=False):
+    super(MLExperimentStore, self).__init__(base_folder, is_verbose, must_exist)
+
+    self.param_fs = self.subfs("weights")
+    self.log_fs = self.subfs("logs")
+    self.checkpoint_fs = self.subfs("checkpoints")
+    self.state_fs = self.subfs("state")

radnn/learn/__init__.py

@@ -0,0 +1,7 @@
+from radnn import mlsys
+
+if mlsys.is_tensorflow_installed:
+  from .keras_optimization_algorithm import KOptimizationAlgorithm
+  from .keras_learning_rate_scheduler import KLearningRateScheduler
+
+from .learning_algorithm import LearningAlgorithm

radnn/learn/keras_learning_rate_scheduler.py

@@ -0,0 +1,31 @@
+import tensorflow.keras as ker
+
+class KLearningRateScheduler(ker.callbacks.LearningRateScheduler):
+  # -----------------------------------------------------------------------------------
+  def __init__(self, config, is_verbose=True):
+    self.config = None
+    self.lr_schedule = None
+    self.is_verbose = is_verbose
+
+    if isinstance(config, dict):
+      self.config = config
+      self.lr_schedule = config["Training.LearningRateSchedule"]
+    elif isinstance(config, list):
+      self.lr_schedule = config
+
+    super(KLearningRateScheduler, self).__init__(self.check_schedule)
+  # -----------------------------------------------------------------------------------
+  def check_schedule(self, epoch, lr):
+    nNewLR = lr
+
+    for nIndex, oSchedule in enumerate(self.lr_schedule):
+      if epoch == oSchedule[0]:
+        nNewLR = oSchedule[1]
+        if self.is_verbose:
+          print("Schedule #%d: Setting LR to %.5f" % (nIndex + 1, nNewLR))
+        break
+
+    if self.is_verbose:
+      print("LR: %.6f" % nNewLR)
+    return nNewLR
+  # -----------------------------------------------------------------------------------

radnn/learn/keras_optimization_algorithm.py

@@ -0,0 +1,32 @@
+import tensorflow.keras as ker
+from .keras_learning_rate_scheduler import KLearningRateScheduler
+
+class KOptimizationAlgorithm(object):
+  # -----------------------------------------------------------------------------------
+  def __init__(self, config, is_verbose=True):
+    self.config = config
+    self.optimizer = None
+    self.callbacks = []
+    self.is_verbose = is_verbose
+
+    self.optimizer_name = self.config["Training.Optimizer"].upper()
+    if self.optimizer_name == "SGD":
+      self.optimizer = ker.optimizers.SGD(learning_rate=self.config["Training.LearningRate"],
+                                          momentum=self.config["Training.Momentum"])
+      if "Training.LearningRateSchedule" in self.config:
+        oLearningRateSchedule = KLearningRateScheduler(self.config)
+      self.callbacks.append(oLearningRateSchedule)
+    elif self.optimizer_name == "ADAM":
+      self.optimizer = ker.optimizers.Adam(learning_rate=self.config["Training.LearningRate"])
+    elif self.optimizer_name == "RMSPROP":
+      # //TODO: Rho
+      if "Training.Momentum" in self.config:
+        self.optimizer = ker.optimizers.RMSprop(learning_rate=self.config["Training.LearningRate"],
+                                                momentum=self.config["Training.Momentum"])
+      else:
+        self.optimizer = ker.optimizers.RMSprop(learning_rate=self.config["Training.LearningRate"])
+
+    assert self.optimizer is not None, f'Unsupported optimizer {self.config["Training.Optimizer"]}'
+    if self.is_verbose:
+      print(f"Learning algorithm {self.optimizer_name}")
+  # -----------------------------------------------------------------------------------

radnn/learn/learning_algorithm.py

@@ -0,0 +1,35 @@
+from radnn.core import is_tensorflow_installed
+
+if is_tensorflow_installed:
+  from .keras_optimization_algorithm import KOptimizationAlgorithm
+
+class LearningAlgorithm(object):
+  # -----------------------------------------------------------------------------------
+  def __init__(self, config, is_verbose=True):
+    self.config = config
+    self.is_verbose = is_verbose
+    self._implementation = None
+
+    self.prepare()
+  # -----------------------------------------------------------------------------------
+  @property
+  def optimizer(self):
+    oResult = None
+    if self._implementation is not None:
+      if isinstance(self._implementation, KOptimizationAlgorithm):
+        oResult = self._implementation.optimizer
+    return oResult
+  # -----------------------------------------------------------------------------------
+  @property
+  def callbacks(self):
+    oResult = None
+    if self._implementation is not None:
+      if isinstance(self._implementation, KOptimizationAlgorithm):
+        oResult = self._implementation.callbacks
+    return oResult
+  # -----------------------------------------------------------------------------------
+  def prepare(self):
+    if is_tensorflow_installed:
+      self._implementation = KOptimizationAlgorithm(self.config, self.is_verbose)
+    return self
+  # -----------------------------------------------------------------------------------

radnn/learn/state/__init__.py

@@ -0,0 +1,4 @@
+from radnn import mlsys
+
+if mlsys.is_tensorflow_installed:
+  from .keras_best_state_saver import KBestStateSaver

radnn/learn/state/keras_best_state_saver.py

@@ -0,0 +1,17 @@
+import os
+import tensorflow.keras as ker
+
+class KBestStateSaver(object):
+  # -----------------------------------------------------------------------------------
+  def __init__(self, experiment_fs, metric, verbose=2):
+    self.experiment_fs = experiment_fs
+    self.metric = metric
+    self.checkpoint_fs = self.experiment_fs.subfs("checkpoints")
+
+    sCheckPointPathTemplate = os.path.join(self.checkpoint_fs.base_folder, "{epoch}")
+
+    self.Callback = ker.callbacks.ModelCheckpoint(  filepath=sCheckPointPathTemplate,
+                                                    verbose=verbose, save_weights_only=True,
+                                                    monitor="val_" + self.metric,
+                                                    mode="max", save_best_only=True)
+  # -----------------------------------------------------------------------------------