radnn 0.0.7.3__py3-none-any.whl → 0.0.8__py3-none-any.whl

radnn/__init__.py

@@ -3,7 +3,8 @@
 # Version 0.0.6    [2025-02-04]
 # Version 0.0.7.2  [2025-02-17]
 # Version 0.0.7.3  [2025-02-21]
-__version__ = "0.0.7"
+# Version 0.0.8    [2025-02-xx]
+__version__ = "0.0.8"
 
 from .system import FileStore, FileSystem
 from .ml_system import MLSystem

radnn/data/__init__.py

@@ -8,3 +8,5 @@ from radnn import mlsys
 if mlsys.is_tensorflow_installed:
   from .tf_classification_data_feed import TFClassificationDataFeed
 
+from .image_dataset_files import ImageDataSetFiles
+

radnn/data/data_feed.py

@@ -57,10 +57,15 @@ class DataFeed(object):
     self.batch_size  = None
 
     self._has_mapped_preprocessing_method = False
+    self._is_multiclass = False
 
     self.feed = self.build_iterator()
     self.pipeline_objects.append(self.feed)
   # --------------------------------------------------------------------------------------------------------------------
+  def multiclass(self):
+    self._is_multiclass = True
+    return self
+  # --------------------------------------------------------------------------------------------------------------------
   def normalize(self):
     self.value_preprocessor = Normalizer(self.dataset.name, self.dataset.filestore)
     if self.value_preprocessor.min is None:

radnn/data/dataset_folder.py

@@ -0,0 +1,55 @@
+from .subset_type import SubsetType
+
+class DataSetFolder(object):
+  # --------------------------------------------------------------------------------------------------------------------
+  def __init__(self, folder_name, filestore):
+    self.folder_name = folder_name
+    self.filestore = filestore
+    self.filestore_ts = None
+    self.filestore_vs = None
+    self.filestore_ut = None
+    self.split_filestores = []
+
+    self.subfolders = self.filestore.list_folders(is_full_path=False)
+    self.is_split, sTSFolder, sVSFolder, sUTFolder = self.get_split_subfolders(self.subfolders)
+    if self.is_split:
+      if sTSFolder is not None:
+        self.filestore_ts = self.filestore.subfs(sTSFolder, must_exist=True)
+        self.split_filestores.append(self.filestore_ts)
+      if sVSFolder is not None:
+        self.filestore_vs = self.filestore.subfs(sVSFolder, must_exist=True)
+        self.split_filestores.append(self.filestore_vs)
+      if sUTFolder is not None:
+        self.filestore_ut = self.filestore.subfs(sUTFolder, must_exist=True)
+        self.split_filestores.append(self.filestore_ut)
+  # --------------------------------------------------------------------------------------------------------------------
+  def get_split_subfolders(self, folders):
+    sTSFolder = None
+    sVSFolder = None
+    sUTFolder = None
+    bIsSplit = False
+    for sFolder in folders:
+      oFolderSubsetType = SubsetType(sFolder)
+      if oFolderSubsetType.is_training_set:
+        sTSFolder = sFolder
+        bIsSplit = True
+      elif oFolderSubsetType.is_validation_set:
+        sVSFolder = sFolder
+        bIsSplit = True
+      elif oFolderSubsetType.is_unknown_test_set:
+        sUTFolder = sFolder
+        bIsSplit = True
+
+    #  When confusing terminology is uses and 'test' means 'validation'
+    if (sUTFolder is not None) and (sVSFolder is None):
+      sVSFolder = sUTFolder
+      sUTFolder = None
+
+    return bIsSplit, sTSFolder, sVSFolder, sUTFolder
+  # --------------------------------------------------------------------------------------------------------------------
+  def __str__(self):
+    return "./" + self.folder_name
+  # --------------------------------------------------------------------------------------------------------------------
+  def __repr__(self):
+    return self.__str__()
+  # --------------------------------------------------------------------------------------------------------------------

radnn/data/image_dataset_files.py

@@ -0,0 +1,175 @@
+from radnn import FileStore
+from radnn.utils import camel_case
+from radnn.system.files import FileList
+from .dataset_folder import DataSetFolder
+import sys
+from tqdm import tqdm
+from datetime import datetime
+
+class ImageDataSetFiles(object):
+  # --------------------------------------------------------------------------------------------------------------------
+  def __init__(self, images_fs, name="files", is_progress_indicator=True):
+    self.images_fs :FileStore = images_fs
+    self.name = name
+    self.is_progress_indicator = is_progress_indicator
+    self.class_names :dict = dict()
+    self.class_folders :list = []
+    self.files :FileList = dict()
+    self.files_ts :FileList = dict()
+    self.files_vs :FileList = dict()
+    self.files_ut :FileList = dict()
+    self.total_file_count = 0
+    self.is_split_on_main_folder = False
+    self.is_split_in_class_folders = False
+    self.run_date_time = None
+
+    self.detect_class_names_from_folders()
+  # --------------------------------------------------------------------------------------------------------------------
+  def load(self, extensions="*.jpg;  *.png"):
+    if not self.load_file_lists():
+      self.detect_files(extensions)
+  # --------------------------------------------------------------------------------------------------------------------
+  def load_file_lists(self):
+    bResult = False
+    oDict = self.images_fs.obj.load(f"{self.name}-file-info.pkl")
+    if oDict is not None:
+      self.run_date_time = oDict["RunDateTime"]
+      self.class_names = oDict["ClassNames"]
+      self.class_folders = oDict["ClassFolders"]
+      self.files = oDict["ClassFiles.All"]
+      self.files_ts = oDict["ClassFiles.TrainingSet"]
+      self.files_vs = oDict["ClassFiles.ValidationSet"]
+      self.files_ut = oDict["ClassFiles.UnknownTestSet"]
+      self.total_file_count = oDict["TotalFileCount"]
+      self.is_split_on_main_folder = oDict["IsSplitOnMainFolder"]
+      self.is_split_in_class_folders = oDict["IsSplitInClassFolders"]
+      bResult = True
+
+    return bResult
+  # --------------------------------------------------------------------------------------------------------------------
+  def save_file_lists(self):
+    oDict = dict()
+    oDict["RunDateTime"] = self.run_date_time
+    oDict["ClassNames"] = self.class_names
+    oDict["ClassFolders"] = self.class_folders
+    oDict["ClassFiles.All"] = self.files
+    oDict["ClassFiles.TrainingSet"] = self.files_ts
+    oDict["ClassFiles.ValidationSet"] = self.files_vs
+    oDict["ClassFiles.UnknownTestSet"] = self.files_ut
+    oDict["TotalFileCount"] = self.total_file_count
+    oDict["IsSplitOnMainFolder"] = self.is_split_on_main_folder
+    oDict["IsSplitInClassFolders"] = self.is_split_in_class_folders
+    self.images_fs.obj.save(oDict, f"{self.name}-file-info.pkl")
+  # --------------------------------------------------------------------------------------------------------------------
+  def detect_class_names_from_folders(self):
+    oClassNamesFS = self.images_fs
+    oMainFolder = DataSetFolder("/", self.images_fs)
+    oFolders = oMainFolder.subfolders
+
+    self.is_split_on_main_folder = oMainFolder.is_split
+    if self.is_split_on_main_folder:
+      # Detect the class names under the training set subfolder
+      oClassNamesFS = oMainFolder.filestore_ts
+      oFolders = oClassNamesFS.list_folders(is_full_path=False)
+
+    for nIndex, sFolder in enumerate(oFolders):
+      sClassName = camel_case(sFolder)
+      self.class_names[nIndex] = sClassName
+      oClassFS = oClassNamesFS.subfs(sFolder, must_exist=True)
+      oClassFolder = DataSetFolder(sFolder, oClassFS)
+      if not self.is_split_on_main_folder:
+        if oClassFolder.is_split:
+          self.is_split_in_class_folders = True
+      self.class_folders.append(oClassFolder)
+
+    return self.class_folders
+
+  # --------------------------------------------------------------------------------------------------------------------
+  def traverse_sub_folders(self, extensions, progress):
+    for nClassIndex, oClassFolder in enumerate(self.class_folders):
+      if progress is not None:
+        progress.set_description(f"Finding files for class {self.class_names[nClassIndex]}")
+        progress.refresh()
+      self.files[nClassIndex] = oClassFolder.filestore.filelist(extensions)
+      self.total_file_count += len(self.files[nClassIndex])
+      if progress is not None:
+        progress.update(1)
+  # --------------------------------------------------------------------------------------------------------------------
+  def traverse_sub_folders_with_split(self, extensions, progress):
+    self.total_file_count = 0
+    for nClassIndex, oClassFolder in enumerate(self.class_folders):
+      if progress is not None:
+        progress.set_description(f"Finding files for class {self.class_names[nClassIndex]}")
+        progress.refresh()
+      if oClassFolder.is_split:
+        oClassAllFiles = FileList()
+        for nIndex, oSplitFileStore in enumerate(oClassFolder.split_filestores):
+          if oSplitFileStore is not None:
+            oFileList = oSplitFileStore.filelist(extensions)
+            for oFile in oFileList.full_paths:
+              dSplit = None
+              if oSplitFileStore == oClassFolder.filestore_ts:
+                dSplit = self.files_ts
+              elif oSplitFileStore == oClassFolder.filestore_vs:
+                dSplit = self.files_vs
+              elif oSplitFileStore == oClassFolder.filestore_ut:
+                dSplit = self.files_ut
+
+              if dSplit is not None:
+                if nClassIndex not in dSplit:
+                  dSplit[nClassIndex] = []
+                dSplit[nClassIndex].append(oFile)
+
+              oClassAllFiles.append(oFile)
+      else:
+        raise Exception(f"No split subfolders for class {nIndex} '{self.class_names[nIndex]}',\n"
+                        + f"that is stored in {oClassFolder.filestore}\n"
+                        + f"All of the classes should have the same split.")
+      self.files[nClassIndex] = oClassAllFiles
+      self.total_file_count += len(self.files[nClassIndex])
+      if progress is not None:
+        progress.update(1)
+
+
+    if progress is not None:
+      progress.set_description("Finished")
+      progress.refresh()
+  # --------------------------------------------------------------------------------------------------------------------
+  def detect_files(self, extensions="*.jpg;  *.png"):
+    oProgress = None
+    if len(self.class_folders) > 0:
+      if (not self.is_split_on_main_folder) and (not self.is_split_in_class_folders):
+        if self.is_progress_indicator:
+          oProgress = tqdm(total=len(self.class_folders), ncols=80)
+        try:
+          self.traverse_sub_folders(extensions, oProgress)
+        finally:
+          if self.is_progress_indicator:
+            oProgress.close()
+
+      elif (not self.is_split_on_main_folder) and self.is_split_in_class_folders:
+        if self.is_progress_indicator:
+          oProgress = tqdm(total=len(self.class_folders), ncols=80)
+        try:
+          self.traverse_sub_folders_with_split(extensions, oProgress)
+        finally:
+          if self.is_progress_indicator:
+            oProgress.close()
+
+      self.save_file_lists()
+  # --------------------------------------------------------------------------------------------------------------------
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

radnn/data/subset_type.py

@@ -24,8 +24,14 @@
 # ......................................................................................
 class SubsetType(object):
   def __init__(self, name):
-    self.name = name
-
+    self.name = name.lower()
+    self.type = -1 # Unknown
+    if self.is_training_set:
+      self.type = 0
+    elif self.is_validation_set:
+      self.type = 1
+    elif self.is_unknown_test_set:
+      self.type = 2
   @property
   def is_training_set(self):
     return (self.name == "training") or (self.name == "train") or (self.name == "ts")

radnn/data/tf_classification_data_feed.py

@@ -66,23 +66,39 @@ class TFClassificationDataFeed(DataFeed):
       feed = tf.data.Dataset.from_tensor_slices((self.dataset.ut_samples, self.dataset.ut_labels))
     return feed
   # --------------------------------------------------------------------------------------------------------------------
-  def preprocess_normalize(self, samples, labels):
+  def preprocess_normalize_onehot(self, samples, labels):
     tSamples = tf.cast(samples, tf.float32)
     tSamples = (tSamples - self.value_preprocessor.min) / (self.value_preprocessor.max - self.value_preprocessor.min)
     tTargetsOneHot = tf.one_hot(labels, self.dataset.class_count)
     return tSamples, tTargetsOneHot
   # --------------------------------------------------------------------------------------------------------------------
-  def preprocess_standardize(self, samples, labels):
+  def preprocess_standardize_onehot(self, samples, labels):
     tSamples = tf.cast(samples, tf.float32)
     tSamples = (tSamples - self.value_preprocessor.mean) / self.value_preprocessor.std
     tTargetsOneHot = tf.one_hot(labels, self.dataset.class_count)
     return tSamples, tTargetsOneHot
   # --------------------------------------------------------------------------------------------------------------------
+  def preprocess_normalize(self, samples, labels):
+    tSamples = tf.cast(samples, tf.float32)
+    tSamples = (tSamples - self.value_preprocessor.min) / (self.value_preprocessor.max - self.value_preprocessor.min)
+    return tSamples, labels
+  # --------------------------------------------------------------------------------------------------------------------
+  def preprocess_standardize(self, samples, labels):
+    tSamples = tf.cast(samples, tf.float32)
+    tSamples = (tSamples - self.value_preprocessor.mean) / self.value_preprocessor.std
+    return tSamples, labels
+  # --------------------------------------------------------------------------------------------------------------------
   def build_preprocessor(self, feed):
-    if isinstance(self.value_preprocessor, Standardizer):
-      feed = feed.map(self.preprocess_standardize, num_parallel_calls=8)
-    elif isinstance(self.value_preprocessor, Normalizer):
-      feed = feed.map(self.preprocess_normalize, num_parallel_calls=8)
+    if self._is_multiclass:
+      if isinstance(self.value_preprocessor, Standardizer):
+        feed = feed.map(self.preprocess_standardize_onehot, num_parallel_calls=8)
+      elif isinstance(self.value_preprocessor, Normalizer):
+        feed = feed.map(self.preprocess_normalize_onehot, num_parallel_calls=8)
+    else:
+      if isinstance(self.value_preprocessor, Standardizer):
+        feed = feed.map(self.preprocess_standardize, num_parallel_calls=8)
+      elif isinstance(self.value_preprocessor, Normalizer):
+        feed = feed.map(self.preprocess_normalize, num_parallel_calls=8)
     return feed
   # --------------------------------------------------------------------------------------------------------------------
   def build_random_shuffler(self, feed):

radnn/experiment/ml_experiment_config.py

@@ -31,8 +31,8 @@ import os
 import json
 import re
 from datetime import datetime
-
-from radnn.system import FileSystem
+from radnn.ml_system import mlsys
+from radnn.system import FileSystem, FileStore
 
 # --------------------------------------------------------------------------------------
 def model_code_mllib(p_oDict):
@@ -146,9 +146,11 @@ def experiment_code_and_timestamp(filename):
 
 
 # =========================================================================================================================
-class MLExperimentConfig(dict):
+class MLExperimentConfig(object):
   # --------------------------------------------------------------------------------------
   def __init__(self, filename=None, base_name=None, number=None, variation=None, fold_number=None, hyperparams=None):
+    self._kv = dict()
+
     self["Experiment.BaseName"] = base_name
     self.filename = filename
     if self.filename is not None:
@@ -164,28 +166,19 @@ class MLExperimentConfig(dict):
     if hyperparams is not None:
       self.assign(hyperparams)
   # --------------------------------------------------------------------------------------
+  def __getitem__(self, key):
+    return self._kv[key]
+  # --------------------------------------------------------------------------------------
+  def __setitem__(self, key, value):
+    self._kv[key] = value
+  # --------------------------------------------------------------------------------------
+  def __contains__(self, key):
+      return key in self._kv
+  # --------------------------------------------------------------------------------------
   @property
   def experiment_code(self):
     return get_experiment_code(self)
   # --------------------------------------------------------------------------------------
-  def load(self, filename=None, must_exist=False):
-    if filename is None:
-      filename = self.filename
-
-    # reading the data from the file
-    if os.path.exists(filename):
-      with open(filename) as oFile:
-        sConfig = oFile.read()
-        self.setDefaults()
-        dConfigDict = json.loads(sConfig)
-
-      for sKey in dConfigDict.keys():
-        self[sKey] = dConfigDict[sKey]
-    else:
-      if must_exist:
-        raise Exception("Experiment configuration file %s is not found." % filename)
-    return self
-  # --------------------------------------------------------------------------------------
   def assign(self, config_dict):
     for sKey in config_dict.keys():
       self[sKey] = config_dict[sKey]
@@ -200,16 +193,19 @@ class MLExperimentConfig(dict):
     if filename is not None:
       self.filename = filename
 
-    sJSON = json.dumps(self, sort_keys=False, indent=4)
+    sJSON = json.dumps(self._kv, sort_keys=False, indent=4)
     with open(self.filename, "w") as oFile:
       oFile.write(sJSON)
       oFile.close()
-
     return self
   # --------------------------------------------------------------------------------------
-  def save(self, fs, filename_only=None):
-    if isinstance(fs, FileSystem):
+  def save(self, fs=None, filename_only=None):
+    if fs is None:
+      fs = mlsys.filesys.configs
+    elif isinstance(fs, FileSystem):
       fs = fs.configs
+    elif not isinstance(fs, FileStore):
+      raise Exception("Unsupporting persistent storage")
 
     if filename_only is None:
       filename_only = get_experiment_code(self)
@@ -221,19 +217,48 @@ class MLExperimentConfig(dict):
     # Backwards compatibility 0.6.0
     return self.save()
   # --------------------------------------------------------------------------------------
-  def load_config(self, fs, filename_only):
-    if isinstance(fs, FileSystem):
+  def load_from_json(self, filename=None, must_exist=False):
+    if filename is None:
+      filename = self.filename
+
+    # reading the data from the file
+    if os.path.exists(filename):
+      with open(filename) as oFile:
+        sConfig = oFile.read()
+        self.setDefaults()
+        dConfigDict = json.loads(sConfig)
+
+      for sKey in dConfigDict.keys():
+        self._kv[sKey] = dConfigDict[sKey]
+    else:
+      if must_exist:
+        raise Exception("Experiment configuration file %s is not found." % filename)
+    return self
+  # --------------------------------------------------------------------------------------
+  def load(self, fs=None, filename_only=None):
+    if fs is None:
+      fs = mlsys.filesys.configs
+    elif isinstance(fs, FileSystem):
       fs = fs.configs
+    elif not isinstance(fs, FileStore):
+      raise Exception("Unsupporting persistent storage")
+
+    if filename_only is None:
+      filename_only = get_experiment_code(self)
 
     sFileName = fs.file(filename_only + ".json")
-    return self.load(sFileName)
+    return self.load_from_json(sFileName)
+  # --------------------------------------------------------------------------------------
+  def load_config(self, fs, filename_only):
+    # Backwards compatibility 0.6.0
+    return self.load(fs, filename_only)
   # --------------------------------------------------------------------------------------
   def setDefaults(self):
     pass
   # --------------------------------------------------------------------------------------
   def __str__(self)->str:
     sResult = ""
-    for sKey in self.keys():
+    for sKey in self._kv.keys():
       sResult += f'  {sKey}: \"{self[sKey]}\",\n'
 
     sResult = "{\n" + sResult + "}"

radnn/images/__init__.py

@@ -0,0 +1,2 @@
+from .colors import image_rgb_to_hslb, image_rgb_to_hif
+from .image_processor import ImageProcessor

radnn/images/colors.py

@@ -0,0 +1,28 @@
+import numpy as np
+from matplotlib import colors
+from matplotlib.colors import to_rgba
+
+LUMA_W = np.asarray([0.29889531 / 255.0, 0.58662247 / 255.0, 0.11448223 / 255.0], dtype=np.float32)
+
+# ------------------------------------------------------------------------------------
+# Analyse the image to H,S,L, B
+def image_rgb_to_hslb(image):
+  '''
+  Analyzes an image and returns the hue, saturation, luma and brightness
+  :param image:
+  :return: image in HSLB format
+  '''
+  img_hsv = colors.rgb_to_hsv(image / 255.0)
+  luma = np.dot(image, LUMA_W.T)
+  return np.stack([img_hsv[..., 0], img_hsv[..., 1], luma, img_hsv[..., 2]], axis=-1).astype(np.float32)
+# ------------------------------------------------------------------------------------
+def image_rgb_to_hif(image):
+  oImageHSLB = image_rgb_to_hslb(image)
+
+  img_hsv = colors.rgb_to_hsv(image / 255.0)
+  luma = np.dot(image, LUMA_W.T)
+  return np.stack([img_hsv[..., 0], img_hsv[..., 1], luma], axis=-1).astype(np.float32)
+# ------------------------------------------------------------------------------------
+def color(name):
+  return tuple([int(x*255) for x in to_rgba(name)])
+# ------------------------------------------------------------------------------------