radnn 0.0.8__py3-none-any.whl → 0.1.0__py3-none-any.whl

radnn/data/sample_set.py

@@ -22,30 +22,84 @@
 # SOFTWARE.
 
 # ......................................................................................
-from .dataset_base import DataSetBase
-from .subset_type import SubsetType
+import numpy as np
+import pandas as pd
+from .errors import *
+from sklearn.utils.class_weight import compute_class_weight
+from .sample_set_kind import SampleSetKindInfo
 
+from radnn.core import RequiredLibs
+oReqs = RequiredLibs()
+if oReqs.is_torch_installed:
+  # [TODO] Additional coupling to torch (+)->MANOLO
+  import platform
+  import torch
+  from torch.utils.data import DataLoader
 
 
 class SampleSet(object):
+  LOADER_NUM_WORKERS = 8
+
   # --------------------------------------------------------------------------------------------------------------------
-  def __init__(self, subset_type="custom", has_ids=False):
-    self.subset_type: SubsetType = SubsetType(subset_type)
-    self.parent_dataset = None
+  def __init__(self, parent_dataset, subset_kind: str, **kwargs):
+    '''
+    Create a sample subset on a dataset
+
+    :param parent_dataset: The parent dataset that implements the composition relationship.
+    :param subset_kind: The kind of the subset. Can be one of:
+            training/train/ts  for the known samples that are presented to an algorithm
+            validation/val/vs  for the known samples that are not presented to an algorithm, used to validate during its execution
+            testing/test/ut    for the unknown samples that are not given to the researcher that develops the algorithm
+
+    :**param sample_file_records: A Pandas dataframe that contains two columns: 1) The list of sample file absolute paths and 2) Correspondng labels.
+    :**param is_classification: True if the dataset used for a classification task.
+    :**param batch_size: The count of samples in a minibatch.
+    :**param sample_transform_augment: A callback method that performs a single sample preprocessing task.
+    '''
+    self.parent_dataset = parent_dataset
+    self.info = SampleSetKindInfo(subset_kind)
+    self.info.is_classification = kwargs.get("is_classification", False)
+    self.batch_size = kwargs.get("batch_size", None)
+    self.transform_augment = kwargs.get("sample_transform_augment", None)
+    self._sample_count                      = 0
 
-    self.has_ids = has_ids
+    self.ids                                = None
+    self.samples: list|np.ndarray|None      = kwargs.get("samples", None)
+    if (self.samples is not None) and isinstance(self.samples, list):
+      self.samples = np.array(self.samples, np.float32)
+    self._sample_count = self.samples.shape[0]
+    self.labels: list|np.ndarray|None       = kwargs.get("labels", None)
 
-    self.ids    = None
-    self.samples = None
-    self.sample_count = None
-    self.labels = None
+    self.files                              = None
+    self._sample_directory: pd.DataFrame | None  = kwargs.get("sample_file_records", None)
+    if self._sample_directory is not None:
+      self.files = self._sample_directory.iloc[:, 0].to_list()
+      self._sample_count = len(self.files)
+      self.labels       = self._sample_directory.iloc[:, 1].to_list()
 
+    assert self._sample_count > 0, ERR_SUBSET_MUST_HAVE_SAMPLES
     self._step = 1
     self._iter_start_pos = 0
     self._iter_counter = 0
+    self._are_samples_in_memory = self.samples is not None
+
+    self.has_sample_ids = self.ids is not None
 
-    self.feed = None
+    self.minibatch_count = self._sample_count
+    if self.batch_size is not None:
+      self.loader = DataLoader(self, batch_size=self.batch_size, shuffle=self.info.must_shuffle,
+                               num_workers=self.LOADER_NUM_WORKERS)
+      self.minibatch_count = len(self.loader)
+    else:
+      self.loader = DataLoader(self, shuffle=self.info.must_shuffle, num_workers=self.LOADER_NUM_WORKERS)
 
+    if self.info.is_classification:
+      self.info.class_indices = np.sort(np.unique(self.labels))
+      self.info.class_weights = compute_class_weight(class_weight='balanced', classes=self.info.class_indices, y=np.array(self.labels))
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def sample_count(self):
+    return self._sample_count
   # --------------------------------------------------------------------------------------------------------------------
   @property
   def has_labels(self):
@@ -53,114 +107,173 @@ class SampleSet(object):
   # --------------------------------------------------------------------------------------------------------------------
   @property
   def data_tuple(self):
-    if self.has_ids:
-      if self.labels is None:
-        return (self.ids, self.samples)
+    '''
+    :return: For in-memory samples: A tuple with the samples, their respective labels and their ids.
+             For file-based dataiterators: None
+    '''
+    if self._are_samples_in_memory:
+      if self.has_sample_ids:
+        if self.labels is None:
+          return (self.ids, self.samples)
+        else:
+          return (self.ids, self.samples, self.labels)
       else:
-        return (self.ids, self.samples, self.labels)
+        if self.labels is None:
+          return self.samples
+        else:
+          return (self.ids, self.samples, self.labels)
     else:
-      if self.labels is None:
-        return self.samples
-      else:
-        return (self.ids, self.samples, self.labels)
+      return None
   # --------------------------------------------------------------------------------------------------------------------
-  def subset_of(self, parent_dataset: DataSetBase):
-    self.parent_dataset = parent_dataset
-    if self.parent_dataset is not None:
-      if self.subset_type.is_training_set:
-        if self.parent_dataset.ts_samples is not None:
-          self.parent_dataset.ts = self
-          self.ids = self.parent_dataset.ts_sample_ids
-          self.samples = self.parent_dataset.ts_samples
-          self.sample_count = self.parent_dataset.ts_sample_count
-          self.labels = self.parent_dataset.ts_labels
-      elif self.subset_type.is_validation_set:
-        if self.parent_dataset.vs_samples is not None:
-          self.parent_dataset.vs = self
-          self.ids = self.parent_dataset.vs_sample_ids
-          self.samples = self.parent_dataset.vs_samples
-          self.sample_count = self.parent_dataset.vs_sample_count
-          self.labels = self.parent_dataset.vs_labels
-      elif self.subset_type.is_unknown_test_set:
-        if self.parent_dataset.ut_samples is not None:
-          self.parent_dataset.ut = self
-          self.ids = self.parent_dataset.ut_sample_ids
-          self.samples = self.parent_dataset.ut_samples
-          self.sample_count = self.parent_dataset.ut_sample_count
-          self.labels = self.parent_dataset.ut_labels
-
-      self.has_ids = self.ids is not None
-  # --------------------------------------------------------------------------------------------------------------------
-  '''
-  def create_feed(self, has_ids=False):
-    self.has_ids = has_ids
-    if is_tensorflow_installed:
-      import tensorflow as tf
-
-      if has_ids:
-        self.feed = tf.data.Dataset.from_tensor_slices((self.ids, self.samples, self.labels))
-      else:
-        self.feed = tf.data.Dataset.from_tensor_slices((self.samples, self.labels))
-
-      self.feed = self.feed.map(preprocess_tf, num_parallel_calls=8)
-
-      if (self.subset_type == "training") or (self.subset_type == "train") or (self.subset_type == "ts"):
-  # -----------------------------------------------------------------------------------
-  def preprocess_tf(self, sample_pack):
-
-    import tensorflow as tf
-
-    if self.has_ids:
-      nId, nSample, nLabel = sample_pack
+  def __len__(self):
+    return self._sample_count
+  # --------------------------------------------------------------------------------------------------------------------
+  def __getitem__(self, index):
+    if self._are_samples_in_memory:
+      tSample = self.do_load_sample(index)
     else:
-      nSample, nLabel = sample_pack
-
-    tImage = tf.cast(p_tImageInVS, tf.float32)  # //[BF] overflow of standardization
-    tNormalizedImage = self.normalizeImage(tImage)
-
-    tTargetOneHot = tf.one_hot(p_tLabelInVS, self.ClassCount)
-
-    return tNormalizedImage, tTargetOneHot
-  '''
+      tSample = self.do_load_sample_from_file(index)
+    if self.transform_augment is not None:
+      tSample = self.transform_augment(tSample)
 
+    if self.ids is not None:
+      if self.labels is not None:
+        # Supervised with ids
+        return self.ids[index], tSample, self.do_load_target(index)
+      else:
+        # Unsupervised with ids
+        return self.ids[index], tSample
+    else:
+      if self.labels is not None:
+        # Supervised
+        return tSample, self.do_load_target(index)
+      else:
+        # Unsupervised
+        return tSample
   # --------------------------------------------------------------------------------------------------------------------
   def __iter__(self):
     self._iter_counter = 0
-    if self.ids is not None:
-      if self.labels is not None:
-        yield from self._generator_for_supervised_with_ids()
+
+    if self._are_samples_in_memory:
+      if self.ids is not None:
+        if self.labels is not None:
+          yield from self._numpy_generator_for_supervised_with_ids()
+        else:
+          yield from self._numpy_generator_for_unsupervised_with_ids()
       else:
-        yield from self._generator_for_unsupervised_with_ids()
+        if self.labels is not None:
+          yield from self._numpy_generator_for_supervised()
+        else:
+          yield from self._numpy_generator_for_unsupervised()
     else:
       if self.labels is not None:
-        yield from self._generator_for_supervised()
+        yield from self._file_generator_for_supervised()
       else:
-        yield from self._generator_for_unsupervised()
+        yield from self._file_generator_for_unsupervised()
+  # --------------------------------------------------------------------------------------------------------------------
+  def do_load_sample_from_file(self, index):
+    '''
+    Override this method with the custom code of loading a single sample file
+    :param index: The index of the sample in the sample file list
+    '''
+    pass
+  # --------------------------------------------------------------------------------------------------------------------
+  def do_load_sample(self, index):
+    '''
+    Override this method with the custom code of getting a single sample file from a numpy array
+    :param index: The index of the sample in the sample subset
+    :return:
+    '''
+    return torch.tensor(self.samples[index, ...], dtype=torch.float32)
+  # --------------------------------------------------------------------------------------------------------------------
+  def do_load_target(self, index):
+    '''
+    Override this method with some custom code that loads sample label(s) or transforms them into a training target (e.g. one-hot encoding)
+    :param index: The index of the sample in the sample subset
+    '''
+    return torch.tensor(self.labels[index, ...], dtype=torch.long)
+  # --------------------------------------------------------------------------------------------------------------------
+  def _file_generator_for_supervised(self):
+    nIndex = self._iter_start_pos
+    while self._iter_counter < self.sample_count:
+      tSample = self.do_load_sample_from_file(nIndex)
+      if self.transform_augment is not None:
+        tSample = self.transform_augment(tSample)
+      yield (tSample, self.do_load_target(nIndex))
+      nIndex += self._step
+  # --------------------------------------------------------------------------------------------------------------------
+  def _file_generator_for_unsupervised(self):
+    nIndex = self._iter_start_pos
+    while self._iter_counter < self.sample_count:
+      tSample = self.do_load_sample_from_file(nIndex)
+      if self.transform_augment is not None:
+        tSample = self.transform_augment(tSample)
+      yield (tSample)
+      nIndex += self._step
   # --------------------------------------------------------------------------------------------------------------------
-  def _generator_for_supervised(self):
+  def _numpy_generator_for_supervised(self):
     nIndex = self._iter_start_pos
     while self._iter_counter < self.sample_count:
-      yield (self.samples[nIndex, ...], self.labels[nIndex, ...])
+      tSample = self.do_load_sample(nIndex)
+      if self.transform_augment is not None:
+        tSample = self.transform_augment(tSample)
+      yield (tSample, self.do_load_target(nIndex))
       nIndex += self._step
   # --------------------------------------------------------------------------------------------------------------------
-  def _generator_for_unsupervised(self):
+  def _numpy_generator_for_supervised_with_ids(self):
     nIndex = self._iter_start_pos
     while self._iter_counter < self.sample_count:
-      yield self.samples[nIndex, ...]
+      tSample = self.do_load_sample(nIndex)
+      if self.transform_augment is not None:
+        tSample = self.transform_augment(tSample)
+      yield (self.ids[nIndex], tSample, self.do_load_target(nIndex))
       nIndex += self._step
   # --------------------------------------------------------------------------------------------------------------------
-  def _generator_for_supervised_with_ids(self):
+  def _numpy_generator_for_unsupervised(self):
     nIndex = self._iter_start_pos
     while self._iter_counter < self.sample_count:
-      yield (self.ids[nIndex], self.samples[nIndex, ...], self.labels[nIndex, ...])
+      tSample = self.do_load_sample(nIndex)
+      if self.transform_augment is not None:
+        tSample = self.transform_augment(tSample)
+      yield tSample
       nIndex += self._step
   # --------------------------------------------------------------------------------------------------------------------
-  def _generator_for_unsupervised_with_ids(self):
+  def _numpy_generator_for_unsupervised_with_ids(self):
     nIndex = self._iter_start_pos
     while self._iter_counter < self.sample_count:
-      yield (self.ids[nIndex], self.samples[nIndex, ...])
+      tSample = self.do_load_sample(nIndex)
+      if self.transform_augment is not None:
+        tSample = self.transform_augment(tSample)
+      yield (self.ids[nIndex], tSample)
       nIndex += self._step
   # --------------------------------------------------------------------------------------------------------------------
+  def print_info(self):
+    sDescription = self.info.kind_description
+    sDescription = sDescription[0].upper() + sDescription[1:]
+    sMinibatches = ""
+    if self.minibatch_count is not None:
+      sMinibatches = f" minbatches: {self.minibatch_count}"
+    if (self.samples is not None) and isinstance(self.samples, np.ndarray):
+      print(f"  |__ {sDescription} set samples: {self.sample_count}   shape: {self.samples.shape}{sMinibatches}")
+      print(f"          |__ {sDescription} set labels: {self.sample_count}   shape: {self.labels.shape}")
+    else:
+      print(f"  |__ {sDescription} set samples: {self.sample_count} {sMinibatches}")
 
+    if (self.labels is not None) and isinstance(self.labels, np.ndarray):
+        print(f"      |__ Labels: {self.sample_count}   shape:{self.labels.shape}")
+    else:
+      print(f"      |__ Labels: {self.sample_count}")
+  # --------------------------------------------------------------------------------------------------------------------
+  def __str__(self):
+    return f"{self.info.kind_description} samples:{self.sample_count} minibatches:{self.minibatch_count}"
+  # --------------------------------------------------------------------------------------------------------------------
+  def __repr__(self):
+    return self.__str__()
+  # --------------------------------------------------------------------------------------------------------------------
 
 
+# Fix for torch on Windows
+if platform.system().lower() == "windows":
+  SampleSet.LOADER_NUM_WORKERS = 0
+else:
+  SampleSet.LOADER_NUM_WORKERS = 8

radnn/data/sample_set_kind.py

@@ -0,0 +1,126 @@
+# ......................................................................................
+# MIT License
+
+# Copyright (c) 2019-2025 Pantelis I. Kaplanoglou
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+# ......................................................................................
+from enum import Enum
+
+
+# ======================================================================================================================
+class SampleSetKind(Enum):
+  TRAINING_SET = 0
+  VALIDATION_SET = 1
+  UNKNOWN_TEST_SET = 2
+
+
+# ======================================================================================================================
+class SampleSetKindInfo(dict):
+  # --------------------------------------------------------------------------------------------------------------------
+  def __init__(self, kind_str: str|None=None, kind: int | None=None, **kwargs):
+    super().__init__(**kwargs)
+    self._kind: int | None = kind
+    self._kind_str: str | None = None
+    if kind_str is not None:
+      self._kind_str = kind_str.lower()
+
+    # Invoke the property getters
+    self["kind"]      = self.kind
+    self["kind_str"]  = self.kind_description
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def kind_description(self):
+    if (self._kind_str is None):
+      nKind = self.get("type", -1)
+      if nKind == SampleSetKind.TRAINING_SET.value:
+        self._kind_str = "training"
+      elif nKind == SampleSetKind.value:
+        self._kind_str = "validation"
+      elif nKind == SampleSetKind.value:
+        self._kind_str = "training"
+    return self._kind_str
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def kind(self):
+    if (self._kind is None):
+      nKind = -1 # Unknown
+      if self.is_training_set:
+        nKind = SampleSetKind.TRAINING_SET.value
+      elif self.is_validation_set:
+        nKind = SampleSetKind.VALIDATION_SET.value
+      elif self.is_unknown_test_set:
+        nKind = SampleSetKind.UNKNOWN_TEST_SET.value
+      self._kind = nKind
+    return self._kind
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def must_shuffle(self):
+    return self.kind == SampleSetKind.TRAINING_SET.value
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def is_training_set(self):
+    return (self._kind_str == "training") or (self._kind_str == "train") or (self._kind_str == "ts")
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def is_validation_set(self):
+    return (self._kind_str == "validation") or (self._kind_str == "val") or (self._kind_str == "vs")
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def is_unknown_test_set(self):
+    return (self._kind_str == "testing") or (self._kind_str == "test") or (self._kind_str == "ut")
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def is_classification(self):
+    return self["Task"] == "classification"
+  # --------------------------------------------------------------------------------------------------------------------
+  @is_classification.setter
+  def is_classification(self, value):
+    if value:
+      self["Task"] = "classification"
+      self["Classes.Count"] = None
+      self["Classes.Indices"] = None
+      self["Classes.Weights"] = None
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def class_count(self):
+    return self["Classes.Count"]
+  # --------------------------------------------------------------------------------------------------------------------
+  @class_count.setter
+  def class_count(self, value):
+    self["Classes.Count"] = value
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def class_indices(self):
+    return self["Classes.Indices"]
+  # --------------------------------------------------------------------------------------------------------------------
+  @class_indices.setter
+  def class_indices(self, value):
+    self["Classes.Indices"] = value
+  # --------------------------------------------------------------------------------------------------------------------
+  @property
+  def class_weights(self):
+    return self["Classes.Weights"]
+  # --------------------------------------------------------------------------------------------------------------------
+  @class_weights.setter
+  def class_weights(self, value):
+    self["Classes.Weights"] = value
+  # --------------------------------------------------------------------------------------------------------------------

radnn/data/sequence_dataset.py

@@ -23,7 +23,9 @@
 
 # ......................................................................................
 import numpy as np
-from .dataset_base import DataSetBase
+
+from .sample_set import SampleSet
+from .dataset_base import DataSetBase, DataSetCallbacks
 
 
 # ----------------------------------------------------------------------------------------------------------------------
@@ -54,37 +56,26 @@ def generate_sequence_clips(samples, labels, window_size, stride, is_padding_zer
 
 
 
-class SequenceDataset(DataSetBase):
-  # --------------------------------------------------------------------------------------------------------------------
-  def __init__(self, name,  fs, clip_window_size=None, clip_stride=None, is_padding_zeros=False, random_seed=None, is_classification=True):
-    super(SequenceDataset, self).__init__(name, fs, random_seed, is_classification)
-    self.clip_window_size = clip_window_size
-    self.clip_stride = clip_stride
-    self.is_padding_zeros = is_padding_zeros
-    self.card["clips.window_size"] = self.clip_window_size
-    self.card["clips.stride"] = self.clip_stride
-    self.card["clips.is_padding_zeros"] = self.is_padding_zeros
+class SequenceDataSet(DataSetBase):
   # --------------------------------------------------------------------------------------------------------------------
-  @property
-  def ts_sequence_clips(self):
-    return generate_sequence_clips(self.ts_samples, self.ts_labels, self.clip_window_size, self.clip_stride,
-                                   self.is_padding_zeros)
+  def __init__(self, name: str, variant: str|None=None, file_store=None, random_seed: int | None=None, callbacks: DataSetCallbacks | None = None):
+    super().__init__(name, variant, file_store, random_seed, callbacks)
+
+    #self, name,  fs, clip_window_size=None, clip_stride=None, is_padding_zeros=False, random_seed=None, is_classification=True):
+    #super(SequenceDataSet, self).__init__(name, fs, random_seed, is_classification)
+
+    self.clip_window_size: int|None = None
+    self.clip_stride: int|None = None
+    self.is_padding_zeros: bool|None = None
   # --------------------------------------------------------------------------------------------------------------------
-  @property
-  def vs_sequence_clips(self):
-    if self.vs_samples is not None:
-      return generate_sequence_clips(self.vs_samples, self.vs_labels, self.clip_window_size, self.clip_stride,
-                                     self.is_padding_zeros)
-    else:
-      return None
+  def do_read_hyperparams(self):
+    self.clip_window_size = self.hparams.get("Data.Clip.WindowSize", None)
+    self.clip_stride = self.hparams.get("Data.Clip.Stride", None)
+    self.is_padding_zeros = self.hparams.get("Data.Clip.IsPaddingZeroes", None)
   # --------------------------------------------------------------------------------------------------------------------
-  @property
-  def ut_sequence_clips(self):
-    if self.ut_samples is not None:
-      return generate_sequence_clips(self.ut_samples, self.ut_labels, self.clip_window_size, self.clip_stride,
-                                     self.is_padding_zeros)
-    else:
-      return None
+  def generate_clips(self, subset: SampleSet):
+    return generate_sequence_clips(self, subset.samples, subset.labels,
+                                   self.clip_window_size, self.clip_stride, self.is_padding_zeros)
   # --------------------------------------------------------------------------------------------------------------------
   def convert_samples_to_clips(self, clip_window_size=None, clip_stride=None, is_padding_zeros=False):
     if clip_window_size is not None:
@@ -94,14 +85,18 @@ class SequenceDataset(DataSetBase):
     if is_padding_zeros and (not self.is_padding_zeros):
       self.is_padding_zeros = is_padding_zeros
 
+    # TODO: Reinstate dataset card
+    '''
     self.card["clips.window_size"] = self.clip_window_size
     self.card["clips.stride"] = self.clip_stride
     self.card["clips.is_padding_zeros"] = self.is_padding_zeros
+    '''
 
     # Create training set clips
+    oSubsets = [self.ts, self.vs, self.us]
     nClips = []
     nClipLabels = []
-    for (nClip, nClipLabel) in self.ts_sequence_clips:
+    for (nClip, nClipLabel) in self.generate_clips():
       nClips.append(nClip)
       nClipLabels.append(nClipLabel)
     nClips = np.asarray(nClips)

radnn/data/structs/__init__.py

@@ -0,0 +1 @@
+from .tree import Tree, TreeNode, TreeNodeList, TreeNodeQueue