radnn 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

radnn/__init__.py

@@ -9,7 +9,9 @@
 # Version 0.1.1    [2025-01-08]
 # Version 0.1.4    [2025-01-26]
 # Version 0.1.5    [2025-02-02]
-__version__ = "0.1.5"
+# Version 0.1.6    [2025-02-03]
+# Version 0.1.7    [2025-02-04]
+__version__ = "0.1.7"
 
 from .system import FileStore, FileSystem
 from .ml_system import MLSystem

radnn/data/custom_data_set.py

@@ -23,7 +23,7 @@
 
 # ......................................................................................
 
-from sklearn.model_selection import train_test_split  # import a standalone procedure function from the pacakge
+from sklearn.model_selection import train_test_split  # import a standalone procedure toyfunction from the pacakge
 from sklearn.preprocessing import StandardScaler, MinMaxScaler
 from radnn import mlsys
 from radnn.data.sample_set_simple import SampleSet

radnn/data/dataset_base.py

@@ -26,12 +26,15 @@ import os
 import numpy as np
 import pandas as pd
 from abc import ABC, abstractmethod
-from .sample_set import SampleSet
+from .sample_set_simple import SampleSet
 from .sample_set_kind import SampleSetKind
 from .sample_preprocessor import SamplePreprocessor, VoidPreprocessor
 from .errors import *
 from radnn import FileStore
 from radnn import mlsys
+from .constants import DataPreprocessingKind
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import MinMaxScaler, StandardScaler
 
 # ======================================================================================================================
 class DataSetCallbacks(object):
@@ -73,7 +76,7 @@ class DataSetBase(ABC):
     self.feature_count = None
     self.class_count = None
     self.sample_count = None
-    
+
     self.callbacks: DataSetCallbacks = callbacks
     
     self.hprm: dict | None = None
@@ -81,8 +84,39 @@ class DataSetBase(ABC):
     self.vs: SampleSet | None = None
     self.us: SampleSet | None = None
     self.preprocessor: SamplePreprocessor = VoidPreprocessor(self)
+    
+    self.is_split = False
     # ................................................................
 
+  # --------------------------------------------------------------------------------------
+  def split(self, validation_samples_pc=0.10,
+            preprocessing: DataPreprocessingKind | None = DataPreprocessingKind.STANDARDIZE,
+            random_seed: int=2021):
+    
+    nTS_Samples, nVS_Samples, nTS_Labels, nVS_Labels = train_test_split(self.all_samples, self.all_labels,
+                                                                        test_size=validation_samples_pc,
+                                                                        random_state=random_seed)
+    if preprocessing == DataPreprocessingKind.MIN_MAX_NORMALIZE:
+      self.preprocessor = MinMaxScaler().fit(nTS_Samples)
+    elif preprocessing == DataPreprocessingKind.STANDARDIZE:
+      self.preprocessor = StandardScaler().fit(nTS_Samples)
+    else:
+      self.preprocessor = None
+
+    if self.preprocessor is not None:
+      nTS_Samples = self.preprocessor.transform(nTS_Samples)
+      nVS_Samples = self.preprocessor.transform(nVS_Samples)
+
+    # (Re)creating the subsets of the dataset after the splits have been created
+    self.ts = SampleSet(self, nTS_Samples, nTS_Labels, kind=SampleSetKind.TRAINING_SET)
+    if preprocessing == DataPreprocessingKind.STANDARDIZE:
+      self.ts.mean = self.preprocessor.mean_
+      self.ts.std = self.preprocessor.scale_
+
+    self.vs = SampleSet(self, nVS_Samples, nVS_Labels, kind=SampleSetKind.VALIDATION_SET)
+    
+    self.is_split = True
+    return self
   # --------------------------------------------------------------------------------------------------------------------
   @property
   def dataset_code(self):
@@ -115,11 +149,32 @@ class DataSetBase(ABC):
   def load_data(self):
     pass # Must implement
   # --------------------------------------------------------------------------------------------------------------------
-  def load_cache(self):
-    pass # Optionally override
+  def load_cache(self, is_vector_samples=True, is_last_axis_features=True):
+    nSuffix = ""
+    if is_vector_samples:
+      nSuffix = "-vec"
+    elif not is_last_axis_features:
+      nSuffix = "-torch"
+
+    nTSSamples = self.fs.obj.load(f"{self.dataset_code}-TS-Samples{nSuffix}.pkl")
+    nVSSamples = self.fs.obj.load(f"{self.dataset_code}-VS-Samples{nSuffix}.pkl")
+
+    nTSLabels = self.fs.obj.load(f"{self.dataset_code}-TS-Labels{nSuffix}.pkl")
+    nVSLabels = self.fs.obj.load(f"{self.dataset_code}-VS-Labels{nSuffix}.pkl")
+
+    return nTSSamples, nVSSamples, nTSLabels, nVSLabels
   # --------------------------------------------------------------------------------------------------------------------
-  def save_cache(self):
-    pass # Optionally override
+  def save_cache(self, ts_samples, vs_samples, ts_labels, vs_labels, is_vector_samples=True, is_last_axis_features=True):
+    nSuffix = ""
+    if is_vector_samples:
+      nSuffix = "-vec"
+    elif not is_last_axis_features:
+      nSuffix = "-torch"
+    self.fs.obj.save(ts_samples, f"{self.dataset_code}-TS-Samples{nSuffix}.pkl")
+    self.fs.obj.save(vs_samples, f"{self.dataset_code}-VS-Samples{nSuffix}.pkl")
+
+    self.fs.obj.save(ts_labels, f"{self.dataset_code}-TS-Labels{nSuffix}.pkl")
+    self.fs.obj.save(vs_labels, f"{self.dataset_code}-VS-Labels{nSuffix}.pkl")
   # --------------------------------------------------------------------------------------------------------------------
   def prepare(self, hyperparams: dict | None = None):
     self.hprm = hyperparams
@@ -134,23 +189,29 @@ class DataSetBase(ABC):
         self.callbacks.lazy_download(self.fs)
     
     if (self.random_seed is not None):
-      assert self.callbacks is not None, ERR_NO_CALLBACKS
-      assert self.callbacks._random_seed is not None, ERR_DS_NO_RANDOM_SEED_INITIALIZER_CALLBACK
-      self.callbacks.random_seed(self.random_seed)
-    
+      bIsInitRandomSeed = False
+      if self.callbacks is not None:
+        if self.callbacks._random_seed is not None:
+          self.callbacks.random_seed(self.random_seed)
+          bIsInitRandomSeed = True
+      if not bIsInitRandomSeed:
+        mlsys.random_seed_all(self.random_seed)
+
     self.ts = None
     self.vs = None
     self.us = None
-    # VIRTUAL CALL:  Imports the data from the source local/remote filestore to the local cache.
+
+    # VIRTUAL CALL:  Imports the dataset from the source local/remote filestore to the local cache.
     self.load_data()
     
-    assert self.ts is not None, ERR_DS_SUBSET_MUST_HAVE_TS
-    assert self.ts.kind == SampleSetKind.TRAINING_SET, ERR_DS_SUBSET_INVALID_SETUP
-    if self.vs is not None:
+    if self.is_split:
+      assert self.ts is not None, ERR_DS_SUBSET_MUST_HAVE_TS
       assert self.ts.kind == SampleSetKind.TRAINING_SET, ERR_DS_SUBSET_INVALID_SETUP
+      if self.vs is not None:
+        assert self.vs.kind == SampleSetKind.VALIDATION_SET, ERR_DS_SUBSET_INVALID_SETUP
+        
     if self.us is not None:
-      assert self.ts.kind == SampleSetKind.TRAINING_SET, ERR_DS_SUBSET_INVALID_SETUP
-
+      assert self.us.kind == SampleSetKind.UNKNOWN_TEST_SET, ERR_DS_SUBSET_INVALID_SETUP
       
     return self
   # --------------------------------------------------------------------------------------------------------------------

radnn/data/dataset_base_legacy.py

@@ -25,7 +25,7 @@
 
 
 import numpy as np
-from sklearn.model_selection import train_test_split  # import a standalone procedure function from the pacakge
+from sklearn.model_selection import train_test_split  # import a standalone procedure toyfunction from the pacakge
 
 
 # =========================================================================================================================

radnn/data/errors.py

@@ -28,7 +28,7 @@ ERR_MLSYS_FILESYS_NOT_INITIALIZED = "The filesystem for the Machine Learning hos
 
 ERR_NO_CALLBACKS = "You should assign callbacks to the dataset perform proper random seed initialization for your framework."
 ERR_DS_NO_RANDOM_SEED_INITIALIZER_CALLBACK = "Callback method for random seed initialization has not been defined."
-ERR_DS_CALLBACKS_NO_LAZY_DOWNLOADER = "Callback method for downloading the data has not been defined."
+ERR_DS_CALLBACKS_NO_LAZY_DOWNLOADER = "Callback method for downloading the dataset has not been defined."
 
 ERR_DS_SUBSET_MUST_HAVE_TS = "A dataset must have at least a training subset."
 ERR_DS_SUBSET_INVALID_SETUP = "Invalid sample subset setup. Please use one of the valid kinds: 'training/train/ts', 'validation/val/vs', 'testing/test/us'."

radnn/data/sample_preprocessor.py

@@ -47,6 +47,9 @@ class SamplePreprocessor(ABC):
     pass
   # --------------------------------------------------------------------------------------------------------------------
 
+
+
+
 # ======================================================================================================================
 class VoidPreprocessor(SamplePreprocessor):
   # --------------------------------------------------------------------------------------------------------------------

radnn/data/sample_set_simple.py

@@ -11,7 +11,10 @@ class SampleSet(object):
     if self.ids is None:
       self.ids = np.arange(len(self.samples)) + 1
     self.kind: SampleSetKind = kind
-    
+
+    self.mean = None
+    self.std = None
+
     self.loader = None
     self._sample_count = None
     self._minibatch_count = None

radnn/experiment/ml_experiment.py

@@ -49,7 +49,7 @@ from .ml_experiment_store import MLExperimentStore
 
 
 # --------------------------------------------------------------------------------------------------------------------
-# Define a custom sorting function
+# Define a custom sorting toyfunction
 def _sort_by_last_path_element(folder):
   # Split the path into its components
   components = folder.split(os.pathsep)
@@ -226,7 +226,7 @@ class MLExperiment:
       self.model_fs.log_fs.json.save(dTiming, f"timing_info_{self._end_train_time.strftime('%Y-%m-%dT%H%M%S')}.json",
                              is_sorted_keys=False)
 
-    # //TODO: Keep cost function names and other learning parameters for evaluation
+    # //TODO: Keep cost toyfunction names and other learning parameters for evaluation
   # --------------------------------------------------------------------------------------------------------------------
   def load(self, use_last_checkpoint=False, model_root_folder=None):
     self._currentModelFolder = self.model_fs.base_folder

radnn/experiment/ml_experiment_log.py

@@ -1,49 +1,55 @@
 import numpy as np
 from radnn.system.filesystem import FileStore
+
+
 class MLExperimentLog:
   # --------------------------------------------------------------------------------------------------------------------
-  def __init__(self, filename: str, experiment_info: dict | None = None):
+  def __init__(self, filename: str, experiment_info: dict | None = None, is_autoinit: bool = False):
     self.filename = filename
     if experiment_info is None:
       experiment_info = {}
     self.experiment_info = experiment_info
-    self.logs = { "experiment": experiment_info,
-                  "epoch": [],
-                  "epoch_time": [],
-                  "train_step_loss": [],
-                  "train_step_accuracy": [],
-                  "train_loss": [],
-                  "train_accuracy": [],
-                  "val_loss": [],
-                  "val_accuracy": [],
-                  "learning_rate": []
-            }
+    self.is_autoinit = is_autoinit
+    if self.is_autoinit:
+      self.logs = {"experiment": experiment_info, "epoch": [], "epoch_time": []}
+    else:
+      self.logs = {"experiment": experiment_info, "epoch": [], "epoch_time": [],
+                   "train_step_loss": [],
+                   "train_step_accuracy": [],
+                   "train_loss": [],
+                   "train_accuracy": [],
+                   "val_loss": [],
+                   "val_accuracy": [],
+                   "learning_rate": []}
+  
   # --------------------------------------------------------------------------------------------------------------------
-  def assign_series(self, is_autoinit=False, **kwargs):
-    if is_autoinit:
+  def assign_series(self, **kwargs):
+    if self.is_autoinit:
       for key, value in kwargs.items():
         if key not in self.logs:
           self.logs[key] = []
-      
+    
     for key, value in kwargs.items():
       self.logs[key] = value
+  
   # --------------------------------------------------------------------------------------------------------------------
-  def append(self, is_autoinit=False, **kwargs):
-    if is_autoinit:
+  def append(self, **kwargs):
+    if self.is_autoinit:
       for key, value in kwargs.items():
         if key not in self.logs:
           self.logs[key] = []
-
+    
     for key, value in kwargs.items():
       self.logs[key].append(value)
     return self
+  
   # --------------------------------------------------------------------------------------------------------------------
   def load(self, experiment_fs: FileStore):
     self.logs = experiment_fs.json.load(self.filename)
     return self
+  
   # --------------------------------------------------------------------------------------------------------------------
   def save(self, experiment_fs: FileStore):
     experiment_fs.json.save(self.logs, self.filename)
     return self
   # --------------------------------------------------------------------------------------------------------------------
-    

radnn/system/hosts/windows_host.py

@@ -72,7 +72,7 @@ class WindowsHost(object):
   def set_windows_sleep_resolution(cls, msecs=1):
     """
     Requests a minimum resolution for periodic timers. This increases accuracy
-    for the waiting interval of the time.sleep function
+    for the waiting interval of the time.sleep toyfunction
     """
     oWinMM = ctypes.WinDLL('oWinMM')
     oWinMM.timeBeginPeriod(msecs)

{radnn-0.1.5.dist-info → radnn-0.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: radnn
-Version: 0.1.5
+Version: 0.1.7
 Summary: Rapid Deep Neural Networks
 Author-email: "Pantelis I. Kaplanoglou" <pikaplanoglou@ihu.gr>
 License-Expression: MIT

{radnn-0.1.5.dist-info → radnn-0.1.7.dist-info}/RECORD

@@ -1,4 +1,4 @@
-radnn/__init__.py,sha256=l4r8WIDCPKjjAuyib32zzfASqpvs3E6Rd3yIsTzyGRM,603
+radnn/__init__.py,sha256=_xOujzWa_LHOE_imQO45vC0q8VCdcGoTfoyzb9-IH7A,669
 radnn/core.py,sha256=p3CXa192LhpvopTTJO3wPCJzUzKuH4KZSEzzOg1FIP0,11090
 radnn/errors.py,sha256=YiaXHWAGSijOyufTzOVqFvba4CxpMUxQDdQIfZS9YK0,2979
 radnn/ml_system.py,sha256=lTxMgs81TLjKxMGjB10RKD5TKUp9koDPLQSbpQmGINk,7681
@@ -8,16 +8,16 @@ radnn/benchmark/latency.py,sha256=waaRYuW0ySH82Isj9VaIor3AKQWMqyetVxQae4zm8SM,12
 radnn/benchmark/vram.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 radnn/data/__init__.py,sha256=y4l-g0dq74vJbFpPkP7zLKEbuXjv97QcCcUtGQODLZQ,441
 radnn/data/constants.py,sha256=yeJoWCQMc0wEGnzapE3XJM9ciMNEZfv5Qk6gjiC8ldA,357
-radnn/data/custom_data_set.py,sha256=h-QZg1lBGq6oj-Z_mmkIiWI1s2vJXKMoEoWWbKKv25g,5493
+radnn/data/custom_data_set.py,sha256=RzpHIm0q0iGu26ylKU1soM9GqigVYcDUjp092pSpN8w,5496
 radnn/data/data_hyperparams.py,sha256=eOSii6eTjtNH0hfWdysWK7Tcsdx8eUgaFneSxTCmQig,2942
-radnn/data/dataset_base.py,sha256=oqKGMmA1oxSoD28zLO_qm912ZHwO2z7smcoyzmWQt-Q,9294
-radnn/data/dataset_base_legacy.py,sha256=qbYQ3oEVdbje1p50eGgIt4kxhi6PnBOIs5tEGl8dLik,12148
+radnn/data/dataset_base.py,sha256=T-p8Q7Hnc8-HIAjhb71IEnTk7t4cugUDnLhV4g9wpjA,12208
+radnn/data/dataset_base_legacy.py,sha256=cAAO2x9cRHgNiASTFXLXoDj1Kkz4ozag8JKuGkJJQOs,12151
 radnn/data/dataset_factory.py,sha256=ntfTKz4sIoFcpohytz-9HnHAyvcyfLJmjdaWBdSVZa4,5430
-radnn/data/errors.py,sha256=LD9WeTk4V63UjZkDC4R_DejOISv01AI0bRGyRP9TC98,2315
-radnn/data/sample_preprocessor.py,sha256=Pcbow4ZQX88s-nViaQ5Csp72DSy29aaAweVKXNqk5XQ,3254
+radnn/data/errors.py,sha256=xQN9Ly6yHY0KJVxFrB7Af4Y2R-cZTc3xktvW2iQ1xzQ,2318
+radnn/data/sample_preprocessor.py,sha256=VWPH4pPyawxAUKh2nrLVulXY3Y4fc4qTIIeqCXxgSOo,3260
 radnn/data/sample_set.py,sha256=NDvHdKshqVtBE44ZB_5iYBDyItuC6oP-HLNaU-csrCw,14296
 radnn/data/sample_set_kind.py,sha256=h793DtOsnkBI_gTQq1yx3FB9RiggOV2tJJzySx55JbM,6404
-radnn/data/sample_set_simple.py,sha256=nWfuBvvgcTouiRJXqnOO8P-ZTZVXJyTHIabQV9_ca9I,4496
+radnn/data/sample_set_simple.py,sha256=RKmnQSV1i_HjvMtFgh-KjZiQre8MzgbCh4Z6GZi-74I,4537
 radnn/data/sequence_dataset.py,sha256=t7eDcBn74EsKf-feWwLP6UkRAxjvTXChLlmv-CwBXJ4,5959
 radnn/data/structs/__init__.py,sha256=7XVuoBctqZWtDa_QBwhmUtTWbtkwdPIEyNfjKXSya9I,61
 radnn/data/structs/tree.py,sha256=ZVcP1hkxrLXNfmGHTb4Fk9kPmfmZebHfDfg9sYbBQAo,9774
@@ -40,10 +40,10 @@ radnn/evaluation/__init__.py,sha256=7dXDyJfOpSAr7G8jfDofsW4YEHNElCTTyMXuLCtpoOI,
 radnn/evaluation/evaluate_classification.py,sha256=riiVmCjGYfxTldrF2aOKH2xKmm6jAGfmXRDygbfUzYE,7378
 radnn/experiment/__init__.py,sha256=B-rw0Jjt6w51sS_-b88667gvOrolb0LVQU4L3kX8AFM,374
 radnn/experiment/identification.py,sha256=JxrVJ9hUUu7mOKmwIq_bddgckvsm7sP3hGWMD7udu3Q,405
-radnn/experiment/ml_experiment.py,sha256=oJF_XmbqWt0S1XbM19CjjQLDEjVnB9xm4ccXMz-wmY0,19559
+radnn/experiment/ml_experiment.py,sha256=m2FKRFbbMEleQSb1UzclCtyTyBqYvpy4rqpp6XPAnfc,19565
 radnn/experiment/ml_experiment_config.py,sha256=KweFMqdbWhUDM95raB8WhPcsmBbwRktaylbDsVPgJIg,11205
 radnn/experiment/ml_experiment_env.py,sha256=zoB5NxvFn5CyTq_FRsxB01HrnfnHLcYJUOTgPjL4_ac,11447
-radnn/experiment/ml_experiment_log.py,sha256=JuETe9M-3JTDQyxCpOit3uPC2qiXm5r0OOAx-YdJpUQ,2217
+radnn/experiment/ml_experiment_log.py,sha256=K2aZHrO90D50jUkx-qJHAmRVJkxhdvIh5R62t8qSZCQ,2345
 radnn/experiment/ml_experiment_store.py,sha256=Ph-4DQ9zEjhxZlQ13t-qnmCyxsKwsO2Df_Kj0vbMS_Q,396
 radnn/images/__init__.py,sha256=Mk7zKHQRDmCX-A4b1xw-3yxIwEApY-wlZTKiQr3eCqE,100
 radnn/images/colors.py,sha256=l6caSV2a_TURl1qHYKdehQDk0MCVMz-614OuVx-wnsg,1248
@@ -102,7 +102,7 @@ radnn/system/files/zipfile.py,sha256=ZcK8u5USjm_VE6d33ybu011sfZGH9upqOU46Y-oIdoo
 radnn/system/hosts/__init__.py,sha256=k2gkMJhe96Nf-V2ex6jZqmCRX9vA_K6gFB8J8Ii9ahc,261
 radnn/system/hosts/colab_host.py,sha256=i0s43KjdJ-gjLGyQAItubz2gZvOj-DbFnH1EGYguoVk,4000
 radnn/system/hosts/linux_host.py,sha256=AuOTpQ3OB1SXvsS1F-ksLVL44HXeRz5UEM2jbQ_1nbg,1623
-radnn/system/hosts/windows_host.py,sha256=smSnK2hNeBSLJFRw9Wh8Uni0RVVuFCxMzwFv_WzkCuY,4253
+radnn/system/hosts/windows_host.py,sha256=C3QSAkH5Hhk0y0Joy-WMa7EK6WQ7mUfAItzdTdj2ZxM,4256
 radnn/system/threads/__init__.py,sha256=PJrNngI79hne-fAhdn1mGIHNWbtuOMoHoNR4RXB5P2Y,252
 radnn/system/threads/semaphore_lock.py,sha256=UGf5f2WBo6sknuhPL-1Vqsg-25HroqfKPrGsoIeNPEo,3073
 radnn/system/threads/thread_context.py,sha256=wbRmeIoJSZaLH6Z_Gra-X2uqYLmMFL7ZLpHJzOzlIgE,7761
@@ -111,8 +111,8 @@ radnn/system/threads/thread_safe_string_collection.py,sha256=vdRMvwJ8CcLmsJ1uild
 radnn/system/threads/thread_worker.py,sha256=5KANBBHwnyaMvjyelBT1eyZCzRtH7MNZiHUhN1Xl1BY,3466
 radnn/test/__init__.py,sha256=XL9SgTJ6bGm3b0tcU3CroenP9rBm5XpDJozFGUv0UkQ,35
 radnn/test/tensor_hash.py,sha256=Jh4hSaSOLzSWF1_UI0ZLWL6zdi2SbswM1GNEuuFIYso,4203
-radnn-0.1.5.dist-info/licenses/LICENSE.txt,sha256=NMbnQdAQ2kWWQp2_8Sv2HG1p3jNvrPjYFSixnxPA3uE,1106
-radnn-0.1.5.dist-info/METADATA,sha256=qD08vielf__h6f0r2WTvJjrO4t_nVANtZu8ZZtNd3wk,1253
-radnn-0.1.5.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-radnn-0.1.5.dist-info/top_level.txt,sha256=FKlLIm6gRAeZlRzs-HCBJAB1q9ELJ7MgaL-qqFuPo6M,6
-radnn-0.1.5.dist-info/RECORD,,
+radnn-0.1.7.dist-info/licenses/LICENSE.txt,sha256=NMbnQdAQ2kWWQp2_8Sv2HG1p3jNvrPjYFSixnxPA3uE,1106
+radnn-0.1.7.dist-info/METADATA,sha256=zeZawVCoLlPrRQqi-oR4cqM3DTXZjFAu-9MbA9d0HEg,1253
+radnn-0.1.7.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+radnn-0.1.7.dist-info/top_level.txt,sha256=FKlLIm6gRAeZlRzs-HCBJAB1q9ELJ7MgaL-qqFuPo6M,6
+radnn-0.1.7.dist-info/RECORD,,