radnn 0.0.8__py3-none-any.whl → 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. radnn/__init__.py +5 -5
  2. radnn/benchmark/__init__.py +1 -0
  3. radnn/benchmark/latency.py +55 -0
  4. radnn/core.py +146 -2
  5. radnn/data/__init__.py +5 -10
  6. radnn/data/dataset_base.py +100 -260
  7. radnn/data/dataset_base_legacy.py +280 -0
  8. radnn/data/errors.py +32 -0
  9. radnn/data/sample_preprocessor.py +58 -0
  10. radnn/data/sample_set.py +203 -90
  11. radnn/data/sample_set_kind.py +126 -0
  12. radnn/data/sequence_dataset.py +25 -30
  13. radnn/data/structs/__init__.py +1 -0
  14. radnn/data/structs/tree.py +322 -0
  15. radnn/data_beta/__init__.py +12 -0
  16. radnn/{data → data_beta}/data_feed.py +1 -1
  17. radnn/data_beta/dataset_base.py +337 -0
  18. radnn/data_beta/sample_set.py +166 -0
  19. radnn/data_beta/sequence_dataset.py +134 -0
  20. radnn/data_beta/structures/__init__.py +2 -0
  21. radnn/data_beta/structures/dictionary.py +41 -0
  22. radnn/{data → data_beta}/tf_classification_data_feed.py +5 -2
  23. radnn/errors.py +10 -2
  24. radnn/experiment/__init__.py +2 -0
  25. radnn/experiment/identification.py +7 -0
  26. radnn/experiment/ml_experiment.py +7 -2
  27. radnn/experiment/ml_experiment_log.py +47 -0
  28. radnn/images/image_processor.py +4 -1
  29. radnn/learn/__init__.py +0 -7
  30. radnn/learn/keras/__init__.py +4 -0
  31. radnn/learn/{state → keras}/keras_best_state_saver.py +5 -1
  32. radnn/learn/{learning_algorithm.py → keras/keras_learning_algorithm.py} +5 -9
  33. radnn/learn/{keras_learning_rate_scheduler.py → keras/keras_learning_rate_scheduler.py} +4 -1
  34. radnn/learn/{keras_optimization_algorithm.py → keras/keras_optimization_combo.py} +7 -3
  35. radnn/learn/torch/__init__.py +3 -0
  36. radnn/learn/torch/ml_model_freezer.py +330 -0
  37. radnn/learn/torch/ml_trainer.py +461 -0
  38. radnn/learn/torch/staircase_lr_scheduler.py +21 -0
  39. radnn/ml_system.py +68 -52
  40. radnn/models/__init__.py +5 -0
  41. radnn/models/cnn/__init__.py +0 -0
  42. radnn/models/cnn/cnn_stem_setup.py +35 -0
  43. radnn/models/model_factory.py +85 -0
  44. radnn/models/model_hyperparams.py +128 -0
  45. radnn/models/model_info.py +91 -0
  46. radnn/plots/plot_learning_curve.py +19 -8
  47. radnn/system/__init__.py +1 -0
  48. radnn/system/files/__init__.py +1 -1
  49. radnn/system/files/csvfile.py +37 -5
  50. radnn/system/files/filelist.py +30 -0
  51. radnn/system/files/fileobject.py +11 -1
  52. radnn/system/files/imgfile.py +1 -1
  53. radnn/system/files/jsonfile.py +37 -9
  54. radnn/system/files/picklefile.py +3 -3
  55. radnn/system/files/textfile.py +39 -10
  56. radnn/system/files/zipfile.py +96 -0
  57. radnn/system/filestore.py +147 -47
  58. radnn/system/filesystem.py +3 -3
  59. radnn/test/__init__.py +1 -0
  60. radnn/test/tensor_hash.py +130 -0
  61. radnn/utils.py +16 -2
  62. radnn-0.1.0.dist-info/METADATA +30 -0
  63. radnn-0.1.0.dist-info/RECORD +99 -0
  64. {radnn-0.0.8.dist-info → radnn-0.1.0.dist-info}/WHEEL +1 -1
  65. {radnn-0.0.8.dist-info → radnn-0.1.0.dist-info/licenses}/LICENSE.txt +1 -1
  66. radnn/learn/state/__init__.py +0 -4
  67. radnn-0.0.8.dist-info/METADATA +0 -58
  68. radnn-0.0.8.dist-info/RECORD +0 -70
  69. /radnn/{data → data_beta}/dataset_folder.py +0 -0
  70. /radnn/{data → data_beta}/image_dataset.py +0 -0
  71. /radnn/{data → data_beta}/image_dataset_files.py +0 -0
  72. /radnn/{data → data_beta}/preprocess/__init__.py +0 -0
  73. /radnn/{data → data_beta}/preprocess/normalizer.py +0 -0
  74. /radnn/{data → data_beta}/preprocess/standardizer.py +0 -0
  75. /radnn/{data → data_beta}/subset_type.py +0 -0
  76. {radnn-0.0.8.dist-info → radnn-0.1.0.dist-info}/top_level.txt +0 -0
@@ -6,7 +6,7 @@
6
6
  # ______________________________________________________________________________________
7
7
  # ......................................................................................
8
8
 
9
- # Copyright (c) 2018-2025 Pantelis I. Kaplanoglou
9
+ # Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
10
10
 
11
11
  # Permission is hereby granted, free of charge, to any person obtaining a copy
12
12
  # of this software and associated documentation files (the "Software"), to deal
@@ -38,15 +38,50 @@ class TextFile(FileObject):
38
38
  def __init__(self, filename, parent_folder=None, error_template=None, is_verbose=False):
39
39
  super(TextFile, self).__init__(filename, parent_folder, error_template)
40
40
  self.is_verbose = is_verbose
41
+ self._opened_filename = None
42
+ self._encoding = None
43
+ # ----------------------------------------------------------------------------------
44
+ def open(self, filename, encoding=None):
45
+ self._opened_filename = filename
46
+ self._encoding = encoding
47
+ return self
41
48
 
42
49
  # ----------------------------------------------------------------------------------
50
+ def close(self):
51
+ self._opened_filename = None
52
+ self._encoding = None
53
+ # ----------------------------------------------------------------------------------
54
+ @property
55
+ def rows(self):
56
+ sFileName = self._useFileName(self._opened_filename)
57
+
58
+ if os.path.isfile(sFileName):
59
+ if self._encoding is not None:
60
+ oEncodingToTry = [self._encoding]
61
+ else:
62
+ oEncodingToTry = ["utf-8", "utf-16", "latin1", "ascii"] # Add more if needed
63
+ bIsLoaded = False
64
+ for sEnc in oEncodingToTry:
65
+ try:
66
+ with open(sFileName, "r", encoding=sEnc) as oFile:
67
+ yield oFile.read()
68
+ bIsLoaded = True
69
+ break
70
+ except (UnicodeDecodeError, UnicodeError):
71
+ continue
72
+ if not bIsLoaded:
73
+ raise ValueError("Unsupported encoding")
74
+ # ----------------------------------------------------------------------------------
43
75
  def load(self, filename=None, encoding=None):
44
76
  filename = self._useFileName(filename)
45
77
 
46
- oEncodingToTry = ["utf-8", "utf-16", "latin1", "ascii"] # Add more if needed
47
-
48
78
  sText = None
49
- if encoding is None:
79
+ if os.path.isfile(filename):
80
+ if self._encoding is not None:
81
+ oEncodingToTry = [encoding]
82
+ else:
83
+ oEncodingToTry = ["utf-8", "utf-16", "latin1", "ascii"] # Add more if needed
84
+
50
85
  bIsLoaded = False
51
86
  for sEnc in oEncodingToTry:
52
87
  try:
@@ -58,9 +93,6 @@ class TextFile(FileObject):
58
93
  continue
59
94
  if not bIsLoaded:
60
95
  raise ValueError("Unsupported encoding")
61
- else:
62
- with open(filename, "r", encoding=encoding) as oFile:
63
- sText = oFile.read()
64
96
 
65
97
  return sText
66
98
  # --------------------------------------------------------------------------------------------------------------------
@@ -74,9 +106,6 @@ class TextFile(FileObject):
74
106
  p_sFileName : Full path to the text file
75
107
  p_sText : Text to write
76
108
  """
77
- if (self.parent_folder is not None):
78
- sFilename = os.path.join(self.parent_folder, sFilename)
79
-
80
109
  if self.is_verbose:
81
110
  print(" {.} Saving text to %s" % sFilename)
82
111
 
@@ -0,0 +1,96 @@
1
+ # ======================================================================================
2
+ #
3
+ # Rapid Deep Neural Networks
4
+ #
5
+ # Licensed under the MIT License
6
+ # ______________________________________________________________________________________
7
+ # ......................................................................................
8
+
9
+ # Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
10
+
11
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
12
+ # of this software and associated documentation files (the "Software"), to deal
13
+ # in the Software without restriction, including without limitation the rights
14
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15
+ # copies of the Software, and to permit persons to whom the Software is
16
+ # furnished to do so, subject to the following conditions:
17
+
18
+ # The above copyright notice and this permission notice shall be included in all
19
+ # copies or substantial portions of the Software.
20
+
21
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27
+ # SOFTWARE.
28
+
29
+ # .......................................................................................
30
+ import os
31
+ import zipfile
32
+ from .fileobject import FileObject
33
+
34
+ class ZipFile(FileObject):
35
+ # --------------------------------------------------------------------------------------------------------------------
36
+ def __init__(self, filename, parent_folder=None, error_template=None, is_verbose=False):
37
+ super().__init__(filename, parent_folder, error_template)
38
+ self.is_verbose = is_verbose
39
+ self._opened_filename = None
40
+ self._encoding = None
41
+
42
+ # --------------------------------------------------------------------------------------------------------
43
+ def save(self, source_path, must_replace=True):
44
+ """
45
+ Compress a folder (including subfolders) into a .zip file.
46
+
47
+ Args:
48
+ output_zip_path (str): The path where the .zip file will be saved.
49
+ """
50
+ dest_zip_file = os.path.join(self.parent_folder, self.filename)
51
+ if must_replace:
52
+ if os.path.exists(dest_zip_file):
53
+ os.remove(dest_zip_file)
54
+
55
+ # Create the zip file
56
+ with zipfile.ZipFile(dest_zip_file, 'w', zipfile.ZIP_DEFLATED) as zipf:
57
+ for root, dirs, files in os.walk(source_path):
58
+ for file in files:
59
+ abs_path = os.path.join(root, file)
60
+ # Get relative path for proper folder structure inside zip
61
+ rel_path = os.path.relpath(abs_path, start=source_path)
62
+ zipf.write(abs_path, rel_path)
63
+
64
+ # --------------------------------------------------------------------------------------------------------
65
+ def extract(self, dest_path, has_progress_bar=True, is_verbose=True):
66
+ """
67
+ Extracts a .zip file into the specified folder.
68
+
69
+ Args:
70
+ zip_filename (str): Path to the .zip file.
71
+ destination_folder (str): Folder where the contents should be extracted.
72
+ """
73
+ source_zip_file = os.path.join(self.parent_folder, self.filename)
74
+
75
+ # Ensure the zip file exists
76
+ if not os.path.isfile(source_zip_file):
77
+ raise FileNotFoundError(f"Zip file not found: {source_zip_file}")
78
+
79
+ # Ensure destination directory exists
80
+ os.makedirs(dest_path, exist_ok=True)
81
+
82
+ # Extract the zip file
83
+ with zipfile.ZipFile(source_zip_file, 'r') as zip_ref:
84
+ file_list = zip_ref.infolist()
85
+ total_files = len(file_list)
86
+
87
+ oIterator = file_list
88
+ if is_verbose and has_progress_bar:
89
+ from tqdm import tqdm
90
+ oIterator = tqdm(file_list, total=total_files, desc="Extracting", unit="file")
91
+
92
+ for file_info in oIterator:
93
+ zip_ref.extract(file_info, dest_path)
94
+ if is_verbose:
95
+ print(f" |_ Extracted '{source_zip_file}' to '{dest_path}'")
96
+ # ----------------------------------------------------------------------------------
radnn/system/filestore.py CHANGED
@@ -6,7 +6,7 @@
6
6
  # ______________________________________________________________________________________
7
7
  # ......................................................................................
8
8
 
9
- # Copyright (c) 2018-2025 Pantelis I. Kaplanoglou
9
+ # Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
10
10
 
11
11
  # Permission is hereby granted, free of charge, to any person obtaining a copy
12
12
  # of this software and associated documentation files (the "Software"), to deal
@@ -31,6 +31,9 @@ import os
31
31
  import shutil
32
32
  import glob
33
33
  import sys
34
+ import zipfile
35
+ from typing import Any, Type
36
+ import importlib.util
34
37
  if (sys.version_info.major == 3) and (sys.version_info.minor <= 7):
35
38
  import pickle5 as pickle
36
39
  else:
@@ -40,11 +43,13 @@ from radnn.system.files import JSONFile
40
43
  from radnn.system.files import PickleFile
41
44
  from radnn.system.files import TextFile
42
45
  from radnn.system.files import CSVFile
43
- from radnn.ml_system import mlsys
44
- if mlsys.is_opencv_installed:
45
- from radnn.system.files.imgfile import PNGFile
46
+ from radnn.system.files import ZipFile
47
+ from radnn.errors import *
46
48
 
47
49
 
50
+ _is_opencv_installed = importlib.util.find_spec("cv2") is not None
51
+ if _is_opencv_installed:
52
+ from radnn.system.files.imgfile import PNGFile
48
53
 
49
54
 
50
55
  # =======================================================================================================================
@@ -60,13 +65,13 @@ class FileStore(object):
60
65
  else:
61
66
  os.makedirs(self.base_folder)
62
67
 
63
- self.is_verbose = is_verbose
64
- self.json = JSONFile(None, parent_folder=self.base_folder)
65
- self.obj = PickleFile(None, parent_folder=self.base_folder)
66
- self.text = TextFile(None, parent_folder=self.base_folder)
67
- self.csv = CSVFile(None, parent_folder=self.base_folder)
68
- if mlsys.is_opencv_installed:
69
- self.img = PNGFile(None, parent_folder=base_folder)
68
+ self.is_verbose = is_verbose
69
+ self.json: JSONFile = JSONFile(None, parent_folder=self.base_folder)
70
+ self.obj: PickleFile = PickleFile(None, parent_folder=self.base_folder)
71
+ self.text: TextFile = TextFile(None, parent_folder=self.base_folder)
72
+ self.csv: CSVFile = CSVFile(None, parent_folder=self.base_folder)
73
+ if _is_opencv_installed:
74
+ self.img: PNGFile = PNGFile(None, parent_folder=base_folder)
70
75
  self.donefs = None
71
76
  #................................................................................
72
77
  # --------------------------------------------------------------------------------------------------------
@@ -113,6 +118,48 @@ class FileStore(object):
113
118
  file_name += file_ext
114
119
  return os.path.join(self.absolute_path, file_name)
115
120
  # --------------------------------------------------------------------------------------------------------
121
+ def get_file_kind(self, file_name: str):
122
+ name, ext = os.path.splitext(file_name)
123
+ ext = ext.lower()
124
+ if ext == ".pkl":
125
+ return "obj"
126
+ elif ext == ".txt":
127
+ return "text"
128
+ elif ext == ".json":
129
+ return "json"
130
+ elif ext == ".png":
131
+ return "img"
132
+ elif ext == ".zip":
133
+ return "zip"
134
+ else:
135
+ return "?"
136
+ # --------------------------------------------------------------------------------------------------------
137
+ def artifact(self, file_name: str, kind: str | None = None) -> Any:
138
+ if kind is None:
139
+ kind = self.get_file_kind(file_name)
140
+
141
+ oFile = None
142
+ if kind == "obj":
143
+ oFile = PickleFile(file_name, parent_folder=self.base_folder)
144
+ elif kind == "text":
145
+ oFile = TextFile(file_name, parent_folder=self.base_folder)
146
+ elif kind == "json":
147
+ oFile = JSONFile(file_name, parent_folder=self.base_folder)
148
+ elif kind == "csv":
149
+ oFile = CSVFile(file_name, parent_folder=self.base_folder)
150
+ elif kind == "img":
151
+ if _is_opencv_installed:
152
+ oFile = PNGFile(file_name, parent_folder=self.base_folder)
153
+ else:
154
+ print("Open CV is not installed") #TODO: Support PIL
155
+ elif kind == "zip":
156
+ oFile = ZipFile(file_name, parent_folder=self.base_folder)
157
+
158
+ if oFile is None:
159
+ raise Exception(FILESTORE_DATAFILE_KIND_NOT_SUPPORTED % str(kind))
160
+
161
+ return oFile
162
+ # --------------------------------------------------------------------------------------------------------
116
163
  def entries(self):
117
164
  return os.listdir(self.base_folder)
118
165
  # --------------------------------------------------------------------------------------------------------
@@ -173,6 +220,95 @@ class FileStore(object):
173
220
  if is_full_path and (sFileName is not None):
174
221
  sFileName = os.path.join(self.donefs.base_folder, sFileName)
175
222
  return sFileName
223
+
224
+ # --------------------------------------------------------------------------------------------------------
225
+ def move_to(self, source_files: str | list, fs):
226
+ oFiles = source_files
227
+ if isinstance(source_files, str):
228
+ # A pattern is given
229
+ if "*" in source_files:
230
+ oFiles = self.list_files(source_files, is_full_path=False)
231
+ else:
232
+ oFiles = [source_files]
233
+ oDestFiles = []
234
+ for sFile in oFiles:
235
+ sDestFile = os.path.join(fs.absolute_path, sFile)
236
+ oDestFiles.append(sDestFile)
237
+ shutil.move(os.path.join(self.absolute_path, sFile),sDestFile)
238
+ return oDestFiles
239
+
240
+ # --------------------------------------------------------------------------------------------------------
241
+ def copy_to(self, source_files: str | list, fs):
242
+ oFiles = source_files
243
+ if isinstance(source_files, str):
244
+ # A pattern is given
245
+ if "*" in source_files:
246
+ oFiles = self.list_files(source_files, is_full_path=False)
247
+ else:
248
+ oFiles = [source_files]
249
+ oDestFiles = []
250
+ for sFile in oFiles:
251
+ sDestFile = os.path.join(fs.absolute_path, sFile)
252
+ oDestFiles.append(sDestFile)
253
+ shutil.copy(os.path.join(self.absolute_path, sFile),sDestFile)
254
+ return oDestFiles
255
+
256
+ # --------------------------------------------------------------------------------------------------------
257
+ def remove_existing(self, filename):
258
+ sFileName = self.file(filename)
259
+ if os.path.exists(sFileName):
260
+ os.remove(sFileName)
261
+ # --------------------------------------------------------------------------------------------------------
262
+ def compress_to_zip(self, output_zip_path, must_replace=True):
263
+ """
264
+ Compress a folder (including subfolders) into a .zip file.
265
+
266
+ Args:
267
+ output_zip_path (str): The path where the .zip file will be saved.
268
+ """
269
+ if must_replace:
270
+ if os.path.exists(output_zip_path):
271
+ os.remove(output_zip_path)
272
+
273
+ # Create the zip file
274
+ with zipfile.ZipFile(output_zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
275
+ for root, dirs, files in os.walk(self.absolute_path):
276
+ for file in files:
277
+ abs_path = os.path.join(root, file)
278
+ # Get relative path for proper folder structure inside zip
279
+ rel_path = os.path.relpath(abs_path, start=self.absolute_path)
280
+ zipf.write(abs_path, rel_path)
281
+ # --------------------------------------------------------------------------------------------------------
282
+ def extract_from_zip(self, zip_filename, has_progress_bar=True):
283
+ """
284
+ Extracts a .zip file into the specified folder.
285
+
286
+ Args:
287
+ zip_filename (str): Path to the .zip file.
288
+ destination_folder (str): Folder where the contents should be extracted.
289
+ """
290
+ # Ensure the zip file exists
291
+ if not os.path.isfile(zip_filename):
292
+ raise FileNotFoundError(f"Zip file not found: {zip_filename}")
293
+
294
+ sDestFolder = self.absolute_path
295
+ # Ensure destination directory exists
296
+ os.makedirs(sDestFolder, exist_ok=True)
297
+
298
+ # Extract the zip file
299
+ with zipfile.ZipFile(zip_filename, 'r') as zip_ref:
300
+ file_list = zip_ref.infolist()
301
+ total_files = len(file_list)
302
+
303
+ oIterator = file_list
304
+ if has_progress_bar:
305
+ from tqdm import tqdm
306
+ oIterator = tqdm(file_list, total=total_files, desc="Extracting", unit="file")
307
+
308
+ for file_info in oIterator:
309
+ zip_ref.extract(file_info, sDestFolder)
310
+
311
+ print(f" |_ Extracted '{zip_filename}' to '{sDestFolder}'")
176
312
  # --------------------------------------------------------------------------------------------------------
177
313
  def purge_done(self):
178
314
  #//TODO: Remove from the current filestore all files that are moved into the .done filestore
@@ -187,42 +323,6 @@ class FileStore(object):
187
323
  # ======================================================================================================================
188
324
 
189
325
 
190
- # .................... Tests ....................
191
- if __name__ == "__main__":
192
- VISUALIZE = False
193
- import numpy as np
194
-
195
- # Lazy initialize a file store folder
196
- oFS = FileStore("C:\MLData\SomeDataSet")
197
-
198
- # Serialize data into a sub folder
199
- oData = np.asarray([1, 2, 3]).astype(np.float32)
200
- print(oFS.subfs("test"))
201
- oFS.subfs("test").obj.save(oData, "sample1.pkl")
202
-
203
- # Get paths
204
- print("==== Paths ====")
205
- print(oFS.subfs("test").file("sample1.png"))
206
- print(oFS.folder("test"))
207
-
208
- # Deserialize data from a sub folder
209
- oLoadedData = oFS.subfs("test").obj.load("sample1.pkl")
210
- print(oLoadedData)
211
-
212
- # List files
213
- print("==== Files ====")
214
- sFiles = oFS.subfs("test").list_files("*.pkl", True)
215
- print("\n".join(sFiles))
216
-
217
- # Load a PNG file
218
- if (is_opencv_installed()):
219
- oPng = FileStore("C:\\UX").subfs("PNG").img.load("Ideas.png")
220
- print(oPng.shape)
221
-
222
- if VISUALIZE:
223
- import matplotlib.pyplot as plt
224
- plt.imshow(oPng)
225
- plt.show()
226
326
 
227
327
 
228
328
 
@@ -77,9 +77,9 @@ class FileSystem(object):
77
77
 
78
78
  # ...................... | Fields | ......................
79
79
  self.setup = dSetup
80
- self.configs = FileStore(config_folder, must_exist=self._must_exist)
81
- self.models = FileStore(model_folder, must_exist=self._must_exist)
82
- self.datasets = FileStore(dataset_folder, must_exist=self._must_exist)
80
+ self.configs: FileStore = FileStore(config_folder, must_exist=self._must_exist)
81
+ self.models: FileStore = FileStore(model_folder, must_exist=self._must_exist)
82
+ self.datasets: FileStore = FileStore(dataset_folder, must_exist=self._must_exist)
83
83
  # ........................................................
84
84
  # --------------------------------------------------------------------------------------------------------
85
85
  def group(self, group_name):
radnn/test/__init__.py ADDED
@@ -0,0 +1 @@
1
+ from .tensor_hash import TensorHash
@@ -0,0 +1,130 @@
1
+ import hashlib
2
+ import numpy as np
3
+ from radnn import mlsys, FileStore
4
+
5
+ if mlsys.framework == "torch":
6
+ import torch
7
+ elif mlsys.framework == "tensorflow":
8
+ import tensorflow as tf
9
+
10
+
11
+ class TensorHash(object):
12
+ def __init__(self, test_fs, filename):
13
+ self.fs: FileStore = test_fs
14
+ self.filename = filename
15
+ self.hashes = []
16
+ self.loaded_hashes = None
17
+ self.is_loaded = False
18
+
19
+ def collect_model_params(self, model):
20
+ if mlsys.framework == "torch":
21
+ for key, value in model.state_dict().items():
22
+ # print(key, value.shape)
23
+ self.collect_hash(value)
24
+ return self
25
+
26
+ def end_collection(self):
27
+ if self.is_loaded:
28
+ if self.compare():
29
+ print(f"[v] {self.filename} Reproducibility for {len(self.hashes)} collected values")
30
+ else:
31
+ print(f"[x] {self.filename} Non deterministic behaviour!")
32
+ else:
33
+ self.save()
34
+
35
+ def collect_hash(self, x: torch.Tensor, normalize_shape=True) -> str:
36
+ if mlsys.framework == "torch":
37
+ return self.calculate_hash_torch(x, normalize_shape)
38
+ elif mlsys.framework == "tensorflow":
39
+ return self.calculate_hash_tf(x, normalize_shape)
40
+ else:
41
+ return ""
42
+
43
+ def calculate_hash_tf(self, x, normalize_shape: bool = True) -> str:
44
+ if not tf.is_tensor(x):
45
+ # allow numpy arrays too if you want:
46
+ if isinstance(x, np.ndarray):
47
+ x = tf.convert_to_tensor(x)
48
+ else:
49
+ raise TypeError("x must be a tf.Tensor (or numpy.ndarray)")
50
+
51
+ t = x # keep as tf.Tensor
52
+
53
+ # Canonicalize shape to (28, 28, 1) if requested
54
+ if normalize_shape:
55
+ if t.shape.rank == 2:
56
+ # (28, 28) -> (28, 28, 1)
57
+ t = tf.expand_dims(t, axis=-1)
58
+ elif t.shape.rank == 3:
59
+ # (1, 28, 28) -> (28, 28, 1) (common in CHW)
60
+ # Only do this if first dim is 1.
61
+ if t.shape[0] == 1:
62
+ t = tf.transpose(t, perm=[1, 2, 0])
63
+
64
+ # Canonicalize dtype
65
+ if t.dtype.is_floating:
66
+ t = tf.clip_by_value(t, 0.0, 1.0)
67
+ t = tf.round(t * 255.0)
68
+ t = tf.cast(t, tf.uint8)
69
+ else:
70
+ t = tf.cast(t, tf.uint8)
71
+
72
+ # Hash raw bytes (ensure deterministic C-order bytes)
73
+ # tf.io.serialize_tensor includes dtype/shape metadata, so we avoid it.
74
+ b = t.numpy().tobytes(order="C")
75
+ sResult = hashlib.sha256(b).hexdigest()
76
+ self.hashes.append(sResult)
77
+ return sResult
78
+
79
+ def calculate_hash_torch(self, x: torch.Tensor, normalize_shape=True) -> str:
80
+ if not isinstance(x, torch.Tensor):
81
+ raise TypeError("x must be a torch.Tensor")
82
+
83
+ # Move to CPU, detach, make contiguous
84
+ t = x.detach().to("cpu").contiguous()
85
+
86
+ # Canonicalize shape to (28, 28, 1) if requested
87
+ if normalize_shape:
88
+ if t.ndim == 2: # (28,28)
89
+ t = t.unsqueeze(-1) # (28,28,1)
90
+ elif t.ndim == 3 and t.shape[0] == 1: # (1,28,28) -> (28,28,1)
91
+ t = t.permute(1, 2, 0).contiguous()
92
+
93
+ # Canonicalize dtype to avoid hash changing across float16/float32/etc.
94
+ # If your data is in [0,1] floats, this will quantize deterministically.
95
+ if t.dtype.is_floating_point:
96
+ t = (t.clamp(0, 1) * 255.0).round().to(torch.uint8)
97
+ else:
98
+ # If already uint8/int, you can keep it; here we convert to uint8 for consistency.
99
+ t = t.to(torch.uint8)
100
+
101
+ # Hash raw bytes
102
+ b = t.numpy().tobytes(order="C")
103
+ sResult = hashlib.sha256(b).hexdigest()
104
+ self.hashes.append(sResult)
105
+ return sResult
106
+
107
+ def save(self, filename=None):
108
+ if filename is not None:
109
+ self.filename = filename
110
+
111
+ self.fs.text.save(self.hashes, self.filename)
112
+ return self
113
+
114
+ def load(self, filename=None):
115
+ if filename is not None:
116
+ self.filename = filename
117
+
118
+ self.loaded_hashes = self.fs.text.load(self.filename)
119
+ self.is_loaded = self.loaded_hashes is not None
120
+ return self
121
+
122
+ def compare(self):
123
+ bResult = True
124
+ if self.loaded_hashes is not None:
125
+ for nIndex, sHash in enumerate(self.hashes):
126
+ if sHash != self.hashes[nIndex]:
127
+ bResult = False
128
+ break
129
+
130
+ return bResult
radnn/utils.py CHANGED
@@ -6,7 +6,7 @@
6
6
  # ______________________________________________________________________________________
7
7
  # ......................................................................................
8
8
 
9
- # Copyright (c) 2024-2025 Pantelis I. Kaplanoglou
9
+ # Copyright (c) 2024-2026 Pantelis I. Kaplanoglou
10
10
 
11
11
  # Permission is hereby granted, free of charge, to any person obtaining a copy
12
12
  # of this software and associated documentation files (the "Software"), to deal
@@ -28,6 +28,7 @@
28
28
 
29
29
  # .......................................................................................
30
30
  import numpy as np
31
+ import json
31
32
  import time
32
33
  import hashlib
33
34
  import zlib
@@ -35,8 +36,21 @@ import contextlib
35
36
 
36
37
  phi=(1.0+np.sqrt(5.0))/2.0
37
38
 
39
+ # ======================================================================================================================
40
+ class classproperty(property):
41
+ def __get__(self, obj, owner):
42
+ return self.fget(owner)
43
+ # ======================================================================================================================
38
44
 
39
- # --------------------------------------------------------------------------------------
45
+
46
+ # ----------------------------------------------------------------------------------------------------------------------
47
+ def to_json(obj, is_sorted_keys=False, is_utf8=False):
48
+ if isinstance(obj, dict):
49
+ sJSON = json.dumps(obj, sort_keys=is_sorted_keys, indent=4, ensure_ascii=is_utf8)
50
+ else:
51
+ sJSON = json.dumps(obj, default=lambda o: obj.__dict__, sort_keys=is_sorted_keys, indent=4, ensure_ascii=is_utf8)
52
+ return sJSON
53
+ # ----------------------------------------------------------------------------------------------------------------------
40
54
  '''
41
55
  Checks if the p_sSettingsName is inside the settings dictionary p_dConfig
42
56
  and returns its value, otherwise the p_oDefault value
@@ -0,0 +1,30 @@
1
+ Metadata-Version: 2.4
2
+ Name: radnn
3
+ Version: 0.1.0
4
+ Summary: Rapid Deep Neural Networks
5
+ Author-email: "Pantelis I. Kaplanoglou" <pikaplanoglou@ihu.gr>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/pikaplan/radnn
8
+ Project-URL: Documentation, https://radnn.readthedocs.io/
9
+ Classifier: Intended Audience :: Science/Research
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Programming Language :: Python
12
+ Classifier: Topic :: Software Development
13
+ Classifier: Topic :: Scientific/Engineering
14
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
15
+ Classifier: Development Status :: 2 - Pre-Alpha
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3.9
19
+ Classifier: Programming Language :: Python :: 3.10
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Requires-Python: >=3.10
23
+ Description-Content-Type: text/markdown
24
+ License-File: LICENSE.txt
25
+ Requires-Dist: numpy>=1.26.4
26
+ Requires-Dist: matplotlib>=3.8.4
27
+ Requires-Dist: pandas>=2.2.1
28
+ Requires-Dist: scikit-learn>=1.4.2
29
+ Requires-Dist: tqdm>=4.67.1
30
+ Dynamic: license-file