radnn 0.0.8__py3-none-any.whl → 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- radnn/__init__.py +5 -5
- radnn/benchmark/__init__.py +1 -0
- radnn/benchmark/latency.py +55 -0
- radnn/core.py +146 -2
- radnn/data/__init__.py +5 -10
- radnn/data/dataset_base.py +100 -260
- radnn/data/dataset_base_legacy.py +280 -0
- radnn/data/errors.py +32 -0
- radnn/data/sample_preprocessor.py +58 -0
- radnn/data/sample_set.py +203 -90
- radnn/data/sample_set_kind.py +126 -0
- radnn/data/sequence_dataset.py +25 -30
- radnn/data/structs/__init__.py +1 -0
- radnn/data/structs/tree.py +322 -0
- radnn/data_beta/__init__.py +12 -0
- radnn/{data → data_beta}/data_feed.py +1 -1
- radnn/data_beta/dataset_base.py +337 -0
- radnn/data_beta/sample_set.py +166 -0
- radnn/data_beta/sequence_dataset.py +134 -0
- radnn/data_beta/structures/__init__.py +2 -0
- radnn/data_beta/structures/dictionary.py +41 -0
- radnn/{data → data_beta}/tf_classification_data_feed.py +5 -2
- radnn/errors.py +10 -2
- radnn/experiment/__init__.py +2 -0
- radnn/experiment/identification.py +7 -0
- radnn/experiment/ml_experiment.py +7 -2
- radnn/experiment/ml_experiment_log.py +47 -0
- radnn/images/image_processor.py +4 -1
- radnn/learn/__init__.py +0 -7
- radnn/learn/keras/__init__.py +4 -0
- radnn/learn/{state → keras}/keras_best_state_saver.py +5 -1
- radnn/learn/{learning_algorithm.py → keras/keras_learning_algorithm.py} +5 -9
- radnn/learn/{keras_learning_rate_scheduler.py → keras/keras_learning_rate_scheduler.py} +4 -1
- radnn/learn/{keras_optimization_algorithm.py → keras/keras_optimization_combo.py} +7 -3
- radnn/learn/torch/__init__.py +3 -0
- radnn/learn/torch/ml_model_freezer.py +330 -0
- radnn/learn/torch/ml_trainer.py +461 -0
- radnn/learn/torch/staircase_lr_scheduler.py +21 -0
- radnn/ml_system.py +68 -52
- radnn/models/__init__.py +5 -0
- radnn/models/cnn/__init__.py +0 -0
- radnn/models/cnn/cnn_stem_setup.py +35 -0
- radnn/models/model_factory.py +85 -0
- radnn/models/model_hyperparams.py +128 -0
- radnn/models/model_info.py +91 -0
- radnn/plots/plot_learning_curve.py +19 -8
- radnn/system/__init__.py +1 -0
- radnn/system/files/__init__.py +1 -1
- radnn/system/files/csvfile.py +37 -5
- radnn/system/files/filelist.py +30 -0
- radnn/system/files/fileobject.py +11 -1
- radnn/system/files/imgfile.py +1 -1
- radnn/system/files/jsonfile.py +37 -9
- radnn/system/files/picklefile.py +3 -3
- radnn/system/files/textfile.py +39 -10
- radnn/system/files/zipfile.py +96 -0
- radnn/system/filestore.py +147 -47
- radnn/system/filesystem.py +3 -3
- radnn/test/__init__.py +1 -0
- radnn/test/tensor_hash.py +130 -0
- radnn/utils.py +16 -2
- radnn-0.1.0.dist-info/METADATA +30 -0
- radnn-0.1.0.dist-info/RECORD +99 -0
- {radnn-0.0.8.dist-info → radnn-0.1.0.dist-info}/WHEEL +1 -1
- {radnn-0.0.8.dist-info → radnn-0.1.0.dist-info/licenses}/LICENSE.txt +1 -1
- radnn/learn/state/__init__.py +0 -4
- radnn-0.0.8.dist-info/METADATA +0 -58
- radnn-0.0.8.dist-info/RECORD +0 -70
- /radnn/{data → data_beta}/dataset_folder.py +0 -0
- /radnn/{data → data_beta}/image_dataset.py +0 -0
- /radnn/{data → data_beta}/image_dataset_files.py +0 -0
- /radnn/{data → data_beta}/preprocess/__init__.py +0 -0
- /radnn/{data → data_beta}/preprocess/normalizer.py +0 -0
- /radnn/{data → data_beta}/preprocess/standardizer.py +0 -0
- /radnn/{data → data_beta}/subset_type.py +0 -0
- {radnn-0.0.8.dist-info → radnn-0.1.0.dist-info}/top_level.txt +0 -0
radnn/system/files/textfile.py
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
# ______________________________________________________________________________________
|
|
7
7
|
# ......................................................................................
|
|
8
8
|
|
|
9
|
-
# Copyright (c) 2018-
|
|
9
|
+
# Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
|
|
10
10
|
|
|
11
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
@@ -38,15 +38,50 @@ class TextFile(FileObject):
|
|
|
38
38
|
def __init__(self, filename, parent_folder=None, error_template=None, is_verbose=False):
|
|
39
39
|
super(TextFile, self).__init__(filename, parent_folder, error_template)
|
|
40
40
|
self.is_verbose = is_verbose
|
|
41
|
+
self._opened_filename = None
|
|
42
|
+
self._encoding = None
|
|
43
|
+
# ----------------------------------------------------------------------------------
|
|
44
|
+
def open(self, filename, encoding=None):
|
|
45
|
+
self._opened_filename = filename
|
|
46
|
+
self._encoding = encoding
|
|
47
|
+
return self
|
|
41
48
|
|
|
42
49
|
# ----------------------------------------------------------------------------------
|
|
50
|
+
def close(self):
|
|
51
|
+
self._opened_filename = None
|
|
52
|
+
self._encoding = None
|
|
53
|
+
# ----------------------------------------------------------------------------------
|
|
54
|
+
@property
|
|
55
|
+
def rows(self):
|
|
56
|
+
sFileName = self._useFileName(self._opened_filename)
|
|
57
|
+
|
|
58
|
+
if os.path.isfile(sFileName):
|
|
59
|
+
if self._encoding is not None:
|
|
60
|
+
oEncodingToTry = [self._encoding]
|
|
61
|
+
else:
|
|
62
|
+
oEncodingToTry = ["utf-8", "utf-16", "latin1", "ascii"] # Add more if needed
|
|
63
|
+
bIsLoaded = False
|
|
64
|
+
for sEnc in oEncodingToTry:
|
|
65
|
+
try:
|
|
66
|
+
with open(sFileName, "r", encoding=sEnc) as oFile:
|
|
67
|
+
yield oFile.read()
|
|
68
|
+
bIsLoaded = True
|
|
69
|
+
break
|
|
70
|
+
except (UnicodeDecodeError, UnicodeError):
|
|
71
|
+
continue
|
|
72
|
+
if not bIsLoaded:
|
|
73
|
+
raise ValueError("Unsupported encoding")
|
|
74
|
+
# ----------------------------------------------------------------------------------
|
|
43
75
|
def load(self, filename=None, encoding=None):
|
|
44
76
|
filename = self._useFileName(filename)
|
|
45
77
|
|
|
46
|
-
oEncodingToTry = ["utf-8", "utf-16", "latin1", "ascii"] # Add more if needed
|
|
47
|
-
|
|
48
78
|
sText = None
|
|
49
|
-
if
|
|
79
|
+
if os.path.isfile(filename):
|
|
80
|
+
if self._encoding is not None:
|
|
81
|
+
oEncodingToTry = [encoding]
|
|
82
|
+
else:
|
|
83
|
+
oEncodingToTry = ["utf-8", "utf-16", "latin1", "ascii"] # Add more if needed
|
|
84
|
+
|
|
50
85
|
bIsLoaded = False
|
|
51
86
|
for sEnc in oEncodingToTry:
|
|
52
87
|
try:
|
|
@@ -58,9 +93,6 @@ class TextFile(FileObject):
|
|
|
58
93
|
continue
|
|
59
94
|
if not bIsLoaded:
|
|
60
95
|
raise ValueError("Unsupported encoding")
|
|
61
|
-
else:
|
|
62
|
-
with open(filename, "r", encoding=encoding) as oFile:
|
|
63
|
-
sText = oFile.read()
|
|
64
96
|
|
|
65
97
|
return sText
|
|
66
98
|
# --------------------------------------------------------------------------------------------------------------------
|
|
@@ -74,9 +106,6 @@ class TextFile(FileObject):
|
|
|
74
106
|
p_sFileName : Full path to the text file
|
|
75
107
|
p_sText : Text to write
|
|
76
108
|
"""
|
|
77
|
-
if (self.parent_folder is not None):
|
|
78
|
-
sFilename = os.path.join(self.parent_folder, sFilename)
|
|
79
|
-
|
|
80
109
|
if self.is_verbose:
|
|
81
110
|
print(" {.} Saving text to %s" % sFilename)
|
|
82
111
|
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
7
|
+
# ......................................................................................
|
|
8
|
+
|
|
9
|
+
# Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
|
|
10
|
+
|
|
11
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
13
|
+
# in the Software without restriction, including without limitation the rights
|
|
14
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
15
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
16
|
+
# furnished to do so, subject to the following conditions:
|
|
17
|
+
|
|
18
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
19
|
+
# copies or substantial portions of the Software.
|
|
20
|
+
|
|
21
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
22
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
23
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
24
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
25
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
26
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
27
|
+
# SOFTWARE.
|
|
28
|
+
|
|
29
|
+
# .......................................................................................
|
|
30
|
+
import os
|
|
31
|
+
import zipfile
|
|
32
|
+
from .fileobject import FileObject
|
|
33
|
+
|
|
34
|
+
class ZipFile(FileObject):
|
|
35
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
36
|
+
def __init__(self, filename, parent_folder=None, error_template=None, is_verbose=False):
|
|
37
|
+
super().__init__(filename, parent_folder, error_template)
|
|
38
|
+
self.is_verbose = is_verbose
|
|
39
|
+
self._opened_filename = None
|
|
40
|
+
self._encoding = None
|
|
41
|
+
|
|
42
|
+
# --------------------------------------------------------------------------------------------------------
|
|
43
|
+
def save(self, source_path, must_replace=True):
|
|
44
|
+
"""
|
|
45
|
+
Compress a folder (including subfolders) into a .zip file.
|
|
46
|
+
|
|
47
|
+
Args:
|
|
48
|
+
output_zip_path (str): The path where the .zip file will be saved.
|
|
49
|
+
"""
|
|
50
|
+
dest_zip_file = os.path.join(self.parent_folder, self.filename)
|
|
51
|
+
if must_replace:
|
|
52
|
+
if os.path.exists(dest_zip_file):
|
|
53
|
+
os.remove(dest_zip_file)
|
|
54
|
+
|
|
55
|
+
# Create the zip file
|
|
56
|
+
with zipfile.ZipFile(dest_zip_file, 'w', zipfile.ZIP_DEFLATED) as zipf:
|
|
57
|
+
for root, dirs, files in os.walk(source_path):
|
|
58
|
+
for file in files:
|
|
59
|
+
abs_path = os.path.join(root, file)
|
|
60
|
+
# Get relative path for proper folder structure inside zip
|
|
61
|
+
rel_path = os.path.relpath(abs_path, start=source_path)
|
|
62
|
+
zipf.write(abs_path, rel_path)
|
|
63
|
+
|
|
64
|
+
# --------------------------------------------------------------------------------------------------------
|
|
65
|
+
def extract(self, dest_path, has_progress_bar=True, is_verbose=True):
|
|
66
|
+
"""
|
|
67
|
+
Extracts a .zip file into the specified folder.
|
|
68
|
+
|
|
69
|
+
Args:
|
|
70
|
+
zip_filename (str): Path to the .zip file.
|
|
71
|
+
destination_folder (str): Folder where the contents should be extracted.
|
|
72
|
+
"""
|
|
73
|
+
source_zip_file = os.path.join(self.parent_folder, self.filename)
|
|
74
|
+
|
|
75
|
+
# Ensure the zip file exists
|
|
76
|
+
if not os.path.isfile(source_zip_file):
|
|
77
|
+
raise FileNotFoundError(f"Zip file not found: {source_zip_file}")
|
|
78
|
+
|
|
79
|
+
# Ensure destination directory exists
|
|
80
|
+
os.makedirs(dest_path, exist_ok=True)
|
|
81
|
+
|
|
82
|
+
# Extract the zip file
|
|
83
|
+
with zipfile.ZipFile(source_zip_file, 'r') as zip_ref:
|
|
84
|
+
file_list = zip_ref.infolist()
|
|
85
|
+
total_files = len(file_list)
|
|
86
|
+
|
|
87
|
+
oIterator = file_list
|
|
88
|
+
if is_verbose and has_progress_bar:
|
|
89
|
+
from tqdm import tqdm
|
|
90
|
+
oIterator = tqdm(file_list, total=total_files, desc="Extracting", unit="file")
|
|
91
|
+
|
|
92
|
+
for file_info in oIterator:
|
|
93
|
+
zip_ref.extract(file_info, dest_path)
|
|
94
|
+
if is_verbose:
|
|
95
|
+
print(f" |_ Extracted '{source_zip_file}' to '{dest_path}'")
|
|
96
|
+
# ----------------------------------------------------------------------------------
|
radnn/system/filestore.py
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
# ______________________________________________________________________________________
|
|
7
7
|
# ......................................................................................
|
|
8
8
|
|
|
9
|
-
# Copyright (c) 2018-
|
|
9
|
+
# Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
|
|
10
10
|
|
|
11
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
@@ -31,6 +31,9 @@ import os
|
|
|
31
31
|
import shutil
|
|
32
32
|
import glob
|
|
33
33
|
import sys
|
|
34
|
+
import zipfile
|
|
35
|
+
from typing import Any, Type
|
|
36
|
+
import importlib.util
|
|
34
37
|
if (sys.version_info.major == 3) and (sys.version_info.minor <= 7):
|
|
35
38
|
import pickle5 as pickle
|
|
36
39
|
else:
|
|
@@ -40,11 +43,13 @@ from radnn.system.files import JSONFile
|
|
|
40
43
|
from radnn.system.files import PickleFile
|
|
41
44
|
from radnn.system.files import TextFile
|
|
42
45
|
from radnn.system.files import CSVFile
|
|
43
|
-
from radnn.
|
|
44
|
-
|
|
45
|
-
from radnn.system.files.imgfile import PNGFile
|
|
46
|
+
from radnn.system.files import ZipFile
|
|
47
|
+
from radnn.errors import *
|
|
46
48
|
|
|
47
49
|
|
|
50
|
+
_is_opencv_installed = importlib.util.find_spec("cv2") is not None
|
|
51
|
+
if _is_opencv_installed:
|
|
52
|
+
from radnn.system.files.imgfile import PNGFile
|
|
48
53
|
|
|
49
54
|
|
|
50
55
|
# =======================================================================================================================
|
|
@@ -60,13 +65,13 @@ class FileStore(object):
|
|
|
60
65
|
else:
|
|
61
66
|
os.makedirs(self.base_folder)
|
|
62
67
|
|
|
63
|
-
self.is_verbose
|
|
64
|
-
self.json = JSONFile(None, parent_folder=self.base_folder)
|
|
65
|
-
self.obj = PickleFile(None, parent_folder=self.base_folder)
|
|
66
|
-
self.text = TextFile(None, parent_folder=self.base_folder)
|
|
67
|
-
self.csv = CSVFile(None, parent_folder=self.base_folder)
|
|
68
|
-
if
|
|
69
|
-
self.img = PNGFile(None, parent_folder=base_folder)
|
|
68
|
+
self.is_verbose = is_verbose
|
|
69
|
+
self.json: JSONFile = JSONFile(None, parent_folder=self.base_folder)
|
|
70
|
+
self.obj: PickleFile = PickleFile(None, parent_folder=self.base_folder)
|
|
71
|
+
self.text: TextFile = TextFile(None, parent_folder=self.base_folder)
|
|
72
|
+
self.csv: CSVFile = CSVFile(None, parent_folder=self.base_folder)
|
|
73
|
+
if _is_opencv_installed:
|
|
74
|
+
self.img: PNGFile = PNGFile(None, parent_folder=base_folder)
|
|
70
75
|
self.donefs = None
|
|
71
76
|
#................................................................................
|
|
72
77
|
# --------------------------------------------------------------------------------------------------------
|
|
@@ -113,6 +118,48 @@ class FileStore(object):
|
|
|
113
118
|
file_name += file_ext
|
|
114
119
|
return os.path.join(self.absolute_path, file_name)
|
|
115
120
|
# --------------------------------------------------------------------------------------------------------
|
|
121
|
+
def get_file_kind(self, file_name: str):
|
|
122
|
+
name, ext = os.path.splitext(file_name)
|
|
123
|
+
ext = ext.lower()
|
|
124
|
+
if ext == ".pkl":
|
|
125
|
+
return "obj"
|
|
126
|
+
elif ext == ".txt":
|
|
127
|
+
return "text"
|
|
128
|
+
elif ext == ".json":
|
|
129
|
+
return "json"
|
|
130
|
+
elif ext == ".png":
|
|
131
|
+
return "img"
|
|
132
|
+
elif ext == ".zip":
|
|
133
|
+
return "zip"
|
|
134
|
+
else:
|
|
135
|
+
return "?"
|
|
136
|
+
# --------------------------------------------------------------------------------------------------------
|
|
137
|
+
def artifact(self, file_name: str, kind: str | None = None) -> Any:
|
|
138
|
+
if kind is None:
|
|
139
|
+
kind = self.get_file_kind(file_name)
|
|
140
|
+
|
|
141
|
+
oFile = None
|
|
142
|
+
if kind == "obj":
|
|
143
|
+
oFile = PickleFile(file_name, parent_folder=self.base_folder)
|
|
144
|
+
elif kind == "text":
|
|
145
|
+
oFile = TextFile(file_name, parent_folder=self.base_folder)
|
|
146
|
+
elif kind == "json":
|
|
147
|
+
oFile = JSONFile(file_name, parent_folder=self.base_folder)
|
|
148
|
+
elif kind == "csv":
|
|
149
|
+
oFile = CSVFile(file_name, parent_folder=self.base_folder)
|
|
150
|
+
elif kind == "img":
|
|
151
|
+
if _is_opencv_installed:
|
|
152
|
+
oFile = PNGFile(file_name, parent_folder=self.base_folder)
|
|
153
|
+
else:
|
|
154
|
+
print("Open CV is not installed") #TODO: Support PIL
|
|
155
|
+
elif kind == "zip":
|
|
156
|
+
oFile = ZipFile(file_name, parent_folder=self.base_folder)
|
|
157
|
+
|
|
158
|
+
if oFile is None:
|
|
159
|
+
raise Exception(FILESTORE_DATAFILE_KIND_NOT_SUPPORTED % str(kind))
|
|
160
|
+
|
|
161
|
+
return oFile
|
|
162
|
+
# --------------------------------------------------------------------------------------------------------
|
|
116
163
|
def entries(self):
|
|
117
164
|
return os.listdir(self.base_folder)
|
|
118
165
|
# --------------------------------------------------------------------------------------------------------
|
|
@@ -173,6 +220,95 @@ class FileStore(object):
|
|
|
173
220
|
if is_full_path and (sFileName is not None):
|
|
174
221
|
sFileName = os.path.join(self.donefs.base_folder, sFileName)
|
|
175
222
|
return sFileName
|
|
223
|
+
|
|
224
|
+
# --------------------------------------------------------------------------------------------------------
|
|
225
|
+
def move_to(self, source_files: str | list, fs):
|
|
226
|
+
oFiles = source_files
|
|
227
|
+
if isinstance(source_files, str):
|
|
228
|
+
# A pattern is given
|
|
229
|
+
if "*" in source_files:
|
|
230
|
+
oFiles = self.list_files(source_files, is_full_path=False)
|
|
231
|
+
else:
|
|
232
|
+
oFiles = [source_files]
|
|
233
|
+
oDestFiles = []
|
|
234
|
+
for sFile in oFiles:
|
|
235
|
+
sDestFile = os.path.join(fs.absolute_path, sFile)
|
|
236
|
+
oDestFiles.append(sDestFile)
|
|
237
|
+
shutil.move(os.path.join(self.absolute_path, sFile),sDestFile)
|
|
238
|
+
return oDestFiles
|
|
239
|
+
|
|
240
|
+
# --------------------------------------------------------------------------------------------------------
|
|
241
|
+
def copy_to(self, source_files: str | list, fs):
|
|
242
|
+
oFiles = source_files
|
|
243
|
+
if isinstance(source_files, str):
|
|
244
|
+
# A pattern is given
|
|
245
|
+
if "*" in source_files:
|
|
246
|
+
oFiles = self.list_files(source_files, is_full_path=False)
|
|
247
|
+
else:
|
|
248
|
+
oFiles = [source_files]
|
|
249
|
+
oDestFiles = []
|
|
250
|
+
for sFile in oFiles:
|
|
251
|
+
sDestFile = os.path.join(fs.absolute_path, sFile)
|
|
252
|
+
oDestFiles.append(sDestFile)
|
|
253
|
+
shutil.copy(os.path.join(self.absolute_path, sFile),sDestFile)
|
|
254
|
+
return oDestFiles
|
|
255
|
+
|
|
256
|
+
# --------------------------------------------------------------------------------------------------------
|
|
257
|
+
def remove_existing(self, filename):
|
|
258
|
+
sFileName = self.file(filename)
|
|
259
|
+
if os.path.exists(sFileName):
|
|
260
|
+
os.remove(sFileName)
|
|
261
|
+
# --------------------------------------------------------------------------------------------------------
|
|
262
|
+
def compress_to_zip(self, output_zip_path, must_replace=True):
|
|
263
|
+
"""
|
|
264
|
+
Compress a folder (including subfolders) into a .zip file.
|
|
265
|
+
|
|
266
|
+
Args:
|
|
267
|
+
output_zip_path (str): The path where the .zip file will be saved.
|
|
268
|
+
"""
|
|
269
|
+
if must_replace:
|
|
270
|
+
if os.path.exists(output_zip_path):
|
|
271
|
+
os.remove(output_zip_path)
|
|
272
|
+
|
|
273
|
+
# Create the zip file
|
|
274
|
+
with zipfile.ZipFile(output_zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
|
|
275
|
+
for root, dirs, files in os.walk(self.absolute_path):
|
|
276
|
+
for file in files:
|
|
277
|
+
abs_path = os.path.join(root, file)
|
|
278
|
+
# Get relative path for proper folder structure inside zip
|
|
279
|
+
rel_path = os.path.relpath(abs_path, start=self.absolute_path)
|
|
280
|
+
zipf.write(abs_path, rel_path)
|
|
281
|
+
# --------------------------------------------------------------------------------------------------------
|
|
282
|
+
def extract_from_zip(self, zip_filename, has_progress_bar=True):
|
|
283
|
+
"""
|
|
284
|
+
Extracts a .zip file into the specified folder.
|
|
285
|
+
|
|
286
|
+
Args:
|
|
287
|
+
zip_filename (str): Path to the .zip file.
|
|
288
|
+
destination_folder (str): Folder where the contents should be extracted.
|
|
289
|
+
"""
|
|
290
|
+
# Ensure the zip file exists
|
|
291
|
+
if not os.path.isfile(zip_filename):
|
|
292
|
+
raise FileNotFoundError(f"Zip file not found: {zip_filename}")
|
|
293
|
+
|
|
294
|
+
sDestFolder = self.absolute_path
|
|
295
|
+
# Ensure destination directory exists
|
|
296
|
+
os.makedirs(sDestFolder, exist_ok=True)
|
|
297
|
+
|
|
298
|
+
# Extract the zip file
|
|
299
|
+
with zipfile.ZipFile(zip_filename, 'r') as zip_ref:
|
|
300
|
+
file_list = zip_ref.infolist()
|
|
301
|
+
total_files = len(file_list)
|
|
302
|
+
|
|
303
|
+
oIterator = file_list
|
|
304
|
+
if has_progress_bar:
|
|
305
|
+
from tqdm import tqdm
|
|
306
|
+
oIterator = tqdm(file_list, total=total_files, desc="Extracting", unit="file")
|
|
307
|
+
|
|
308
|
+
for file_info in oIterator:
|
|
309
|
+
zip_ref.extract(file_info, sDestFolder)
|
|
310
|
+
|
|
311
|
+
print(f" |_ Extracted '{zip_filename}' to '{sDestFolder}'")
|
|
176
312
|
# --------------------------------------------------------------------------------------------------------
|
|
177
313
|
def purge_done(self):
|
|
178
314
|
#//TODO: Remove from the current filestore all files that are moved into the .done filestore
|
|
@@ -187,42 +323,6 @@ class FileStore(object):
|
|
|
187
323
|
# ======================================================================================================================
|
|
188
324
|
|
|
189
325
|
|
|
190
|
-
# .................... Tests ....................
|
|
191
|
-
if __name__ == "__main__":
|
|
192
|
-
VISUALIZE = False
|
|
193
|
-
import numpy as np
|
|
194
|
-
|
|
195
|
-
# Lazy initialize a file store folder
|
|
196
|
-
oFS = FileStore("C:\MLData\SomeDataSet")
|
|
197
|
-
|
|
198
|
-
# Serialize data into a sub folder
|
|
199
|
-
oData = np.asarray([1, 2, 3]).astype(np.float32)
|
|
200
|
-
print(oFS.subfs("test"))
|
|
201
|
-
oFS.subfs("test").obj.save(oData, "sample1.pkl")
|
|
202
|
-
|
|
203
|
-
# Get paths
|
|
204
|
-
print("==== Paths ====")
|
|
205
|
-
print(oFS.subfs("test").file("sample1.png"))
|
|
206
|
-
print(oFS.folder("test"))
|
|
207
|
-
|
|
208
|
-
# Deserialize data from a sub folder
|
|
209
|
-
oLoadedData = oFS.subfs("test").obj.load("sample1.pkl")
|
|
210
|
-
print(oLoadedData)
|
|
211
|
-
|
|
212
|
-
# List files
|
|
213
|
-
print("==== Files ====")
|
|
214
|
-
sFiles = oFS.subfs("test").list_files("*.pkl", True)
|
|
215
|
-
print("\n".join(sFiles))
|
|
216
|
-
|
|
217
|
-
# Load a PNG file
|
|
218
|
-
if (is_opencv_installed()):
|
|
219
|
-
oPng = FileStore("C:\\UX").subfs("PNG").img.load("Ideas.png")
|
|
220
|
-
print(oPng.shape)
|
|
221
|
-
|
|
222
|
-
if VISUALIZE:
|
|
223
|
-
import matplotlib.pyplot as plt
|
|
224
|
-
plt.imshow(oPng)
|
|
225
|
-
plt.show()
|
|
226
326
|
|
|
227
327
|
|
|
228
328
|
|
radnn/system/filesystem.py
CHANGED
|
@@ -77,9 +77,9 @@ class FileSystem(object):
|
|
|
77
77
|
|
|
78
78
|
# ...................... | Fields | ......................
|
|
79
79
|
self.setup = dSetup
|
|
80
|
-
self.configs
|
|
81
|
-
self.models
|
|
82
|
-
self.datasets
|
|
80
|
+
self.configs: FileStore = FileStore(config_folder, must_exist=self._must_exist)
|
|
81
|
+
self.models: FileStore = FileStore(model_folder, must_exist=self._must_exist)
|
|
82
|
+
self.datasets: FileStore = FileStore(dataset_folder, must_exist=self._must_exist)
|
|
83
83
|
# ........................................................
|
|
84
84
|
# --------------------------------------------------------------------------------------------------------
|
|
85
85
|
def group(self, group_name):
|
radnn/test/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .tensor_hash import TensorHash
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import numpy as np
|
|
3
|
+
from radnn import mlsys, FileStore
|
|
4
|
+
|
|
5
|
+
if mlsys.framework == "torch":
|
|
6
|
+
import torch
|
|
7
|
+
elif mlsys.framework == "tensorflow":
|
|
8
|
+
import tensorflow as tf
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TensorHash(object):
|
|
12
|
+
def __init__(self, test_fs, filename):
|
|
13
|
+
self.fs: FileStore = test_fs
|
|
14
|
+
self.filename = filename
|
|
15
|
+
self.hashes = []
|
|
16
|
+
self.loaded_hashes = None
|
|
17
|
+
self.is_loaded = False
|
|
18
|
+
|
|
19
|
+
def collect_model_params(self, model):
|
|
20
|
+
if mlsys.framework == "torch":
|
|
21
|
+
for key, value in model.state_dict().items():
|
|
22
|
+
# print(key, value.shape)
|
|
23
|
+
self.collect_hash(value)
|
|
24
|
+
return self
|
|
25
|
+
|
|
26
|
+
def end_collection(self):
|
|
27
|
+
if self.is_loaded:
|
|
28
|
+
if self.compare():
|
|
29
|
+
print(f"[v] {self.filename} Reproducibility for {len(self.hashes)} collected values")
|
|
30
|
+
else:
|
|
31
|
+
print(f"[x] {self.filename} Non deterministic behaviour!")
|
|
32
|
+
else:
|
|
33
|
+
self.save()
|
|
34
|
+
|
|
35
|
+
def collect_hash(self, x: torch.Tensor, normalize_shape=True) -> str:
|
|
36
|
+
if mlsys.framework == "torch":
|
|
37
|
+
return self.calculate_hash_torch(x, normalize_shape)
|
|
38
|
+
elif mlsys.framework == "tensorflow":
|
|
39
|
+
return self.calculate_hash_tf(x, normalize_shape)
|
|
40
|
+
else:
|
|
41
|
+
return ""
|
|
42
|
+
|
|
43
|
+
def calculate_hash_tf(self, x, normalize_shape: bool = True) -> str:
|
|
44
|
+
if not tf.is_tensor(x):
|
|
45
|
+
# allow numpy arrays too if you want:
|
|
46
|
+
if isinstance(x, np.ndarray):
|
|
47
|
+
x = tf.convert_to_tensor(x)
|
|
48
|
+
else:
|
|
49
|
+
raise TypeError("x must be a tf.Tensor (or numpy.ndarray)")
|
|
50
|
+
|
|
51
|
+
t = x # keep as tf.Tensor
|
|
52
|
+
|
|
53
|
+
# Canonicalize shape to (28, 28, 1) if requested
|
|
54
|
+
if normalize_shape:
|
|
55
|
+
if t.shape.rank == 2:
|
|
56
|
+
# (28, 28) -> (28, 28, 1)
|
|
57
|
+
t = tf.expand_dims(t, axis=-1)
|
|
58
|
+
elif t.shape.rank == 3:
|
|
59
|
+
# (1, 28, 28) -> (28, 28, 1) (common in CHW)
|
|
60
|
+
# Only do this if first dim is 1.
|
|
61
|
+
if t.shape[0] == 1:
|
|
62
|
+
t = tf.transpose(t, perm=[1, 2, 0])
|
|
63
|
+
|
|
64
|
+
# Canonicalize dtype
|
|
65
|
+
if t.dtype.is_floating:
|
|
66
|
+
t = tf.clip_by_value(t, 0.0, 1.0)
|
|
67
|
+
t = tf.round(t * 255.0)
|
|
68
|
+
t = tf.cast(t, tf.uint8)
|
|
69
|
+
else:
|
|
70
|
+
t = tf.cast(t, tf.uint8)
|
|
71
|
+
|
|
72
|
+
# Hash raw bytes (ensure deterministic C-order bytes)
|
|
73
|
+
# tf.io.serialize_tensor includes dtype/shape metadata, so we avoid it.
|
|
74
|
+
b = t.numpy().tobytes(order="C")
|
|
75
|
+
sResult = hashlib.sha256(b).hexdigest()
|
|
76
|
+
self.hashes.append(sResult)
|
|
77
|
+
return sResult
|
|
78
|
+
|
|
79
|
+
def calculate_hash_torch(self, x: torch.Tensor, normalize_shape=True) -> str:
|
|
80
|
+
if not isinstance(x, torch.Tensor):
|
|
81
|
+
raise TypeError("x must be a torch.Tensor")
|
|
82
|
+
|
|
83
|
+
# Move to CPU, detach, make contiguous
|
|
84
|
+
t = x.detach().to("cpu").contiguous()
|
|
85
|
+
|
|
86
|
+
# Canonicalize shape to (28, 28, 1) if requested
|
|
87
|
+
if normalize_shape:
|
|
88
|
+
if t.ndim == 2: # (28,28)
|
|
89
|
+
t = t.unsqueeze(-1) # (28,28,1)
|
|
90
|
+
elif t.ndim == 3 and t.shape[0] == 1: # (1,28,28) -> (28,28,1)
|
|
91
|
+
t = t.permute(1, 2, 0).contiguous()
|
|
92
|
+
|
|
93
|
+
# Canonicalize dtype to avoid hash changing across float16/float32/etc.
|
|
94
|
+
# If your data is in [0,1] floats, this will quantize deterministically.
|
|
95
|
+
if t.dtype.is_floating_point:
|
|
96
|
+
t = (t.clamp(0, 1) * 255.0).round().to(torch.uint8)
|
|
97
|
+
else:
|
|
98
|
+
# If already uint8/int, you can keep it; here we convert to uint8 for consistency.
|
|
99
|
+
t = t.to(torch.uint8)
|
|
100
|
+
|
|
101
|
+
# Hash raw bytes
|
|
102
|
+
b = t.numpy().tobytes(order="C")
|
|
103
|
+
sResult = hashlib.sha256(b).hexdigest()
|
|
104
|
+
self.hashes.append(sResult)
|
|
105
|
+
return sResult
|
|
106
|
+
|
|
107
|
+
def save(self, filename=None):
|
|
108
|
+
if filename is not None:
|
|
109
|
+
self.filename = filename
|
|
110
|
+
|
|
111
|
+
self.fs.text.save(self.hashes, self.filename)
|
|
112
|
+
return self
|
|
113
|
+
|
|
114
|
+
def load(self, filename=None):
|
|
115
|
+
if filename is not None:
|
|
116
|
+
self.filename = filename
|
|
117
|
+
|
|
118
|
+
self.loaded_hashes = self.fs.text.load(self.filename)
|
|
119
|
+
self.is_loaded = self.loaded_hashes is not None
|
|
120
|
+
return self
|
|
121
|
+
|
|
122
|
+
def compare(self):
|
|
123
|
+
bResult = True
|
|
124
|
+
if self.loaded_hashes is not None:
|
|
125
|
+
for nIndex, sHash in enumerate(self.hashes):
|
|
126
|
+
if sHash != self.hashes[nIndex]:
|
|
127
|
+
bResult = False
|
|
128
|
+
break
|
|
129
|
+
|
|
130
|
+
return bResult
|
radnn/utils.py
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
# ______________________________________________________________________________________
|
|
7
7
|
# ......................................................................................
|
|
8
8
|
|
|
9
|
-
# Copyright (c) 2024-
|
|
9
|
+
# Copyright (c) 2024-2026 Pantelis I. Kaplanoglou
|
|
10
10
|
|
|
11
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
@@ -28,6 +28,7 @@
|
|
|
28
28
|
|
|
29
29
|
# .......................................................................................
|
|
30
30
|
import numpy as np
|
|
31
|
+
import json
|
|
31
32
|
import time
|
|
32
33
|
import hashlib
|
|
33
34
|
import zlib
|
|
@@ -35,8 +36,21 @@ import contextlib
|
|
|
35
36
|
|
|
36
37
|
phi=(1.0+np.sqrt(5.0))/2.0
|
|
37
38
|
|
|
39
|
+
# ======================================================================================================================
|
|
40
|
+
class classproperty(property):
|
|
41
|
+
def __get__(self, obj, owner):
|
|
42
|
+
return self.fget(owner)
|
|
43
|
+
# ======================================================================================================================
|
|
38
44
|
|
|
39
|
-
|
|
45
|
+
|
|
46
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
47
|
+
def to_json(obj, is_sorted_keys=False, is_utf8=False):
|
|
48
|
+
if isinstance(obj, dict):
|
|
49
|
+
sJSON = json.dumps(obj, sort_keys=is_sorted_keys, indent=4, ensure_ascii=is_utf8)
|
|
50
|
+
else:
|
|
51
|
+
sJSON = json.dumps(obj, default=lambda o: obj.__dict__, sort_keys=is_sorted_keys, indent=4, ensure_ascii=is_utf8)
|
|
52
|
+
return sJSON
|
|
53
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
40
54
|
'''
|
|
41
55
|
Checks if the p_sSettingsName is inside the settings dictionary p_dConfig
|
|
42
56
|
and returns its value, otherwise the p_oDefault value
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: radnn
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Rapid Deep Neural Networks
|
|
5
|
+
Author-email: "Pantelis I. Kaplanoglou" <pikaplanoglou@ihu.gr>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/pikaplan/radnn
|
|
8
|
+
Project-URL: Documentation, https://radnn.readthedocs.io/
|
|
9
|
+
Classifier: Intended Audience :: Science/Research
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Programming Language :: Python
|
|
12
|
+
Classifier: Topic :: Software Development
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
15
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE.txt
|
|
25
|
+
Requires-Dist: numpy>=1.26.4
|
|
26
|
+
Requires-Dist: matplotlib>=3.8.4
|
|
27
|
+
Requires-Dist: pandas>=2.2.1
|
|
28
|
+
Requires-Dist: scikit-learn>=1.4.2
|
|
29
|
+
Requires-Dist: tqdm>=4.67.1
|
|
30
|
+
Dynamic: license-file
|