radnn 0.0.3__tar.gz → 0.0.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {radnn-0.0.3/src/radnn.egg-info → radnn-0.0.5}/PKG-INFO +1 -1
- {radnn-0.0.3 → radnn-0.0.5}/pyproject.toml +1 -1
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn/__init__.py +2 -1
- radnn-0.0.5/src/radnn/core.py +74 -0
- radnn-0.0.5/src/radnn/evaluation/__init__.py +1 -0
- radnn-0.0.5/src/radnn/evaluation/evaluate_classification.py +113 -0
- radnn-0.0.5/src/radnn/experiment/__init__.py +4 -0
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn/experiment/ml_experiment_config.py +85 -22
- radnn-0.0.5/src/radnn/experiment/ml_experiment_env.py +223 -0
- radnn-0.0.5/src/radnn/plots/__init__.py +4 -0
- radnn-0.0.5/src/radnn/plots/plot_confusion_matrix.py +54 -0
- radnn-0.0.5/src/radnn/plots/plot_learning_curve.py +73 -0
- radnn-0.0.5/src/radnn/plots/plot_roc.py +75 -0
- radnn-0.0.5/src/radnn/plots/plot_voronoi_2d.py +95 -0
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn/system/__init__.py +1 -0
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn/system/filesystem.py +44 -22
- radnn-0.0.5/src/radnn/system/hosts/__init__.py +10 -0
- radnn-0.0.5/src/radnn/system/hosts/colab_host.py +52 -0
- radnn-0.0.5/src/radnn/system/hosts/linux_host.py +3 -0
- radnn-0.0.5/src/radnn/system/hosts/windows_host.py +41 -0
- radnn-0.0.5/src/radnn/system/tee_logger.py +60 -0
- {radnn-0.0.3 → radnn-0.0.5/src/radnn.egg-info}/PKG-INFO +1 -1
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn.egg-info/SOURCES.txt +16 -1
- {radnn-0.0.3 → radnn-0.0.5}/tests/test_config.py +3 -2
- radnn-0.0.5/tests/test_experiment_env.py +42 -0
- radnn-0.0.5/tests/test_filestore.py +4 -0
- {radnn-0.0.3 → radnn-0.0.5}/tests/test_filesystem.py +1 -1
- radnn-0.0.5/tests/test_hosts.py +2 -0
- radnn-0.0.3/src/radnn/core.py +0 -45
- radnn-0.0.3/src/radnn/experiment/__init__.py +0 -3
- radnn-0.0.3/tests/test_filestore.py +0 -3
- {radnn-0.0.3 → radnn-0.0.5}/LICENSE.txt +0 -0
- {radnn-0.0.3 → radnn-0.0.5}/README.md +0 -0
- {radnn-0.0.3 → radnn-0.0.5}/setup.cfg +0 -0
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn/system/files/__init__.py +0 -0
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn/system/files/csvfile.py +0 -0
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn/system/files/fileobject.py +0 -0
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn/system/files/imgfile.py +0 -0
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn/system/files/jsonfile.py +0 -0
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn/system/files/picklefile.py +0 -0
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn/system/files/textfile.py +0 -0
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn/system/filestore.py +0 -0
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn.egg-info/dependency_links.txt +0 -0
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn.egg-info/requires.txt +0 -0
- {radnn-0.0.3 → radnn-0.0.5}/src/radnn.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import socket
|
|
3
|
+
import platform
|
|
4
|
+
import subprocess
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
import importlib.util
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
10
|
+
def is_opencv_installed():
|
|
11
|
+
return importlib.util.find_spec("cv2") is not None
|
|
12
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
18
|
+
def system_name() -> str:
|
|
19
|
+
return MLInfrastructure.host_name(False)
|
|
20
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
21
|
+
def now_iso():
|
|
22
|
+
return datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
|
|
23
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
24
|
+
def shell_command_output(command_string):
|
|
25
|
+
oOutput = subprocess.check_output(command_string, shell=True)
|
|
26
|
+
oOutputLines = oOutput.decode().splitlines()
|
|
27
|
+
|
|
28
|
+
oResult = []
|
|
29
|
+
for sLine in oOutputLines:
|
|
30
|
+
oResult.append(sLine)
|
|
31
|
+
|
|
32
|
+
return oResult
|
|
33
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
#TODO: macOS support
|
|
41
|
+
|
|
42
|
+
class MLInfrastructure(object):
|
|
43
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
44
|
+
@classmethod
|
|
45
|
+
def is_linux(cls):
|
|
46
|
+
return not (cls.is_windows or cls.is_colab)
|
|
47
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
48
|
+
@classmethod
|
|
49
|
+
def is_windows(cls):
|
|
50
|
+
sPlatform = platform.system()
|
|
51
|
+
return (sPlatform == "Windows")
|
|
52
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
53
|
+
@classmethod
|
|
54
|
+
def is_colab(cls):
|
|
55
|
+
return "google.colab" in sys.modules
|
|
56
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
57
|
+
@classmethod
|
|
58
|
+
def host_name(cls, is_using_ip_address=True) -> str:
|
|
59
|
+
sPlatform = platform.system()
|
|
60
|
+
sHostName = socket.gethostname()
|
|
61
|
+
sIPAddress = socket.gethostbyname(sHostName)
|
|
62
|
+
|
|
63
|
+
bIsColab = "google.colab" in sys.modules
|
|
64
|
+
if bIsColab:
|
|
65
|
+
sResult = "(colab)"
|
|
66
|
+
if is_using_ip_address:
|
|
67
|
+
sResult += "-" + sIPAddress
|
|
68
|
+
else:
|
|
69
|
+
if sPlatform == "Windows":
|
|
70
|
+
sResult = "(windows)-" + sHostName
|
|
71
|
+
else:
|
|
72
|
+
sResult = "(linux)-" + sHostName
|
|
73
|
+
return sResult
|
|
74
|
+
# ----------------------------------------------------------------------------------------------------------------------
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .evaluate_classification import EvaluateClassification
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# ......................................................................................
|
|
2
|
+
# MIT License
|
|
3
|
+
|
|
4
|
+
# Copyright (c) 2020-2025 Pantelis I. Kaplanoglou
|
|
5
|
+
|
|
6
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
# in the Software without restriction, including without limitation the rights
|
|
9
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
# furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
14
|
+
# copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
|
+
# SOFTWARE.
|
|
23
|
+
|
|
24
|
+
# ......................................................................................
|
|
25
|
+
|
|
26
|
+
import numpy as np
|
|
27
|
+
from sklearn import metrics
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# ==============================================================================================================================
|
|
31
|
+
class EvaluateClassification(object):
|
|
32
|
+
# --------------------------------------------------------------------------------------------------------------
|
|
33
|
+
def __init__(self, actual_classes, predicted_classes, probabilities=None):
|
|
34
|
+
self.actual_classes = actual_classes
|
|
35
|
+
self.predicted_classes = predicted_classes
|
|
36
|
+
self.confusion_matrix = np.asarray(metrics.confusion_matrix(self.actual_classes, self.predicted_classes))
|
|
37
|
+
|
|
38
|
+
self.accuracy = metrics.accuracy_score(self.actual_classes, self.predicted_classes)
|
|
39
|
+
self.precision, self.recall, self.f1score, self.support = metrics.precision_recall_fscore_support(
|
|
40
|
+
self.actual_classes, self.predicted_classes, average=None)
|
|
41
|
+
self.average_precision, self.average_recall, self.average_f1score, self.average_support = metrics.precision_recall_fscore_support(
|
|
42
|
+
self.actual_classes, self.predicted_classes, average='weighted')
|
|
43
|
+
|
|
44
|
+
if probabilities is not None:
|
|
45
|
+
self.auc = metrics.roc_auc_score(actual_classes, probabilities)
|
|
46
|
+
else:
|
|
47
|
+
self.auc = None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
self.true_neg = self.confusion_matrix[0][0]
|
|
51
|
+
self.false_pos = self.confusion_matrix[0][1]
|
|
52
|
+
self.false_neg = self.confusion_matrix[1][0]
|
|
53
|
+
self.true_pos = self.confusion_matrix[1][1]
|
|
54
|
+
|
|
55
|
+
self.class_count = None
|
|
56
|
+
# --------------------------------------------------------------------------------------------------------------
|
|
57
|
+
def print_confusion_matrix(self):
|
|
58
|
+
nSize = len(self.confusion_matrix[0])
|
|
59
|
+
print(" Predicted ")
|
|
60
|
+
print(" --" + "-" * 5 * nSize)
|
|
61
|
+
sLabel = "Actual"
|
|
62
|
+
for nIndex, nRow in enumerate(self.confusion_matrix):
|
|
63
|
+
print(" %s | %s |" % (sLabel, " ".join(["%4d" % x for x in nRow])))
|
|
64
|
+
if nIndex == 0:
|
|
65
|
+
sLabel = " " * len(sLabel)
|
|
66
|
+
print(" --" + "-" * 5 * nSize)
|
|
67
|
+
print()
|
|
68
|
+
# --------------------------------------------------------------------------------------------------------------
|
|
69
|
+
def format_series_as_pc(self, metric_series):
|
|
70
|
+
oValStr = []
|
|
71
|
+
for x in metric_series:
|
|
72
|
+
sX = f"{x*100.0:.2f}"
|
|
73
|
+
oValStr.append(sX)
|
|
74
|
+
|
|
75
|
+
oValues = [f"{x:^7}" for x in oValStr]
|
|
76
|
+
return " |".join(oValues)
|
|
77
|
+
# --------------------------------------------------------------------------------------------------------------
|
|
78
|
+
def print_per_class(self, class_names=None):
|
|
79
|
+
if class_names is not None:
|
|
80
|
+
nClassCount = len(class_names.keys())
|
|
81
|
+
oClasses = [f"{class_names[x]:7}" for x in list(range(nClassCount))]
|
|
82
|
+
else:
|
|
83
|
+
oClasses = sorted(np.unique(self.actual_classes))
|
|
84
|
+
nClassCount = len(oClasses)
|
|
85
|
+
oClasses = [f"{x:^7}" for x in oClasses]
|
|
86
|
+
self.class_count = nClassCount
|
|
87
|
+
|
|
88
|
+
sClasses = " |".join(oClasses)
|
|
89
|
+
nRepeat = 28 + (7+2)*self.class_count
|
|
90
|
+
print(f" |{sClasses}|")
|
|
91
|
+
print("-"*nRepeat)
|
|
92
|
+
print(f"Per Class Recall % |{self.format_series_as_pc(self.recall[:])}|")
|
|
93
|
+
print(f"Per Class Precision % |{self.format_series_as_pc(self.precision[:])}|")
|
|
94
|
+
print("-" * nRepeat)
|
|
95
|
+
# --------------------------------------------------------------------------------------------------------------
|
|
96
|
+
def print_overall(self):
|
|
97
|
+
print(f"Weighted Average Recall % :{self.average_recall*100.0:.3f}")
|
|
98
|
+
print(f"Weighted Average Precision %:{self.average_precision*100.0:.3f}")
|
|
99
|
+
print(f"Accuracy % :{self.accuracy*100.0 :.3f}")
|
|
100
|
+
print(f"Average F1 Score % :{self.average_f1score*100.0:.3f}")
|
|
101
|
+
if (self.class_count == 2) and (self.auc is not None):
|
|
102
|
+
print(f"Area Under the Curve (AUC):{self.auc:.4f}")
|
|
103
|
+
print()
|
|
104
|
+
# --------------------------------------------------------------------------------------------------------------
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
# ==============================================================================================================================
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
# alias for compatibility
|
|
111
|
+
class CEvaluator(EvaluateClassification):
|
|
112
|
+
pass
|
|
113
|
+
|
|
@@ -25,31 +25,88 @@
|
|
|
25
25
|
|
|
26
26
|
import os
|
|
27
27
|
import json
|
|
28
|
+
import re
|
|
29
|
+
from datetime import datetime
|
|
28
30
|
|
|
31
|
+
from radnn.system import FileSystem
|
|
29
32
|
|
|
30
33
|
# --------------------------------------------------------------------------------------
|
|
31
|
-
def
|
|
34
|
+
def legacy_model_code(config_dict):
|
|
35
|
+
if "ModelName" in config_dict:
|
|
36
|
+
sCode = config_dict["ModelName"]
|
|
37
|
+
if "ModelVariation" in config_dict:
|
|
38
|
+
sCode += "_" + config_dict["ModelVariation"]
|
|
39
|
+
if "ExperimentNumber" in config_dict:
|
|
40
|
+
sCode = sCode + "_%02d" % config_dict["ExperimentNumber"]
|
|
41
|
+
else:
|
|
42
|
+
raise Exception("Invalid experiment configuration. Needs at least the key 'ModelName'.")
|
|
43
|
+
return sCode
|
|
44
|
+
# --------------------------------------------------------------------------------------
|
|
45
|
+
def get_experiment_code(config_dict):
|
|
32
46
|
if ("Experiment.BaseName" in config_dict) and ("Experiment.Number" in config_dict):
|
|
33
47
|
sBaseName = config_dict["Experiment.BaseName"]
|
|
34
48
|
nNumber = int(config_dict["Experiment.Number"])
|
|
35
49
|
sVariation = None
|
|
36
50
|
if "Experiment.Variation" in config_dict:
|
|
37
|
-
sVariation = config_dict["Experiment.Variation"]
|
|
51
|
+
sVariation = str(config_dict["Experiment.Variation"])
|
|
38
52
|
nFoldNumber = None
|
|
39
53
|
if "Experiment.FoldNumber" in config_dict:
|
|
40
|
-
nFoldNumber = config_dict["Experiment.FoldNumber"]
|
|
54
|
+
nFoldNumber = int(config_dict["Experiment.FoldNumber"])
|
|
41
55
|
|
|
42
|
-
sCode = "
|
|
56
|
+
sCode = f"{sBaseName}_{nNumber:02d}"
|
|
43
57
|
if sVariation is not None:
|
|
44
|
-
sCode += "
|
|
58
|
+
sCode += "." + sVariation
|
|
45
59
|
if nFoldNumber is not None:
|
|
46
|
-
sCode += "
|
|
60
|
+
sCode += f"-{nFoldNumber:02d}"
|
|
47
61
|
else:
|
|
48
62
|
raise Exception("Invalid experiment configuration. Needs at least two keys 'Experiment.BaseName'\n"
|
|
49
63
|
+ "and `Experiment.Number`.")
|
|
50
64
|
|
|
51
65
|
return sCode
|
|
52
66
|
# --------------------------------------------------------------------------------------
|
|
67
|
+
def get_experiment_code_ex(base_name, number, variation=None, fold_number=None):
|
|
68
|
+
if (base_name is not None) and (number is not None):
|
|
69
|
+
nNumber = int(number)
|
|
70
|
+
sVariation = None
|
|
71
|
+
if variation is not None:
|
|
72
|
+
sVariation = str(variation)
|
|
73
|
+
nFoldNumber = None
|
|
74
|
+
if fold_number is not None:
|
|
75
|
+
nFoldNumber = int(fold_number)
|
|
76
|
+
|
|
77
|
+
sCode = f"{base_name}_{nNumber:02d}"
|
|
78
|
+
if variation is not None:
|
|
79
|
+
sCode += "." + sVariation
|
|
80
|
+
if nFoldNumber is not None:
|
|
81
|
+
sCode += f"-{nFoldNumber:02d}"
|
|
82
|
+
else:
|
|
83
|
+
raise Exception("Invalid experiment code parts. Needs a base name and a number.")
|
|
84
|
+
|
|
85
|
+
return sCode
|
|
86
|
+
# --------------------------------------------------------------------------------------
|
|
87
|
+
def experiment_number_and_variation(experiment_code):
|
|
88
|
+
if type(experiment_code) == int:
|
|
89
|
+
nNumber = int(experiment_code)
|
|
90
|
+
sVariation = None
|
|
91
|
+
else:
|
|
92
|
+
sParts = experiment_code.split(".")
|
|
93
|
+
nNumber = int(sParts[0])
|
|
94
|
+
if len(sParts) > 1:
|
|
95
|
+
sVariation = sParts[1]
|
|
96
|
+
else:
|
|
97
|
+
sVariation = None
|
|
98
|
+
|
|
99
|
+
return nNumber, sVariation
|
|
100
|
+
# --------------------------------------------------------------------------------------
|
|
101
|
+
def experiment_code_and_timestamp(filename):
|
|
102
|
+
sName, _ = os.path.splitext(os.path.split(filename)[1])
|
|
103
|
+
sParts = re.split(r"_", sName, 2)
|
|
104
|
+
sISODate = f"{sParts[0]}T{sParts[1][0:2]}:{sParts[1][2:4]}:{sParts[1][4:6]}"
|
|
105
|
+
sExperimentCode = sParts[2]
|
|
106
|
+
dRunTimestamp = datetime.fromisoformat(sISODate)
|
|
107
|
+
return sExperimentCode, dRunTimestamp
|
|
108
|
+
# --------------------------------------------------------------------------------------
|
|
109
|
+
|
|
53
110
|
|
|
54
111
|
|
|
55
112
|
|
|
@@ -58,27 +115,27 @@ def experiment_code(config_dict):
|
|
|
58
115
|
# =========================================================================================================================
|
|
59
116
|
class MLExperimentConfig(dict):
|
|
60
117
|
# --------------------------------------------------------------------------------------
|
|
61
|
-
def __init__(self, filename=None,
|
|
62
|
-
self["Experiment.BaseName"] =
|
|
118
|
+
def __init__(self, filename=None, base_name=None, number=None, variation=None, fold_number=None, hyperparams=None):
|
|
119
|
+
self["Experiment.BaseName"] = base_name
|
|
63
120
|
self.filename = filename
|
|
64
|
-
if filename is not None:
|
|
121
|
+
if self.filename is not None:
|
|
65
122
|
self.load()
|
|
66
123
|
|
|
67
|
-
if
|
|
68
|
-
self["Experiment.Number"] =
|
|
69
|
-
if
|
|
70
|
-
self["Experiment.Variation"] =
|
|
71
|
-
if
|
|
72
|
-
self["Experiment.FoldNumber"] =
|
|
124
|
+
if number is not None:
|
|
125
|
+
self["Experiment.Number"] = number
|
|
126
|
+
if variation is not None:
|
|
127
|
+
self["Experiment.Variation"] = variation
|
|
128
|
+
if fold_number is not None:
|
|
129
|
+
self["Experiment.FoldNumber"] = fold_number
|
|
73
130
|
|
|
74
131
|
if hyperparams is not None:
|
|
75
132
|
self.assign(hyperparams)
|
|
76
133
|
# --------------------------------------------------------------------------------------
|
|
77
134
|
@property
|
|
78
135
|
def experiment_code(self):
|
|
79
|
-
return
|
|
136
|
+
return get_experiment_code(self)
|
|
80
137
|
# --------------------------------------------------------------------------------------
|
|
81
|
-
def load(self, filename, must_exist=False):
|
|
138
|
+
def load(self, filename=None, must_exist=False):
|
|
82
139
|
if filename is None:
|
|
83
140
|
filename = self.filename
|
|
84
141
|
|
|
@@ -118,16 +175,22 @@ class MLExperimentConfig(dict):
|
|
|
118
175
|
return self
|
|
119
176
|
|
|
120
177
|
# --------------------------------------------------------------------------------------
|
|
121
|
-
def save_config(self,
|
|
178
|
+
def save_config(self, fs, filename_only=None):
|
|
179
|
+
if isinstance(fs, FileSystem):
|
|
180
|
+
fs = fs.configs
|
|
181
|
+
|
|
122
182
|
if filename_only is None:
|
|
123
|
-
filename_only =
|
|
183
|
+
filename_only = get_experiment_code(self)
|
|
124
184
|
|
|
125
|
-
sFileName =
|
|
185
|
+
sFileName = fs.file(filename_only + ".json")
|
|
126
186
|
return self.save(sFileName)
|
|
127
187
|
|
|
128
188
|
# --------------------------------------------------------------------------------------
|
|
129
|
-
def load_config(self,
|
|
130
|
-
|
|
189
|
+
def load_config(self, fs, filename_only):
|
|
190
|
+
if isinstance(fs, FileSystem):
|
|
191
|
+
fs = fs.configs
|
|
192
|
+
|
|
193
|
+
sFileName = fs.file(filename_only + ".json")
|
|
131
194
|
return self.load(sFileName)
|
|
132
195
|
# --------------------------------------------------------------------------------------
|
|
133
196
|
def setDefaults(self):
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
# ......................................................................................
|
|
2
|
+
# MIT License
|
|
3
|
+
|
|
4
|
+
# Copyright (c) 2023-2025 Pantelis I. Kaplanoglou
|
|
5
|
+
|
|
6
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
# in the Software without restriction, including without limitation the rights
|
|
9
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
# furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
14
|
+
# copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
|
+
# SOFTWARE.
|
|
23
|
+
|
|
24
|
+
# ......................................................................................
|
|
25
|
+
|
|
26
|
+
import os
|
|
27
|
+
import shutil
|
|
28
|
+
import sys
|
|
29
|
+
import re
|
|
30
|
+
|
|
31
|
+
from radnn.experiment import MLExperimentConfig
|
|
32
|
+
from radnn.experiment import get_experiment_code_ex, experiment_number_and_variation, experiment_code_and_timestamp
|
|
33
|
+
from radnn.core import now_iso
|
|
34
|
+
from radnn.system import FileSystem, FileStore
|
|
35
|
+
from radnn.system.tee_logger import TeeLogger
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class MLExperimentEnv(dict):
|
|
40
|
+
|
|
41
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
42
|
+
@classmethod
|
|
43
|
+
def experiment_filename_split(cls, filename):
|
|
44
|
+
sTryFileName, sTryExt = os.path.splitext(filename)
|
|
45
|
+
bIsVariationAndFold = "-" in sTryExt
|
|
46
|
+
if bIsVariationAndFold:
|
|
47
|
+
#LREXPLAINET22_MNIST_64.1-01
|
|
48
|
+
sMainParts = filename.split("-")
|
|
49
|
+
assert len(sMainParts) == 2, "Wrong experiment filename"
|
|
50
|
+
sFoldNumber, _ = os.path.splitext(sMainParts[1])
|
|
51
|
+
sParts = sMainParts[0].split("_")
|
|
52
|
+
sModelName = f"{sParts[0]}_{sParts[1]}"
|
|
53
|
+
sModelVar = sParts[2]
|
|
54
|
+
else:
|
|
55
|
+
sFileNameOnly, _ = os.path.splitext(filename)
|
|
56
|
+
sMainParts = sFileNameOnly.split("-")
|
|
57
|
+
if len(sMainParts) > 1:
|
|
58
|
+
sFoldNumber = sMainParts[1]
|
|
59
|
+
else:
|
|
60
|
+
sFoldNumber = None
|
|
61
|
+
sParts = sMainParts[0].split("_")
|
|
62
|
+
sModelName = f"{sParts[0]}_{sParts[1]}"
|
|
63
|
+
sModelVar = sParts[2]
|
|
64
|
+
|
|
65
|
+
return sModelName, sModelVar, sFoldNumber
|
|
66
|
+
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
|
67
|
+
@classmethod
|
|
68
|
+
def preload_config(cls, config_folder, experiment_group=None, experiment_base_name=None, experiment_variation=None, experiment_fold_number=None, experiment_filename=None):
|
|
69
|
+
oPrintOutput = []
|
|
70
|
+
oPrintOutput.append(f"[?] Experiment started at {now_iso()}")
|
|
71
|
+
oPrintOutput.append(f" |__ {'model':<24}: {experiment_base_name}")
|
|
72
|
+
|
|
73
|
+
dExperimentSpec = {"base_name": experiment_base_name, "variation": experiment_variation, "fold_number": experiment_fold_number}
|
|
74
|
+
if experiment_filename is not None:
|
|
75
|
+
# LREXPLAINET18_MNIST_08.1-01.json
|
|
76
|
+
_, sFileNameFull = os.path.split(experiment_filename)
|
|
77
|
+
experiment_base_name, experiment_variation, experiment_fold_number = cls.experiment_filename_split(sFileNameFull)
|
|
78
|
+
experiment_fold_number = int(experiment_fold_number)
|
|
79
|
+
dExperimentSpec = {"base_name": experiment_base_name, "variation": experiment_variation, "fold_number": experiment_fold_number}
|
|
80
|
+
|
|
81
|
+
if experiment_group is not None:
|
|
82
|
+
config_folder = os.path.join(config_folder, experiment_group)
|
|
83
|
+
oConfigFS = FileStore(config_folder, must_exist=True)
|
|
84
|
+
|
|
85
|
+
if "." in experiment_variation:
|
|
86
|
+
sParts = experiment_variation.split(".")
|
|
87
|
+
experiment_variation = f"{int(sParts[0]):02d}.{sParts[1]}"
|
|
88
|
+
else:
|
|
89
|
+
experiment_variation = f"{int(experiment_variation):02d}"
|
|
90
|
+
sMessage = f" |__ {'variation':<24}: {experiment_variation}"
|
|
91
|
+
|
|
92
|
+
if experiment_fold_number is not None:
|
|
93
|
+
experiment_variation = f"{experiment_variation}-{experiment_fold_number:02d}"
|
|
94
|
+
sMessage += f" fold: {experiment_fold_number}"
|
|
95
|
+
oPrintOutput.append(sMessage)
|
|
96
|
+
|
|
97
|
+
if experiment_filename is not None:
|
|
98
|
+
sExperimentFileName = experiment_filename
|
|
99
|
+
else:
|
|
100
|
+
sExperimentFileName = oConfigFS.file(f"{experiment_base_name}_{experiment_variation}.json")
|
|
101
|
+
oConfig = MLExperimentConfig(sExperimentFileName,p_nExperimentNumber=experiment_variation)
|
|
102
|
+
oPrintOutput.append(f" |__ {'configuration file':<24}: {sExperimentFileName}")
|
|
103
|
+
|
|
104
|
+
return oConfig, oPrintOutput, dExperimentSpec
|
|
105
|
+
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
|
106
|
+
|
|
107
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
108
|
+
def __init__(self, config_fs, model_fs=None, base_name=None, number=None, variation=None, fold_number=None,
|
|
109
|
+
experiment_filename=None, experiment_code=None, experiment_config=None, model_filestore=None):
|
|
110
|
+
|
|
111
|
+
if isinstance(config_fs, FileSystem):
|
|
112
|
+
oConfigFS = config_fs.configs
|
|
113
|
+
oModelFS = config_fs.models
|
|
114
|
+
else:
|
|
115
|
+
oConfigFS = config_fs
|
|
116
|
+
oModelFS = model_fs
|
|
117
|
+
if oModelFS is None:
|
|
118
|
+
oModelFS = config_fs
|
|
119
|
+
|
|
120
|
+
# ...................... | Fields | ......................
|
|
121
|
+
self.config_fs = oConfigFS
|
|
122
|
+
self.model_fs = oModelFS
|
|
123
|
+
self.base_name = base_name
|
|
124
|
+
self.number = number
|
|
125
|
+
self.variation = variation
|
|
126
|
+
self.fold_number = fold_number
|
|
127
|
+
|
|
128
|
+
if (experiment_filename is not None) or (experiment_code is not None):
|
|
129
|
+
self.base_name, self.number, self.variation, self.fold_number, _ = self.determine_code_parts(experiment_filename,
|
|
130
|
+
experiment_code, number, variation, fold_number)
|
|
131
|
+
|
|
132
|
+
self.experiment_filename = experiment_filename
|
|
133
|
+
if self.experiment_filename is None:
|
|
134
|
+
sExperimentCode = get_experiment_code_ex(self.base_name, self.number, self.variation, self.fold_number)
|
|
135
|
+
if experiment_code is not None:
|
|
136
|
+
assert sExperimentCode == experiment_code, "Re-created experiment code mismatch."
|
|
137
|
+
self.experiment_filename = self.config_fs.file(f"{sExperimentCode}.json")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
self.is_debugable = False
|
|
141
|
+
self.is_retraining = False
|
|
142
|
+
|
|
143
|
+
self._config = experiment_config
|
|
144
|
+
if self._config is None:
|
|
145
|
+
self._config = MLExperimentConfig(self.experiment_filename, number=self.number)
|
|
146
|
+
self.experiment_fs = self.model_fs.subfs(self.experiment_code)
|
|
147
|
+
# ........................................................
|
|
148
|
+
|
|
149
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
150
|
+
def save_config(self):
|
|
151
|
+
self._config.save(self.experiment_filename)
|
|
152
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
153
|
+
def determine_code_parts(self, experiment_filename, experiment_code, number, variation, fold_number):
|
|
154
|
+
sExperimentCode = None
|
|
155
|
+
if experiment_code is not None:
|
|
156
|
+
sExperimentCode = experiment_code
|
|
157
|
+
elif experiment_filename is not None:
|
|
158
|
+
_, sFileNameFull = os.path.split(experiment_filename)
|
|
159
|
+
sExperimentCode, _ = os.path.splitext(sFileNameFull)
|
|
160
|
+
|
|
161
|
+
sISODate = None
|
|
162
|
+
if sExperimentCode is not None:
|
|
163
|
+
oMatch = re.match(r"^\d{4}-\d{2}-\d{2}_\d{6}_", sExperimentCode)
|
|
164
|
+
if oMatch is not None:
|
|
165
|
+
sISODate = oMatch.group()[:-1]
|
|
166
|
+
sExperimentCode = sExperimentCode[18:]
|
|
167
|
+
|
|
168
|
+
base_name, variation, sFoldNumber = MLExperimentEnv.experiment_filename_split(sExperimentCode)
|
|
169
|
+
if sFoldNumber is not None:
|
|
170
|
+
fold_number = int(sFoldNumber)
|
|
171
|
+
|
|
172
|
+
if number is None:
|
|
173
|
+
number, variation = experiment_number_and_variation(variation)
|
|
174
|
+
|
|
175
|
+
return base_name, number, variation, fold_number, sISODate
|
|
176
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
177
|
+
@property
|
|
178
|
+
def config(self):
|
|
179
|
+
return self._config
|
|
180
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
181
|
+
@property
|
|
182
|
+
def experiment_code(self):
|
|
183
|
+
return get_experiment_code_ex(self.base_name, self.number, self.variation, self.fold_number)
|
|
184
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
185
|
+
def copy_config(self, start_timestamp):
|
|
186
|
+
sOriginalFileName = self.experiment_filename
|
|
187
|
+
sNewFileName = self.experiment_fs.file(f"{start_timestamp}_{self.experiment_code}.json")
|
|
188
|
+
shutil.copy(sOriginalFileName, sNewFileName)
|
|
189
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
190
|
+
def move_log(self, start_timestamp, original_filename):
|
|
191
|
+
self.copy_config(start_timestamp)
|
|
192
|
+
_, original_filename_only = os.path.split(original_filename)
|
|
193
|
+
sNewFileName = f"{start_timestamp}_{self.experiment_code}.{original_filename_only}"
|
|
194
|
+
shutil.move(original_filename, self.experiment_fs.file(sNewFileName))
|
|
195
|
+
sys.stdout = TeeLogger(self.experiment_fs.file(sNewFileName))
|
|
196
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
197
|
+
'''
|
|
198
|
+
def AssignSystemParams(self, p_oParamsDict):
|
|
199
|
+
self.number = p_oParamsDict["ModelNumber"]
|
|
200
|
+
self.is_debugable = p_oParamsDict["IsDebuggable"]
|
|
201
|
+
self.is_retraining = p_oParamsDict["IsRetraining"]
|
|
202
|
+
|
|
203
|
+
self._config = MLExperimentConfig(self.model_fs.file(self.experiment_code + ".json"), number=self.number)
|
|
204
|
+
'''
|
|
205
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
206
|
+
def __str__(self)->str:
|
|
207
|
+
sResult = f"Experiment Code: {self.experiment_code} | Debugable: {self.is_debugable} | Retraining: {self.is_retraining}\n"
|
|
208
|
+
sResult += f" |___ base name: {self.base_name} , number:{self.number} , variation:{self.variation}, fold_number: {self.fold_number}\n"
|
|
209
|
+
sResult += f" |___ file name: {self.experiment_filename}\n"
|
|
210
|
+
sResult += f"Models FileStore : {self.model_fs}\n"
|
|
211
|
+
sResult += f"Configs FileStore : {self.config_fs}\n"
|
|
212
|
+
sResult += f"Experiment FileStore: {self.experiment_fs}\n"
|
|
213
|
+
sResult += f"Experiment Filename : {self.experiment_filename}\n"
|
|
214
|
+
sResult += f"Configuration\n"
|
|
215
|
+
sResult += "-"*40 + "\n"
|
|
216
|
+
sResult += str(self._config) + "\n"
|
|
217
|
+
return sResult
|
|
218
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
219
|
+
def __repr__(self)->str:
|
|
220
|
+
return self.__str__()
|
|
221
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
222
|
+
|
|
223
|
+
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# ......................................................................................
|
|
2
|
+
# MIT License
|
|
3
|
+
|
|
4
|
+
# Copyright (c) 2020-2025 Pantelis I. Kaplanoglou
|
|
5
|
+
|
|
6
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
# in the Software without restriction, including without limitation the rights
|
|
9
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
# furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
14
|
+
# copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
|
+
# SOFTWARE.
|
|
23
|
+
|
|
24
|
+
# ......................................................................................
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
import matplotlib.pyplot as plt
|
|
28
|
+
|
|
29
|
+
class PlotConfusionMatrix(object):
|
|
30
|
+
# --------------------------------------------------------------------------------------
|
|
31
|
+
def __init__(self, confusion_matrix, title="Confusion Matrix"):
|
|
32
|
+
self.confusion_matrix = confusion_matrix
|
|
33
|
+
self.title = title
|
|
34
|
+
# --------------------------------------------------------------------------------------
|
|
35
|
+
def prepare(self):
|
|
36
|
+
fig, ax = plt.subplots(figsize=(7.5, 7.5))
|
|
37
|
+
ax.matshow(self.confusion_matrix, cmap=plt.cm.Blues, alpha=0.3)
|
|
38
|
+
for i in range(self.confusion_matrix.shape[0]):
|
|
39
|
+
for j in range(self.confusion_matrix.shape[1]):
|
|
40
|
+
ax.text(x=j, y=i, s=self.confusion_matrix[i, j], va='center', ha='center', size='xx-large')
|
|
41
|
+
|
|
42
|
+
plt.xlabel('Predicted Label', fontsize=18)
|
|
43
|
+
plt.ylabel('Actual Label', fontsize=18)
|
|
44
|
+
plt.title(self.title, fontsize=18)
|
|
45
|
+
return self
|
|
46
|
+
# --------------------------------------------------------------------------------------
|
|
47
|
+
def save(self, filename):
|
|
48
|
+
plt.savefig(filename, bbox_inches='tight')
|
|
49
|
+
return self
|
|
50
|
+
# --------------------------------------------------------------------------------------
|
|
51
|
+
def show(self):
|
|
52
|
+
plt.show()
|
|
53
|
+
# --------------------------------------------------------------------------------------
|
|
54
|
+
|