radnn 0.0.8__py3-none-any.whl → 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- radnn/__init__.py +5 -5
- radnn/benchmark/__init__.py +1 -0
- radnn/benchmark/latency.py +55 -0
- radnn/core.py +146 -2
- radnn/data/__init__.py +5 -10
- radnn/data/dataset_base.py +100 -260
- radnn/data/dataset_base_legacy.py +280 -0
- radnn/data/errors.py +32 -0
- radnn/data/sample_preprocessor.py +58 -0
- radnn/data/sample_set.py +203 -90
- radnn/data/sample_set_kind.py +126 -0
- radnn/data/sequence_dataset.py +25 -30
- radnn/data/structs/__init__.py +1 -0
- radnn/data/structs/tree.py +322 -0
- radnn/data_beta/__init__.py +12 -0
- radnn/{data → data_beta}/data_feed.py +1 -1
- radnn/data_beta/dataset_base.py +337 -0
- radnn/data_beta/sample_set.py +166 -0
- radnn/data_beta/sequence_dataset.py +134 -0
- radnn/data_beta/structures/__init__.py +2 -0
- radnn/data_beta/structures/dictionary.py +41 -0
- radnn/{data → data_beta}/tf_classification_data_feed.py +5 -2
- radnn/errors.py +10 -2
- radnn/experiment/__init__.py +2 -0
- radnn/experiment/identification.py +7 -0
- radnn/experiment/ml_experiment.py +7 -2
- radnn/experiment/ml_experiment_log.py +47 -0
- radnn/images/image_processor.py +4 -1
- radnn/learn/__init__.py +0 -7
- radnn/learn/keras/__init__.py +4 -0
- radnn/learn/{state → keras}/keras_best_state_saver.py +5 -1
- radnn/learn/{learning_algorithm.py → keras/keras_learning_algorithm.py} +5 -9
- radnn/learn/{keras_learning_rate_scheduler.py → keras/keras_learning_rate_scheduler.py} +4 -1
- radnn/learn/{keras_optimization_algorithm.py → keras/keras_optimization_combo.py} +7 -3
- radnn/learn/torch/__init__.py +3 -0
- radnn/learn/torch/ml_model_freezer.py +330 -0
- radnn/learn/torch/ml_trainer.py +461 -0
- radnn/learn/torch/staircase_lr_scheduler.py +21 -0
- radnn/ml_system.py +68 -52
- radnn/models/__init__.py +5 -0
- radnn/models/cnn/__init__.py +0 -0
- radnn/models/cnn/cnn_stem_setup.py +35 -0
- radnn/models/model_factory.py +85 -0
- radnn/models/model_hyperparams.py +128 -0
- radnn/models/model_info.py +91 -0
- radnn/plots/plot_learning_curve.py +19 -8
- radnn/system/__init__.py +1 -0
- radnn/system/files/__init__.py +1 -1
- radnn/system/files/csvfile.py +37 -5
- radnn/system/files/filelist.py +30 -0
- radnn/system/files/fileobject.py +11 -1
- radnn/system/files/imgfile.py +1 -1
- radnn/system/files/jsonfile.py +37 -9
- radnn/system/files/picklefile.py +3 -3
- radnn/system/files/textfile.py +39 -10
- radnn/system/files/zipfile.py +96 -0
- radnn/system/filestore.py +147 -47
- radnn/system/filesystem.py +3 -3
- radnn/test/__init__.py +1 -0
- radnn/test/tensor_hash.py +130 -0
- radnn/utils.py +16 -2
- radnn-0.1.0.dist-info/METADATA +30 -0
- radnn-0.1.0.dist-info/RECORD +99 -0
- {radnn-0.0.8.dist-info → radnn-0.1.0.dist-info}/WHEEL +1 -1
- {radnn-0.0.8.dist-info → radnn-0.1.0.dist-info/licenses}/LICENSE.txt +1 -1
- radnn/learn/state/__init__.py +0 -4
- radnn-0.0.8.dist-info/METADATA +0 -58
- radnn-0.0.8.dist-info/RECORD +0 -70
- /radnn/{data → data_beta}/dataset_folder.py +0 -0
- /radnn/{data → data_beta}/image_dataset.py +0 -0
- /radnn/{data → data_beta}/image_dataset_files.py +0 -0
- /radnn/{data → data_beta}/preprocess/__init__.py +0 -0
- /radnn/{data → data_beta}/preprocess/normalizer.py +0 -0
- /radnn/{data → data_beta}/preprocess/standardizer.py +0 -0
- /radnn/{data → data_beta}/subset_type.py +0 -0
- {radnn-0.0.8.dist-info → radnn-0.1.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
|
|
3
|
+
class CNNSizeFactor(Enum):
|
|
4
|
+
TINY = 1
|
|
5
|
+
PETITE = 2
|
|
6
|
+
SMALL = 3
|
|
7
|
+
MEDIUM = 4
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class CNNStemSetup(object):
|
|
11
|
+
def __init__(self, size_factor: CNNSizeFactor, input_channels: int = 3, is_vit_like: bool = False):
|
|
12
|
+
self.size_factor: CNNSizeFactor = size_factor
|
|
13
|
+
self.input_channels = input_channels
|
|
14
|
+
self.windows = []
|
|
15
|
+
self.strides = []
|
|
16
|
+
|
|
17
|
+
if is_vit_like:
|
|
18
|
+
self.windows = [4]
|
|
19
|
+
self.strides = [4]
|
|
20
|
+
else:
|
|
21
|
+
if self.size_factor is CNNSizeFactor.TINY:
|
|
22
|
+
self.windows = [3]
|
|
23
|
+
self.strides = [1]
|
|
24
|
+
elif self.size_factor is CNNSizeFactor.PETITE:
|
|
25
|
+
self.windows = [3]
|
|
26
|
+
self.strides = [2]
|
|
27
|
+
elif self.size_factor is CNNSizeFactor.SMALL:
|
|
28
|
+
self.windows = [5]
|
|
29
|
+
self.strides = [2]
|
|
30
|
+
elif self.size_factor is CNNSizeFactor.MEDIUM:
|
|
31
|
+
self.windows = [7, 3] #TODO: Check various papers
|
|
32
|
+
self.strides = [2, 2]
|
|
33
|
+
else:
|
|
34
|
+
raise ValueError(f"Unknown CNN scale factor: {self.size_factor.name}")
|
|
35
|
+
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
7
|
+
# ......................................................................................
|
|
8
|
+
|
|
9
|
+
# Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
|
|
10
|
+
|
|
11
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
13
|
+
# in the Software without restriction, including without limitation the rights
|
|
14
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
15
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
16
|
+
# furnished to do so, subject to the following conditions:
|
|
17
|
+
|
|
18
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
19
|
+
# copies or substantial portions of the Software.
|
|
20
|
+
|
|
21
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
22
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
23
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
24
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
25
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
26
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
27
|
+
# SOFTWARE.
|
|
28
|
+
|
|
29
|
+
# .......................................................................................
|
|
30
|
+
from typing import Dict, Type, Any
|
|
31
|
+
|
|
32
|
+
from radnn.models import ModelHyperparams, ModelInfo
|
|
33
|
+
from radnn.errors import *
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# ======================================================================================================================
|
|
37
|
+
class RadnnModel(type):
|
|
38
|
+
def __new__(mcls, name: str, bases: tuple[type, ...], ns: dict[str, Any]):
|
|
39
|
+
cls = super().__new__(mcls, name, bases, ns)
|
|
40
|
+
# Avoid registering the abstract base itself
|
|
41
|
+
if name != "ModelBuildAdapter":
|
|
42
|
+
key = ns.get("NAME", name) # optional override via class attribute NAME
|
|
43
|
+
if key in ModelFactory.registry:
|
|
44
|
+
raise KeyError(f"The model '{key}' has been alrady registered ")
|
|
45
|
+
ModelFactory.registry[key] = cls
|
|
46
|
+
return cls
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ======================================================================================================================
|
|
50
|
+
class ModelBuildAdapter(metaclass=RadnnModel):
|
|
51
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
52
|
+
def __init__(self, **kwargs):
|
|
53
|
+
self.builder_kwargs = kwargs
|
|
54
|
+
self.hprm: dict | None = self.builder_kwargs.get("hprm", None)
|
|
55
|
+
self.hyperparams: ModelHyperparams | None = self.builder_kwargs.get("model_hyperparams", None)
|
|
56
|
+
if self.hyperparams is not None:
|
|
57
|
+
self.hprm = self.hyperparams.hprm
|
|
58
|
+
self.model = None
|
|
59
|
+
self.model_info: ModelInfo | None = None
|
|
60
|
+
|
|
61
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
62
|
+
def build(self, **kwargs: Any):
|
|
63
|
+
raise NotImplementedError
|
|
64
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# ======================================================================================================================
|
|
68
|
+
class ModelFactory(object):
|
|
69
|
+
registry: Dict[str, Type["ModelBuildAdapter"]] = {}
|
|
70
|
+
|
|
71
|
+
@classmethod
|
|
72
|
+
def produce(cls, hyperparams: dict, **kwargs: Any) -> ModelBuildAdapter:
|
|
73
|
+
oModelBuilder = None
|
|
74
|
+
oModelHyperparams: ModelHyperparams = ModelHyperparams.from_dict(hyperparams)
|
|
75
|
+
assert oModelHyperparams.input_dims is not None, HPARAMS_DATA_INPUT_DIMS
|
|
76
|
+
assert len(oModelHyperparams.input_dims) > 0, HPARAMS_DATA_INPUT_DIMS
|
|
77
|
+
sModelAdapterKey = oModelHyperparams.base_name + "BuildAdapter"
|
|
78
|
+
if sModelAdapterKey in cls.registry:
|
|
79
|
+
cBuilder = ModelFactory.registry[sModelAdapterKey]
|
|
80
|
+
oModelBuilder = cBuilder(model_hyperparams=oModelHyperparams)
|
|
81
|
+
oModel = oModelBuilder.build(hprm=hyperparams, **kwargs)
|
|
82
|
+
oModelBuilder.model_info = ModelInfo(oModel, oModelHyperparams)
|
|
83
|
+
|
|
84
|
+
return oModelBuilder
|
|
85
|
+
# ======================================================================================================================
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
7
|
+
# ......................................................................................
|
|
8
|
+
|
|
9
|
+
# Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
|
|
10
|
+
|
|
11
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
13
|
+
# in the Software without restriction, including without limitation the rights
|
|
14
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
15
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
16
|
+
# furnished to do so, subject to the following conditions:
|
|
17
|
+
|
|
18
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
19
|
+
# copies or substantial portions of the Software.
|
|
20
|
+
|
|
21
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
22
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
23
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
24
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
25
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
26
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
27
|
+
# SOFTWARE.
|
|
28
|
+
|
|
29
|
+
# .......................................................................................
|
|
30
|
+
import numpy as np
|
|
31
|
+
from . import CNNSizeFactor
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class ModelHyperparams(object):
|
|
35
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
36
|
+
def __init__(self, hprm: dict | None = None):
|
|
37
|
+
self.hprm = hprm
|
|
38
|
+
self.model_name = None
|
|
39
|
+
self.base_name = ""
|
|
40
|
+
self.architecture = ""
|
|
41
|
+
self.variant = ""
|
|
42
|
+
self.class_count = 0
|
|
43
|
+
self.input_dims = []
|
|
44
|
+
self.dropout_prob = None
|
|
45
|
+
# self.kind = "" //TODO: Validate expected input according to model kind
|
|
46
|
+
|
|
47
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
48
|
+
@property
|
|
49
|
+
def input_channels(self):
|
|
50
|
+
'''
|
|
51
|
+
:return: The channels from the dimensions of an image input sample
|
|
52
|
+
'''
|
|
53
|
+
return self.input_dims[-1]
|
|
54
|
+
|
|
55
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
56
|
+
@property
|
|
57
|
+
def input_resolution(self):
|
|
58
|
+
'''
|
|
59
|
+
:return: The resolution from the dimensions of an image input sample
|
|
60
|
+
'''
|
|
61
|
+
return self.input_dims[:2]
|
|
62
|
+
|
|
63
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
64
|
+
@property
|
|
65
|
+
def cnn_size_factor(self):
|
|
66
|
+
'''
|
|
67
|
+
Calculates the cnn size factor using the maximum dimension as log normalized where 0 is the size for MNIST input
|
|
68
|
+
and 1 the size for ImageNet input.
|
|
69
|
+
:return: 1 tiny images (MNIST, FMNIST, CIFAR10, CIFAR100)
|
|
70
|
+
2 small images (ImageNet 64x64)
|
|
71
|
+
3
|
|
72
|
+
4
|
|
73
|
+
'''
|
|
74
|
+
|
|
75
|
+
def get_id(x):
|
|
76
|
+
if 0.0 <= x < 0.1:
|
|
77
|
+
return 1
|
|
78
|
+
elif 0.1 <= x < 0.4:
|
|
79
|
+
return 2
|
|
80
|
+
elif 0.4 <= x < 0.8:
|
|
81
|
+
return 3
|
|
82
|
+
elif 0.8 <= x <= 1.0:
|
|
83
|
+
return 4
|
|
84
|
+
else:
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
nHeight, nWidth = self.input_dims[:2]
|
|
88
|
+
nDim = nHeight
|
|
89
|
+
if nHeight < nWidth:
|
|
90
|
+
Dim = nWidth
|
|
91
|
+
nLogDim = np.log10(nDim)
|
|
92
|
+
nMin = np.log10(28)
|
|
93
|
+
nMax = np.log10(300)
|
|
94
|
+
nLogNormalized = (nLogDim - nMin) / (nMax - nMin)
|
|
95
|
+
nSizeFactor = CNNSizeFactor(get_id(nLogNormalized))
|
|
96
|
+
return nSizeFactor
|
|
97
|
+
|
|
98
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
99
|
+
@classmethod
|
|
100
|
+
def from_dict(cls, hyperparams):
|
|
101
|
+
'''
|
|
102
|
+
Deserializes a hyperparameters object from a dictionary
|
|
103
|
+
|
|
104
|
+
:param hyperparams: A dictionary containing the architectural hyperparameters for the model
|
|
105
|
+
:return: a new model hyperparams object
|
|
106
|
+
'''
|
|
107
|
+
oModelHyperparams = ModelHyperparams(hyperparams)
|
|
108
|
+
oModelHyperparams.model_name = hyperparams.get("Model.Name", "?")
|
|
109
|
+
|
|
110
|
+
oModelHyperparams.base_name = oModelHyperparams.model_name
|
|
111
|
+
sParts = oModelHyperparams.base_name.split(".")
|
|
112
|
+
oModelHyperparams.architecture = "base"
|
|
113
|
+
if len(sParts) >= 2:
|
|
114
|
+
oModelHyperparams.base_name = sParts[0]
|
|
115
|
+
oModelHyperparams.architecture = sParts[1].lower()
|
|
116
|
+
|
|
117
|
+
oModelHyperparams.variant = hyperparams.get("Model.Variant", "")
|
|
118
|
+
oModelHyperparams.name = hyperparams.get("Model.Name", "?")
|
|
119
|
+
oModelHyperparams.input_dims = hyperparams.get("Dataset.SampleDims", [])
|
|
120
|
+
oModelHyperparams.class_count = hyperparams.get("Dataset.ClassCount", 0)
|
|
121
|
+
nDropOutKeepProb = hyperparams.get("Training.Regularize.DropOut.KeepProb", None)
|
|
122
|
+
if nDropOutKeepProb is None:
|
|
123
|
+
oModelHyperparams.dropout_prob = hyperparams.get("Training.Regularize.DropOut.DropProb", 0.0)
|
|
124
|
+
else:
|
|
125
|
+
oModelHyperparams.dropout_prob = 1.0 - nDropOutKeepProb
|
|
126
|
+
|
|
127
|
+
return oModelHyperparams
|
|
128
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
7
|
+
# ......................................................................................
|
|
8
|
+
|
|
9
|
+
# Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
|
|
10
|
+
|
|
11
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
13
|
+
# in the Software without restriction, including without limitation the rights
|
|
14
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
15
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
16
|
+
# furnished to do so, subject to the following conditions:
|
|
17
|
+
|
|
18
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
19
|
+
# copies or substantial portions of the Software.
|
|
20
|
+
|
|
21
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
22
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
23
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
24
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
25
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
26
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
27
|
+
# SOFTWARE.
|
|
28
|
+
|
|
29
|
+
# .......................................................................................
|
|
30
|
+
from radnn import FileStore
|
|
31
|
+
from radnn.data.structs import Tree, TreeNode
|
|
32
|
+
|
|
33
|
+
from torchinfo import ModelStatistics, summary
|
|
34
|
+
from torchinfo.layer_info import LayerInfo
|
|
35
|
+
from .model_hyperparams import ModelHyperparams
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class ModelInfo(object):
|
|
39
|
+
def __init__(self, model=None, hyperparams: ModelHyperparams | dict = None, depth=6):
|
|
40
|
+
self.model = model
|
|
41
|
+
self.hyperparams = hyperparams
|
|
42
|
+
if isinstance(hyperparams, dict):
|
|
43
|
+
self.hyperparams = ModelHyperparams.from_dict(hyperparams)
|
|
44
|
+
self.model_tree: Tree = None
|
|
45
|
+
|
|
46
|
+
if self.model is not None:
|
|
47
|
+
nInputDims = tuple([1, hyperparams.input_channels] + hyperparams.input_resolution)
|
|
48
|
+
self.summary: ModelStatistics = summary(model, nInputDims, device='cpu', depth=depth)
|
|
49
|
+
self.build_tree()
|
|
50
|
+
|
|
51
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
52
|
+
@property
|
|
53
|
+
def filename(self):
|
|
54
|
+
return f'{self.hyperparams.model_name}_{str(self.hyperparams.input_dims).replace(", ", "x")[1:-1]}_cls{str(self.hyperparams.class_count)}'
|
|
55
|
+
|
|
56
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
57
|
+
def save(self, fs: str | FileStore):
|
|
58
|
+
if isinstance(fs, str):
|
|
59
|
+
fs = FileStore(fs)
|
|
60
|
+
fs.artifact(self.filename + ".pkl").save(self.summary, is_overwriting=True)
|
|
61
|
+
with open(fs.file(self.filename + ".txt"), "w", encoding="utf-8") as oFile:
|
|
62
|
+
print(self.summary, file=oFile)
|
|
63
|
+
|
|
64
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
65
|
+
def load(self, fs: str | FileStore):
|
|
66
|
+
bIsLoaded = False
|
|
67
|
+
if isinstance(fs, str):
|
|
68
|
+
fs = FileStore(fs)
|
|
69
|
+
self.summary = fs.artifact(self.filename + ".pkl").load()
|
|
70
|
+
bIsLoaded = self.summary is not None
|
|
71
|
+
if bIsLoaded:
|
|
72
|
+
self.build_tree()
|
|
73
|
+
'''
|
|
74
|
+
oNodeList = self.model_tree.traverse_depth_first()
|
|
75
|
+
for oNode in oNodeList:
|
|
76
|
+
print(oNode.path)
|
|
77
|
+
'''
|
|
78
|
+
return bIsLoaded
|
|
79
|
+
|
|
80
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
81
|
+
def build_tree(self):
|
|
82
|
+
def recurse_build_subtree(source_node: LayerInfo, target_node: TreeNode):
|
|
83
|
+
for nIndex, oSourceChild in enumerate(source_node.children):
|
|
84
|
+
sNodeName = f"{oSourceChild.class_name}: {oSourceChild.depth}-{oSourceChild.depth_index}"
|
|
85
|
+
oTargetChild: TreeNode = target_node.new_child(sNodeName)
|
|
86
|
+
oTargetChild.value = oSourceChild
|
|
87
|
+
recurse_build_subtree(oSourceChild, oTargetChild)
|
|
88
|
+
|
|
89
|
+
self.model_tree = Tree()
|
|
90
|
+
recurse_build_subtree(self.summary.summary_list[0], self.model_tree.root)
|
|
91
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
@@ -36,22 +36,33 @@ class PlotLearningCurve(object):
|
|
|
36
36
|
self.model_name = model_name
|
|
37
37
|
print("Keys in training process log:", self.metrics_dict.keys())
|
|
38
38
|
# --------------------------------------------------------------------------------------
|
|
39
|
-
def prepare(self, metric_key="accuracy", custom_title=None, is_legend_right=False):
|
|
39
|
+
def prepare(self, metric_key="accuracy", custom_title=None, metric_low=0, metric_high=1.0, is_legend_right=False, is_keras=False):
|
|
40
|
+
sTrainMetricKey = metric_key
|
|
41
|
+
if not is_keras:
|
|
42
|
+
sTrainMetricKey = "train_" + metric_key
|
|
43
|
+
sValMetricKey = "val_" + metric_key
|
|
44
|
+
oLegend = ["training"]
|
|
45
|
+
bHasValidation = sValMetricKey in self.metrics_dict
|
|
46
|
+
oLegend.append("validation")
|
|
47
|
+
|
|
48
|
+
|
|
40
49
|
plt.clf()
|
|
41
|
-
plt.plot(self.metrics_dict[
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
50
|
+
plt.plot(self.metrics_dict[sTrainMetricKey])
|
|
51
|
+
if bHasValidation:
|
|
52
|
+
plt.plot(self.metrics_dict[sValMetricKey])
|
|
53
|
+
|
|
45
54
|
if custom_title is None:
|
|
46
55
|
plt.title(self.model_name + ' ' + metric_key)
|
|
47
56
|
else:
|
|
48
57
|
plt.title(self.model_name + ' ' + custom_title)
|
|
49
|
-
|
|
58
|
+
|
|
59
|
+
plt.ylabel(metric_key.capitalize().title())
|
|
50
60
|
plt.xlabel("Epoch")
|
|
61
|
+
plt.ylim([metric_low, metric_high])
|
|
51
62
|
if is_legend_right:
|
|
52
|
-
plt.legend(
|
|
63
|
+
plt.legend(oLegend, loc="upper right")
|
|
53
64
|
else:
|
|
54
|
-
plt.legend(
|
|
65
|
+
plt.legend(oLegend, loc="upper left")
|
|
55
66
|
return self
|
|
56
67
|
# --------------------------------------------------------------------------------------
|
|
57
68
|
def prepare_cost(self, cost_function=None):
|
radnn/system/__init__.py
CHANGED
radnn/system/files/__init__.py
CHANGED
radnn/system/files/csvfile.py
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
# ______________________________________________________________________________________
|
|
7
7
|
# ......................................................................................
|
|
8
8
|
|
|
9
|
-
# Copyright (c) 2018-
|
|
9
|
+
# Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
|
|
10
10
|
|
|
11
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
|
|
29
29
|
# .......................................................................................
|
|
30
30
|
import pandas as pd
|
|
31
|
-
|
|
31
|
+
import numpy as np
|
|
32
32
|
|
|
33
33
|
from .fileobject import FileObject
|
|
34
34
|
|
|
@@ -42,8 +42,40 @@ class CSVFile(FileObject):
|
|
|
42
42
|
dResult = pd.read_csv(sFileName, delimiter=delimiter, encoding=encoding)
|
|
43
43
|
return dResult
|
|
44
44
|
# ----------------------------------------------------------------------------------
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
45
|
+
# TODO: Iterate large csv files row by row as done in TextFile
|
|
46
|
+
#@property
|
|
47
|
+
#def rows(self):
|
|
48
|
+
# ----------------------------------------------------------------------------------
|
|
49
|
+
def save(self, obj, filename=None, headers=None, delimiter=",", encoding="utf-8"):
|
|
50
|
+
sFileName = self._useFileName(filename)
|
|
51
|
+
|
|
52
|
+
if isinstance(obj, dict):
|
|
53
|
+
nMaxRowCols = 0
|
|
54
|
+
oList = []
|
|
55
|
+
for k, v in obj.items():
|
|
56
|
+
oRow = [k]
|
|
57
|
+
if isinstance(v, list):
|
|
58
|
+
oRow.extend(v)
|
|
59
|
+
else:
|
|
60
|
+
oRow.append(v)
|
|
61
|
+
if len(oRow) > nMaxRowCols:
|
|
62
|
+
nMaxRowCols = len(oRow)
|
|
63
|
+
oList.append(oRow)
|
|
64
|
+
|
|
65
|
+
obj = []
|
|
66
|
+
for oRow in oList:
|
|
67
|
+
oRowToAdd = oRow + [None]*(nMaxRowCols - len(oRow))
|
|
68
|
+
obj.append(oRowToAdd)
|
|
69
|
+
if len(headers) < nMaxRowCols:
|
|
70
|
+
headers = headers + [None]*(nMaxRowCols - len(headers))
|
|
71
|
+
|
|
72
|
+
if isinstance(obj, list):
|
|
73
|
+
df = pd.DataFrame(obj, columns=headers)
|
|
74
|
+
df.to_csv(sFileName, index=False, header=(headers is not None), sep=delimiter, encoding=encoding)
|
|
48
75
|
|
|
76
|
+
elif isinstance(obj, np.ndarray):
|
|
77
|
+
if len(headers) < obj.shape[1]:
|
|
78
|
+
headers = headers + [None]*(obj.shape[1] - len(headers))
|
|
79
|
+
df = pd.DataFrame(obj, columns=headers)
|
|
80
|
+
df.to_csv(sFileName, index=False, header=(headers is not None), sep=delimiter, encoding=encoding)
|
|
49
81
|
# ----------------------------------------------------------------------------------
|
radnn/system/files/filelist.py
CHANGED
|
@@ -1,3 +1,33 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
7
|
+
# ......................................................................................
|
|
8
|
+
|
|
9
|
+
# Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
|
|
10
|
+
|
|
11
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
13
|
+
# in the Software without restriction, including without limitation the rights
|
|
14
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
15
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
16
|
+
# furnished to do so, subject to the following conditions:
|
|
17
|
+
|
|
18
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
19
|
+
# copies or substantial portions of the Software.
|
|
20
|
+
|
|
21
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
22
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
23
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
24
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
25
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
26
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
27
|
+
# SOFTWARE.
|
|
28
|
+
|
|
29
|
+
# .......................................................................................
|
|
30
|
+
|
|
1
31
|
import os
|
|
2
32
|
|
|
3
33
|
|
radnn/system/files/fileobject.py
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
# ______________________________________________________________________________________
|
|
7
7
|
# ......................................................................................
|
|
8
8
|
|
|
9
|
-
# Copyright (c) 2018-
|
|
9
|
+
# Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
|
|
10
10
|
|
|
11
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
@@ -48,6 +48,16 @@ class FileObject(object):
|
|
|
48
48
|
if not self.default_file_extension.startswith("."):
|
|
49
49
|
self.default_file_extension = "." + self.default_file_extension
|
|
50
50
|
# ----------------------------------------------------------------------------------
|
|
51
|
+
def close(self):
|
|
52
|
+
pass
|
|
53
|
+
# ----------------------------------------------------------------------------------
|
|
54
|
+
def __enter__(self):
|
|
55
|
+
return self
|
|
56
|
+
# ----------------------------------------------------------------------------------
|
|
57
|
+
def __exit__(self, exc_type, exc_value, traceback):
|
|
58
|
+
self.close()
|
|
59
|
+
return True
|
|
60
|
+
# ----------------------------------------------------------------------------------
|
|
51
61
|
def _useFileName(self, filename):
|
|
52
62
|
if filename is not None:
|
|
53
63
|
self.filename = filename
|
radnn/system/files/imgfile.py
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
# ______________________________________________________________________________________
|
|
7
7
|
# ......................................................................................
|
|
8
8
|
|
|
9
|
-
# Copyright (c) 2018-
|
|
9
|
+
# Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
|
|
10
10
|
|
|
11
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
12
|
# of this software and associated documentation files (the "Software"), to deal
|
radnn/system/files/jsonfile.py
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
# ______________________________________________________________________________________
|
|
7
7
|
# ......................................................................................
|
|
8
8
|
|
|
9
|
-
# Copyright (c) 2018-
|
|
9
|
+
# Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
|
|
10
10
|
|
|
11
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
@@ -30,36 +30,64 @@
|
|
|
30
30
|
import os
|
|
31
31
|
import json
|
|
32
32
|
import glob
|
|
33
|
+
from collections import OrderedDict
|
|
34
|
+
|
|
33
35
|
from .fileobject import FileObject
|
|
34
36
|
|
|
37
|
+
#TODO: jsonpickle
|
|
38
|
+
#https://stackoverflow.com/questions/3768895/how-to-make-a-class-json-serializable
|
|
39
|
+
|
|
35
40
|
class JSONFile(FileObject):
|
|
36
41
|
# ----------------------------------------------------------------------------------
|
|
37
42
|
def __init__(self, filename, parent_folder=None, error_template=None):
|
|
38
43
|
super(JSONFile, self).__init__(filename, parent_folder, error_template, "json")
|
|
39
44
|
# ----------------------------------------------------------------------------------
|
|
40
|
-
def load(self, filename=None):
|
|
45
|
+
def load(self, filename=None, encoding=None):
|
|
41
46
|
filename = self._useFileName(filename)
|
|
42
47
|
|
|
43
48
|
dResult = None
|
|
44
49
|
if os.path.exists(filename):
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
50
|
+
|
|
51
|
+
sJSON = None
|
|
52
|
+
if os.path.isfile(filename):
|
|
53
|
+
oEncodingToTry = ["utf-8", "utf-16", "latin1", "ascii"] # Add more if needed
|
|
54
|
+
if encoding is None:
|
|
55
|
+
bIsLoaded = False
|
|
56
|
+
for sEnc in oEncodingToTry:
|
|
57
|
+
try:
|
|
58
|
+
with open(filename, "r", encoding=sEnc) as oFile:
|
|
59
|
+
sJSON = oFile.read()
|
|
60
|
+
bIsLoaded = True
|
|
61
|
+
break
|
|
62
|
+
except (UnicodeDecodeError, UnicodeError):
|
|
63
|
+
continue
|
|
64
|
+
if not bIsLoaded:
|
|
65
|
+
raise ValueError("Unsupported encoding")
|
|
66
|
+
else:
|
|
67
|
+
with open(filename, "r", encoding=encoding) as oFile:
|
|
68
|
+
sJSON = oFile.read()
|
|
69
|
+
if sJSON is not None:
|
|
70
|
+
dResult = json.loads(sJSON, object_pairs_hook=OrderedDict)
|
|
48
71
|
else:
|
|
49
72
|
if self.error_template is not None:
|
|
50
73
|
raise Exception(self.error_template % filename)
|
|
51
74
|
|
|
52
75
|
return dResult
|
|
53
76
|
# ----------------------------------------------------------------------------------
|
|
54
|
-
def save(self, obj, filename=None, is_sorted_keys=
|
|
77
|
+
def save(self, obj, filename=None, is_sorted_keys=False, is_utf8=False):
|
|
55
78
|
filename = self._useFileName(filename)
|
|
56
79
|
|
|
57
80
|
if obj is not None:
|
|
58
81
|
if isinstance(obj, dict):
|
|
59
|
-
sJSON = json.dumps(obj, sort_keys=is_sorted_keys, indent=4)
|
|
82
|
+
sJSON = json.dumps(obj, sort_keys=is_sorted_keys, indent=4, ensure_ascii=is_utf8)
|
|
60
83
|
else:
|
|
61
|
-
sJSON = json.dumps(obj, default=lambda o: obj.__dict__, sort_keys=is_sorted_keys, indent=4)
|
|
62
|
-
|
|
84
|
+
sJSON = json.dumps(obj, default=lambda o: obj.__dict__, sort_keys=is_sorted_keys, indent=4, ensure_ascii=is_utf8)
|
|
85
|
+
|
|
86
|
+
sEncoding = "utf-8"
|
|
87
|
+
if not is_utf8:
|
|
88
|
+
sEncoding = "utf-16"
|
|
89
|
+
|
|
90
|
+
with open(filename, "w", encoding=sEncoding) as oFile:
|
|
63
91
|
oFile.write(sJSON)
|
|
64
92
|
oFile.close()
|
|
65
93
|
# ----------------------------------------------------------------------------------
|
radnn/system/files/picklefile.py
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
# ______________________________________________________________________________________
|
|
7
7
|
# ......................................................................................
|
|
8
8
|
|
|
9
|
-
# Copyright (c) 2018-
|
|
9
|
+
# Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
|
|
10
10
|
|
|
11
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
@@ -64,7 +64,7 @@ class PickleFile(FileObject):
|
|
|
64
64
|
del self._cached
|
|
65
65
|
self._cached = None
|
|
66
66
|
# ----------------------------------------------------------------------------------
|
|
67
|
-
def load(self, filename, is_python2_format=False, error_template=None):
|
|
67
|
+
def load(self, filename: str|None = None, is_python2_format: bool=False, error_template: str|None=None):
|
|
68
68
|
"""
|
|
69
69
|
Deserializes the data from a pickle file if it exists.
|
|
70
70
|
Parameters
|
|
@@ -101,7 +101,7 @@ class PickleFile(FileObject):
|
|
|
101
101
|
|
|
102
102
|
return oData
|
|
103
103
|
# ----------------------------------------------------------------------------------
|
|
104
|
-
def save(self, obj, filename=None, is_overwriting=False, extra_display_label=None):
|
|
104
|
+
def save(self, obj, filename: str | None =None, is_overwriting: bool =False, extra_display_label: str|None=None):
|
|
105
105
|
"""
|
|
106
106
|
Serializes the data to a pickle file if it does not exists.
|
|
107
107
|
Parameters
|