radnn 0.0.8__py3-none-any.whl → 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. radnn/__init__.py +5 -5
  2. radnn/benchmark/__init__.py +1 -0
  3. radnn/benchmark/latency.py +55 -0
  4. radnn/core.py +146 -2
  5. radnn/data/__init__.py +5 -10
  6. radnn/data/dataset_base.py +100 -260
  7. radnn/data/dataset_base_legacy.py +280 -0
  8. radnn/data/errors.py +32 -0
  9. radnn/data/sample_preprocessor.py +58 -0
  10. radnn/data/sample_set.py +203 -90
  11. radnn/data/sample_set_kind.py +126 -0
  12. radnn/data/sequence_dataset.py +25 -30
  13. radnn/data/structs/__init__.py +1 -0
  14. radnn/data/structs/tree.py +322 -0
  15. radnn/data_beta/__init__.py +12 -0
  16. radnn/{data → data_beta}/data_feed.py +1 -1
  17. radnn/data_beta/dataset_base.py +337 -0
  18. radnn/data_beta/sample_set.py +166 -0
  19. radnn/data_beta/sequence_dataset.py +134 -0
  20. radnn/data_beta/structures/__init__.py +2 -0
  21. radnn/data_beta/structures/dictionary.py +41 -0
  22. radnn/{data → data_beta}/tf_classification_data_feed.py +5 -2
  23. radnn/errors.py +10 -2
  24. radnn/experiment/__init__.py +2 -0
  25. radnn/experiment/identification.py +7 -0
  26. radnn/experiment/ml_experiment.py +7 -2
  27. radnn/experiment/ml_experiment_log.py +47 -0
  28. radnn/images/image_processor.py +4 -1
  29. radnn/learn/__init__.py +0 -7
  30. radnn/learn/keras/__init__.py +4 -0
  31. radnn/learn/{state → keras}/keras_best_state_saver.py +5 -1
  32. radnn/learn/{learning_algorithm.py → keras/keras_learning_algorithm.py} +5 -9
  33. radnn/learn/{keras_learning_rate_scheduler.py → keras/keras_learning_rate_scheduler.py} +4 -1
  34. radnn/learn/{keras_optimization_algorithm.py → keras/keras_optimization_combo.py} +7 -3
  35. radnn/learn/torch/__init__.py +3 -0
  36. radnn/learn/torch/ml_model_freezer.py +330 -0
  37. radnn/learn/torch/ml_trainer.py +461 -0
  38. radnn/learn/torch/staircase_lr_scheduler.py +21 -0
  39. radnn/ml_system.py +68 -52
  40. radnn/models/__init__.py +5 -0
  41. radnn/models/cnn/__init__.py +0 -0
  42. radnn/models/cnn/cnn_stem_setup.py +35 -0
  43. radnn/models/model_factory.py +85 -0
  44. radnn/models/model_hyperparams.py +128 -0
  45. radnn/models/model_info.py +91 -0
  46. radnn/plots/plot_learning_curve.py +19 -8
  47. radnn/system/__init__.py +1 -0
  48. radnn/system/files/__init__.py +1 -1
  49. radnn/system/files/csvfile.py +37 -5
  50. radnn/system/files/filelist.py +30 -0
  51. radnn/system/files/fileobject.py +11 -1
  52. radnn/system/files/imgfile.py +1 -1
  53. radnn/system/files/jsonfile.py +37 -9
  54. radnn/system/files/picklefile.py +3 -3
  55. radnn/system/files/textfile.py +39 -10
  56. radnn/system/files/zipfile.py +96 -0
  57. radnn/system/filestore.py +147 -47
  58. radnn/system/filesystem.py +3 -3
  59. radnn/test/__init__.py +1 -0
  60. radnn/test/tensor_hash.py +130 -0
  61. radnn/utils.py +16 -2
  62. radnn-0.1.0.dist-info/METADATA +30 -0
  63. radnn-0.1.0.dist-info/RECORD +99 -0
  64. {radnn-0.0.8.dist-info → radnn-0.1.0.dist-info}/WHEEL +1 -1
  65. {radnn-0.0.8.dist-info → radnn-0.1.0.dist-info/licenses}/LICENSE.txt +1 -1
  66. radnn/learn/state/__init__.py +0 -4
  67. radnn-0.0.8.dist-info/METADATA +0 -58
  68. radnn-0.0.8.dist-info/RECORD +0 -70
  69. /radnn/{data → data_beta}/dataset_folder.py +0 -0
  70. /radnn/{data → data_beta}/image_dataset.py +0 -0
  71. /radnn/{data → data_beta}/image_dataset_files.py +0 -0
  72. /radnn/{data → data_beta}/preprocess/__init__.py +0 -0
  73. /radnn/{data → data_beta}/preprocess/normalizer.py +0 -0
  74. /radnn/{data → data_beta}/preprocess/standardizer.py +0 -0
  75. /radnn/{data → data_beta}/subset_type.py +0 -0
  76. {radnn-0.0.8.dist-info → radnn-0.1.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,35 @@
1
+ from enum import Enum
2
+
3
+ class CNNSizeFactor(Enum):
4
+ TINY = 1
5
+ PETITE = 2
6
+ SMALL = 3
7
+ MEDIUM = 4
8
+
9
+
10
+ class CNNStemSetup(object):
11
+ def __init__(self, size_factor: CNNSizeFactor, input_channels: int = 3, is_vit_like: bool = False):
12
+ self.size_factor: CNNSizeFactor = size_factor
13
+ self.input_channels = input_channels
14
+ self.windows = []
15
+ self.strides = []
16
+
17
+ if is_vit_like:
18
+ self.windows = [4]
19
+ self.strides = [4]
20
+ else:
21
+ if self.size_factor is CNNSizeFactor.TINY:
22
+ self.windows = [3]
23
+ self.strides = [1]
24
+ elif self.size_factor is CNNSizeFactor.PETITE:
25
+ self.windows = [3]
26
+ self.strides = [2]
27
+ elif self.size_factor is CNNSizeFactor.SMALL:
28
+ self.windows = [5]
29
+ self.strides = [2]
30
+ elif self.size_factor is CNNSizeFactor.MEDIUM:
31
+ self.windows = [7, 3] #TODO: Check various papers
32
+ self.strides = [2, 2]
33
+ else:
34
+ raise ValueError(f"Unknown CNN scale factor: {self.size_factor.name}")
35
+
@@ -0,0 +1,85 @@
1
+ # ======================================================================================
2
+ #
3
+ # Rapid Deep Neural Networks
4
+ #
5
+ # Licensed under the MIT License
6
+ # ______________________________________________________________________________________
7
+ # ......................................................................................
8
+
9
+ # Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
10
+
11
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
12
+ # of this software and associated documentation files (the "Software"), to deal
13
+ # in the Software without restriction, including without limitation the rights
14
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15
+ # copies of the Software, and to permit persons to whom the Software is
16
+ # furnished to do so, subject to the following conditions:
17
+
18
+ # The above copyright notice and this permission notice shall be included in all
19
+ # copies or substantial portions of the Software.
20
+
21
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27
+ # SOFTWARE.
28
+
29
+ # .......................................................................................
30
+ from typing import Dict, Type, Any
31
+
32
+ from radnn.models import ModelHyperparams, ModelInfo
33
+ from radnn.errors import *
34
+
35
+
36
+ # ======================================================================================================================
37
+ class RadnnModel(type):
38
+ def __new__(mcls, name: str, bases: tuple[type, ...], ns: dict[str, Any]):
39
+ cls = super().__new__(mcls, name, bases, ns)
40
+ # Avoid registering the abstract base itself
41
+ if name != "ModelBuildAdapter":
42
+ key = ns.get("NAME", name) # optional override via class attribute NAME
43
+ if key in ModelFactory.registry:
44
+ raise KeyError(f"The model '{key}' has been alrady registered ")
45
+ ModelFactory.registry[key] = cls
46
+ return cls
47
+
48
+
49
+ # ======================================================================================================================
50
+ class ModelBuildAdapter(metaclass=RadnnModel):
51
+ # --------------------------------------------------------------------------------------------------------------------
52
+ def __init__(self, **kwargs):
53
+ self.builder_kwargs = kwargs
54
+ self.hprm: dict | None = self.builder_kwargs.get("hprm", None)
55
+ self.hyperparams: ModelHyperparams | None = self.builder_kwargs.get("model_hyperparams", None)
56
+ if self.hyperparams is not None:
57
+ self.hprm = self.hyperparams.hprm
58
+ self.model = None
59
+ self.model_info: ModelInfo | None = None
60
+
61
+ # --------------------------------------------------------------------------------------------------------------------
62
+ def build(self, **kwargs: Any):
63
+ raise NotImplementedError
64
+ # --------------------------------------------------------------------------------------------------------------------
65
+
66
+
67
+ # ======================================================================================================================
68
+ class ModelFactory(object):
69
+ registry: Dict[str, Type["ModelBuildAdapter"]] = {}
70
+
71
+ @classmethod
72
+ def produce(cls, hyperparams: dict, **kwargs: Any) -> ModelBuildAdapter:
73
+ oModelBuilder = None
74
+ oModelHyperparams: ModelHyperparams = ModelHyperparams.from_dict(hyperparams)
75
+ assert oModelHyperparams.input_dims is not None, HPARAMS_DATA_INPUT_DIMS
76
+ assert len(oModelHyperparams.input_dims) > 0, HPARAMS_DATA_INPUT_DIMS
77
+ sModelAdapterKey = oModelHyperparams.base_name + "BuildAdapter"
78
+ if sModelAdapterKey in cls.registry:
79
+ cBuilder = ModelFactory.registry[sModelAdapterKey]
80
+ oModelBuilder = cBuilder(model_hyperparams=oModelHyperparams)
81
+ oModel = oModelBuilder.build(hprm=hyperparams, **kwargs)
82
+ oModelBuilder.model_info = ModelInfo(oModel, oModelHyperparams)
83
+
84
+ return oModelBuilder
85
+ # ======================================================================================================================
@@ -0,0 +1,128 @@
1
+ # ======================================================================================
2
+ #
3
+ # Rapid Deep Neural Networks
4
+ #
5
+ # Licensed under the MIT License
6
+ # ______________________________________________________________________________________
7
+ # ......................................................................................
8
+
9
+ # Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
10
+
11
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
12
+ # of this software and associated documentation files (the "Software"), to deal
13
+ # in the Software without restriction, including without limitation the rights
14
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15
+ # copies of the Software, and to permit persons to whom the Software is
16
+ # furnished to do so, subject to the following conditions:
17
+
18
+ # The above copyright notice and this permission notice shall be included in all
19
+ # copies or substantial portions of the Software.
20
+
21
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27
+ # SOFTWARE.
28
+
29
+ # .......................................................................................
30
+ import numpy as np
31
+ from . import CNNSizeFactor
32
+
33
+
34
+ class ModelHyperparams(object):
35
+ # --------------------------------------------------------------------------------------------------------------------
36
+ def __init__(self, hprm: dict | None = None):
37
+ self.hprm = hprm
38
+ self.model_name = None
39
+ self.base_name = ""
40
+ self.architecture = ""
41
+ self.variant = ""
42
+ self.class_count = 0
43
+ self.input_dims = []
44
+ self.dropout_prob = None
45
+ # self.kind = "" //TODO: Validate expected input according to model kind
46
+
47
+ # --------------------------------------------------------------------------------------------------------------------
48
+ @property
49
+ def input_channels(self):
50
+ '''
51
+ :return: The channels from the dimensions of an image input sample
52
+ '''
53
+ return self.input_dims[-1]
54
+
55
+ # --------------------------------------------------------------------------------------------------------------------
56
+ @property
57
+ def input_resolution(self):
58
+ '''
59
+ :return: The resolution from the dimensions of an image input sample
60
+ '''
61
+ return self.input_dims[:2]
62
+
63
+ # --------------------------------------------------------------------------------------------------------------------
64
+ @property
65
+ def cnn_size_factor(self):
66
+ '''
67
+ Calculates the cnn size factor using the maximum dimension as log normalized where 0 is the size for MNIST input
68
+ and 1 the size for ImageNet input.
69
+ :return: 1 tiny images (MNIST, FMNIST, CIFAR10, CIFAR100)
70
+ 2 small images (ImageNet 64x64)
71
+ 3
72
+ 4
73
+ '''
74
+
75
+ def get_id(x):
76
+ if 0.0 <= x < 0.1:
77
+ return 1
78
+ elif 0.1 <= x < 0.4:
79
+ return 2
80
+ elif 0.4 <= x < 0.8:
81
+ return 3
82
+ elif 0.8 <= x <= 1.0:
83
+ return 4
84
+ else:
85
+ return None
86
+
87
+ nHeight, nWidth = self.input_dims[:2]
88
+ nDim = nHeight
89
+ if nHeight < nWidth:
90
+ Dim = nWidth
91
+ nLogDim = np.log10(nDim)
92
+ nMin = np.log10(28)
93
+ nMax = np.log10(300)
94
+ nLogNormalized = (nLogDim - nMin) / (nMax - nMin)
95
+ nSizeFactor = CNNSizeFactor(get_id(nLogNormalized))
96
+ return nSizeFactor
97
+
98
+ # --------------------------------------------------------------------------------------------------------------------
99
+ @classmethod
100
+ def from_dict(cls, hyperparams):
101
+ '''
102
+ Deserializes a hyperparameters object from a dictionary
103
+
104
+ :param hyperparams: A dictionary containing the architectural hyperparameters for the model
105
+ :return: a new model hyperparams object
106
+ '''
107
+ oModelHyperparams = ModelHyperparams(hyperparams)
108
+ oModelHyperparams.model_name = hyperparams.get("Model.Name", "?")
109
+
110
+ oModelHyperparams.base_name = oModelHyperparams.model_name
111
+ sParts = oModelHyperparams.base_name.split(".")
112
+ oModelHyperparams.architecture = "base"
113
+ if len(sParts) >= 2:
114
+ oModelHyperparams.base_name = sParts[0]
115
+ oModelHyperparams.architecture = sParts[1].lower()
116
+
117
+ oModelHyperparams.variant = hyperparams.get("Model.Variant", "")
118
+ oModelHyperparams.name = hyperparams.get("Model.Name", "?")
119
+ oModelHyperparams.input_dims = hyperparams.get("Dataset.SampleDims", [])
120
+ oModelHyperparams.class_count = hyperparams.get("Dataset.ClassCount", 0)
121
+ nDropOutKeepProb = hyperparams.get("Training.Regularize.DropOut.KeepProb", None)
122
+ if nDropOutKeepProb is None:
123
+ oModelHyperparams.dropout_prob = hyperparams.get("Training.Regularize.DropOut.DropProb", 0.0)
124
+ else:
125
+ oModelHyperparams.dropout_prob = 1.0 - nDropOutKeepProb
126
+
127
+ return oModelHyperparams
128
+ # --------------------------------------------------------------------------------------------------------------------
@@ -0,0 +1,91 @@
1
+ # ======================================================================================
2
+ #
3
+ # Rapid Deep Neural Networks
4
+ #
5
+ # Licensed under the MIT License
6
+ # ______________________________________________________________________________________
7
+ # ......................................................................................
8
+
9
+ # Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
10
+
11
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
12
+ # of this software and associated documentation files (the "Software"), to deal
13
+ # in the Software without restriction, including without limitation the rights
14
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15
+ # copies of the Software, and to permit persons to whom the Software is
16
+ # furnished to do so, subject to the following conditions:
17
+
18
+ # The above copyright notice and this permission notice shall be included in all
19
+ # copies or substantial portions of the Software.
20
+
21
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27
+ # SOFTWARE.
28
+
29
+ # .......................................................................................
30
+ from radnn import FileStore
31
+ from radnn.data.structs import Tree, TreeNode
32
+
33
+ from torchinfo import ModelStatistics, summary
34
+ from torchinfo.layer_info import LayerInfo
35
+ from .model_hyperparams import ModelHyperparams
36
+
37
+
38
+ class ModelInfo(object):
39
+ def __init__(self, model=None, hyperparams: ModelHyperparams | dict = None, depth=6):
40
+ self.model = model
41
+ self.hyperparams = hyperparams
42
+ if isinstance(hyperparams, dict):
43
+ self.hyperparams = ModelHyperparams.from_dict(hyperparams)
44
+ self.model_tree: Tree = None
45
+
46
+ if self.model is not None:
47
+ nInputDims = tuple([1, hyperparams.input_channels] + hyperparams.input_resolution)
48
+ self.summary: ModelStatistics = summary(model, nInputDims, device='cpu', depth=depth)
49
+ self.build_tree()
50
+
51
+ # --------------------------------------------------------------------------------------------------------------------
52
+ @property
53
+ def filename(self):
54
+ return f'{self.hyperparams.model_name}_{str(self.hyperparams.input_dims).replace(", ", "x")[1:-1]}_cls{str(self.hyperparams.class_count)}'
55
+
56
+ # --------------------------------------------------------------------------------------------------------------------
57
+ def save(self, fs: str | FileStore):
58
+ if isinstance(fs, str):
59
+ fs = FileStore(fs)
60
+ fs.artifact(self.filename + ".pkl").save(self.summary, is_overwriting=True)
61
+ with open(fs.file(self.filename + ".txt"), "w", encoding="utf-8") as oFile:
62
+ print(self.summary, file=oFile)
63
+
64
+ # --------------------------------------------------------------------------------------------------------------------
65
+ def load(self, fs: str | FileStore):
66
+ bIsLoaded = False
67
+ if isinstance(fs, str):
68
+ fs = FileStore(fs)
69
+ self.summary = fs.artifact(self.filename + ".pkl").load()
70
+ bIsLoaded = self.summary is not None
71
+ if bIsLoaded:
72
+ self.build_tree()
73
+ '''
74
+ oNodeList = self.model_tree.traverse_depth_first()
75
+ for oNode in oNodeList:
76
+ print(oNode.path)
77
+ '''
78
+ return bIsLoaded
79
+
80
+ # --------------------------------------------------------------------------------------------------------------------
81
+ def build_tree(self):
82
+ def recurse_build_subtree(source_node: LayerInfo, target_node: TreeNode):
83
+ for nIndex, oSourceChild in enumerate(source_node.children):
84
+ sNodeName = f"{oSourceChild.class_name}: {oSourceChild.depth}-{oSourceChild.depth_index}"
85
+ oTargetChild: TreeNode = target_node.new_child(sNodeName)
86
+ oTargetChild.value = oSourceChild
87
+ recurse_build_subtree(oSourceChild, oTargetChild)
88
+
89
+ self.model_tree = Tree()
90
+ recurse_build_subtree(self.summary.summary_list[0], self.model_tree.root)
91
+ # --------------------------------------------------------------------------------------------------------------------
@@ -36,22 +36,33 @@ class PlotLearningCurve(object):
36
36
  self.model_name = model_name
37
37
  print("Keys in training process log:", self.metrics_dict.keys())
38
38
  # --------------------------------------------------------------------------------------
39
- def prepare(self, metric_key="accuracy", custom_title=None, is_legend_right=False):
39
+ def prepare(self, metric_key="accuracy", custom_title=None, metric_low=0, metric_high=1.0, is_legend_right=False, is_keras=False):
40
+ sTrainMetricKey = metric_key
41
+ if not is_keras:
42
+ sTrainMetricKey = "train_" + metric_key
43
+ sValMetricKey = "val_" + metric_key
44
+ oLegend = ["training"]
45
+ bHasValidation = sValMetricKey in self.metrics_dict
46
+ oLegend.append("validation")
47
+
48
+
40
49
  plt.clf()
41
- plt.plot(self.metrics_dict[metric_key])
42
- sValidationMetricName = "val_" + metric_key
43
- if sValidationMetricName in self.metrics_dict:
44
- plt.plot(self.metrics_dict[sValidationMetricName])
50
+ plt.plot(self.metrics_dict[sTrainMetricKey])
51
+ if bHasValidation:
52
+ plt.plot(self.metrics_dict[sValMetricKey])
53
+
45
54
  if custom_title is None:
46
55
  plt.title(self.model_name + ' ' + metric_key)
47
56
  else:
48
57
  plt.title(self.model_name + ' ' + custom_title)
49
- plt.ylabel(metric_key)
58
+
59
+ plt.ylabel(metric_key.capitalize().title())
50
60
  plt.xlabel("Epoch")
61
+ plt.ylim([metric_low, metric_high])
51
62
  if is_legend_right:
52
- plt.legend(["train", 'validation'], loc="upper right")
63
+ plt.legend(oLegend, loc="upper right")
53
64
  else:
54
- plt.legend(["train", "validation"], loc="upper left")
65
+ plt.legend(oLegend, loc="upper left")
55
66
  return self
56
67
  # --------------------------------------------------------------------------------------
57
68
  def prepare_cost(self, cost_function=None):
radnn/system/__init__.py CHANGED
@@ -8,3 +8,4 @@ from .tee_logger import TeeLogger
8
8
 
9
9
 
10
10
 
11
+
@@ -4,4 +4,4 @@ from .picklefile import PickleFile
4
4
  from .textfile import TextFile
5
5
  from .csvfile import CSVFile
6
6
  from .filelist import FileList
7
-
7
+ from .zipfile import ZipFile
@@ -6,7 +6,7 @@
6
6
  # ______________________________________________________________________________________
7
7
  # ......................................................................................
8
8
 
9
- # Copyright (c) 2018-2025 Pantelis I. Kaplanoglou
9
+ # Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
10
10
 
11
11
  # Permission is hereby granted, free of charge, to any person obtaining a copy
12
12
  # of this software and associated documentation files (the "Software"), to deal
@@ -28,7 +28,7 @@
28
28
 
29
29
  # .......................................................................................
30
30
  import pandas as pd
31
-
31
+ import numpy as np
32
32
 
33
33
  from .fileobject import FileObject
34
34
 
@@ -42,8 +42,40 @@ class CSVFile(FileObject):
42
42
  dResult = pd.read_csv(sFileName, delimiter=delimiter, encoding=encoding)
43
43
  return dResult
44
44
  # ----------------------------------------------------------------------------------
45
- def save(self, obj, filename=None):
46
- # TODO Save numpy array to pandas dataframe then to CSV
47
- pass
45
+ # TODO: Iterate large csv files row by row as done in TextFile
46
+ #@property
47
+ #def rows(self):
48
+ # ----------------------------------------------------------------------------------
49
+ def save(self, obj, filename=None, headers=None, delimiter=",", encoding="utf-8"):
50
+ sFileName = self._useFileName(filename)
51
+
52
+ if isinstance(obj, dict):
53
+ nMaxRowCols = 0
54
+ oList = []
55
+ for k, v in obj.items():
56
+ oRow = [k]
57
+ if isinstance(v, list):
58
+ oRow.extend(v)
59
+ else:
60
+ oRow.append(v)
61
+ if len(oRow) > nMaxRowCols:
62
+ nMaxRowCols = len(oRow)
63
+ oList.append(oRow)
64
+
65
+ obj = []
66
+ for oRow in oList:
67
+ oRowToAdd = oRow + [None]*(nMaxRowCols - len(oRow))
68
+ obj.append(oRowToAdd)
69
+ if len(headers) < nMaxRowCols:
70
+ headers = headers + [None]*(nMaxRowCols - len(headers))
71
+
72
+ if isinstance(obj, list):
73
+ df = pd.DataFrame(obj, columns=headers)
74
+ df.to_csv(sFileName, index=False, header=(headers is not None), sep=delimiter, encoding=encoding)
48
75
 
76
+ elif isinstance(obj, np.ndarray):
77
+ if len(headers) < obj.shape[1]:
78
+ headers = headers + [None]*(obj.shape[1] - len(headers))
79
+ df = pd.DataFrame(obj, columns=headers)
80
+ df.to_csv(sFileName, index=False, header=(headers is not None), sep=delimiter, encoding=encoding)
49
81
  # ----------------------------------------------------------------------------------
@@ -1,3 +1,33 @@
1
+ # ======================================================================================
2
+ #
3
+ # Rapid Deep Neural Networks
4
+ #
5
+ # Licensed under the MIT License
6
+ # ______________________________________________________________________________________
7
+ # ......................................................................................
8
+
9
+ # Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
10
+
11
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
12
+ # of this software and associated documentation files (the "Software"), to deal
13
+ # in the Software without restriction, including without limitation the rights
14
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15
+ # copies of the Software, and to permit persons to whom the Software is
16
+ # furnished to do so, subject to the following conditions:
17
+
18
+ # The above copyright notice and this permission notice shall be included in all
19
+ # copies or substantial portions of the Software.
20
+
21
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27
+ # SOFTWARE.
28
+
29
+ # .......................................................................................
30
+
1
31
  import os
2
32
 
3
33
 
@@ -6,7 +6,7 @@
6
6
  # ______________________________________________________________________________________
7
7
  # ......................................................................................
8
8
 
9
- # Copyright (c) 2018-2025 Pantelis I. Kaplanoglou
9
+ # Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
10
10
 
11
11
  # Permission is hereby granted, free of charge, to any person obtaining a copy
12
12
  # of this software and associated documentation files (the "Software"), to deal
@@ -48,6 +48,16 @@ class FileObject(object):
48
48
  if not self.default_file_extension.startswith("."):
49
49
  self.default_file_extension = "." + self.default_file_extension
50
50
  # ----------------------------------------------------------------------------------
51
+ def close(self):
52
+ pass
53
+ # ----------------------------------------------------------------------------------
54
+ def __enter__(self):
55
+ return self
56
+ # ----------------------------------------------------------------------------------
57
+ def __exit__(self, exc_type, exc_value, traceback):
58
+ self.close()
59
+ return True
60
+ # ----------------------------------------------------------------------------------
51
61
  def _useFileName(self, filename):
52
62
  if filename is not None:
53
63
  self.filename = filename
@@ -6,7 +6,7 @@
6
6
  # ______________________________________________________________________________________
7
7
  # ......................................................................................
8
8
 
9
- # Copyright (c) 2018-2025 Pantelis I. Kaplanoglou
9
+ # Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
10
10
 
11
11
  # Permission is hereby granted, free of charge, to any person obtaining a copy
12
12
  # of this software and associated documentation files (the "Software"), to deal
@@ -6,7 +6,7 @@
6
6
  # ______________________________________________________________________________________
7
7
  # ......................................................................................
8
8
 
9
- # Copyright (c) 2018-2025 Pantelis I. Kaplanoglou
9
+ # Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
10
10
 
11
11
  # Permission is hereby granted, free of charge, to any person obtaining a copy
12
12
  # of this software and associated documentation files (the "Software"), to deal
@@ -30,36 +30,64 @@
30
30
  import os
31
31
  import json
32
32
  import glob
33
+ from collections import OrderedDict
34
+
33
35
  from .fileobject import FileObject
34
36
 
37
+ #TODO: jsonpickle
38
+ #https://stackoverflow.com/questions/3768895/how-to-make-a-class-json-serializable
39
+
35
40
  class JSONFile(FileObject):
36
41
  # ----------------------------------------------------------------------------------
37
42
  def __init__(self, filename, parent_folder=None, error_template=None):
38
43
  super(JSONFile, self).__init__(filename, parent_folder, error_template, "json")
39
44
  # ----------------------------------------------------------------------------------
40
- def load(self, filename=None):
45
+ def load(self, filename=None, encoding=None):
41
46
  filename = self._useFileName(filename)
42
47
 
43
48
  dResult = None
44
49
  if os.path.exists(filename):
45
- with open(filename) as oFile:
46
- sJSON = oFile.read()
47
- dResult = json.loads(sJSON)
50
+
51
+ sJSON = None
52
+ if os.path.isfile(filename):
53
+ oEncodingToTry = ["utf-8", "utf-16", "latin1", "ascii"] # Add more if needed
54
+ if encoding is None:
55
+ bIsLoaded = False
56
+ for sEnc in oEncodingToTry:
57
+ try:
58
+ with open(filename, "r", encoding=sEnc) as oFile:
59
+ sJSON = oFile.read()
60
+ bIsLoaded = True
61
+ break
62
+ except (UnicodeDecodeError, UnicodeError):
63
+ continue
64
+ if not bIsLoaded:
65
+ raise ValueError("Unsupported encoding")
66
+ else:
67
+ with open(filename, "r", encoding=encoding) as oFile:
68
+ sJSON = oFile.read()
69
+ if sJSON is not None:
70
+ dResult = json.loads(sJSON, object_pairs_hook=OrderedDict)
48
71
  else:
49
72
  if self.error_template is not None:
50
73
  raise Exception(self.error_template % filename)
51
74
 
52
75
  return dResult
53
76
  # ----------------------------------------------------------------------------------
54
- def save(self, obj, filename=None, is_sorted_keys=True):
77
+ def save(self, obj, filename=None, is_sorted_keys=False, is_utf8=False):
55
78
  filename = self._useFileName(filename)
56
79
 
57
80
  if obj is not None:
58
81
  if isinstance(obj, dict):
59
- sJSON = json.dumps(obj, sort_keys=is_sorted_keys, indent=4)
82
+ sJSON = json.dumps(obj, sort_keys=is_sorted_keys, indent=4, ensure_ascii=is_utf8)
60
83
  else:
61
- sJSON = json.dumps(obj, default=lambda o: obj.__dict__, sort_keys=is_sorted_keys, indent=4)
62
- with open(filename, "w") as oFile:
84
+ sJSON = json.dumps(obj, default=lambda o: obj.__dict__, sort_keys=is_sorted_keys, indent=4, ensure_ascii=is_utf8)
85
+
86
+ sEncoding = "utf-8"
87
+ if not is_utf8:
88
+ sEncoding = "utf-16"
89
+
90
+ with open(filename, "w", encoding=sEncoding) as oFile:
63
91
  oFile.write(sJSON)
64
92
  oFile.close()
65
93
  # ----------------------------------------------------------------------------------
@@ -6,7 +6,7 @@
6
6
  # ______________________________________________________________________________________
7
7
  # ......................................................................................
8
8
 
9
- # Copyright (c) 2018-2025 Pantelis I. Kaplanoglou
9
+ # Copyright (c) 2018-2026 Pantelis I. Kaplanoglou
10
10
 
11
11
  # Permission is hereby granted, free of charge, to any person obtaining a copy
12
12
  # of this software and associated documentation files (the "Software"), to deal
@@ -64,7 +64,7 @@ class PickleFile(FileObject):
64
64
  del self._cached
65
65
  self._cached = None
66
66
  # ----------------------------------------------------------------------------------
67
- def load(self, filename, is_python2_format=False, error_template=None):
67
+ def load(self, filename: str|None = None, is_python2_format: bool=False, error_template: str|None=None):
68
68
  """
69
69
  Deserializes the data from a pickle file if it exists.
70
70
  Parameters
@@ -101,7 +101,7 @@ class PickleFile(FileObject):
101
101
 
102
102
  return oData
103
103
  # ----------------------------------------------------------------------------------
104
- def save(self, obj, filename=None, is_overwriting=False, extra_display_label=None):
104
+ def save(self, obj, filename: str | None =None, is_overwriting: bool =False, extra_display_label: str|None=None):
105
105
  """
106
106
  Serializes the data to a pickle file if it does not exists.
107
107
  Parameters