radnn 0.0.7.2__py3-none-any.whl → 0.0.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- radnn/__init__.py +7 -5
- radnn/core.py +44 -28
- radnn/data/__init__.py +8 -0
- radnn/data/data_feed.py +147 -0
- radnn/data/dataset_base.py +3 -5
- radnn/data/dataset_folder.py +55 -0
- radnn/data/image_dataset.py +0 -2
- radnn/data/image_dataset_files.py +175 -0
- radnn/data/preprocess/normalizer.py +7 -1
- radnn/data/preprocess/standardizer.py +9 -2
- radnn/data/sample_set.py +30 -17
- radnn/data/sequence_dataset.py +0 -2
- radnn/data/subset_type.py +45 -0
- radnn/data/tf_classification_data_feed.py +113 -0
- radnn/errors.py +29 -0
- radnn/evaluation/evaluate_classification.py +7 -3
- radnn/experiment/ml_experiment.py +29 -0
- radnn/experiment/ml_experiment_config.py +61 -32
- radnn/experiment/ml_experiment_env.py +6 -2
- radnn/experiment/ml_experiment_store.py +0 -1
- radnn/images/__init__.py +2 -0
- radnn/images/colors.py +28 -0
- radnn/images/image_processor.py +513 -0
- radnn/learn/learning_algorithm.py +4 -3
- radnn/ml_system.py +59 -18
- radnn/plots/plot_auto_multi_image.py +27 -17
- radnn/plots/plot_confusion_matrix.py +7 -4
- radnn/plots/plot_learning_curve.py +7 -3
- radnn/plots/plot_multi_scatter.py +7 -3
- radnn/plots/plot_roc.py +8 -4
- radnn/plots/plot_voronoi_2d.py +8 -5
- radnn/stats/__init__.py +1 -0
- radnn/stats/descriptive_stats.py +45 -0
- radnn/system/files/__init__.py +1 -0
- radnn/system/files/csvfile.py +8 -5
- radnn/system/files/filelist.py +40 -0
- radnn/system/files/fileobject.py +9 -4
- radnn/system/files/imgfile.py +8 -4
- radnn/system/files/jsonfile.py +8 -4
- radnn/system/files/picklefile.py +8 -4
- radnn/system/files/textfile.py +37 -7
- radnn/system/filestore.py +36 -18
- radnn/system/filesystem.py +8 -3
- radnn/system/hosts/colab_host.py +29 -0
- radnn/system/hosts/linux_host.py +29 -0
- radnn/system/hosts/windows_host.py +39 -1
- radnn/system/tee_logger.py +7 -3
- radnn/system/threads/__init__.py +5 -0
- radnn/system/threads/semaphore_lock.py +58 -0
- radnn/system/threads/thread_context.py +175 -0
- radnn/system/threads/thread_safe_queue.py +163 -0
- radnn/system/threads/thread_safe_string_collection.py +66 -0
- radnn/system/threads/thread_worker.py +68 -0
- radnn/utils.py +96 -2
- {radnn-0.0.7.2.dist-info → radnn-0.0.8.dist-info}/METADATA +1 -1
- radnn-0.0.8.dist-info/RECORD +70 -0
- radnn-0.0.7.2.dist-info/RECORD +0 -53
- {radnn-0.0.7.2.dist-info → radnn-0.0.8.dist-info}/LICENSE.txt +0 -0
- {radnn-0.0.7.2.dist-info → radnn-0.0.8.dist-info}/WHEEL +0 -0
- {radnn-0.0.7.2.dist-info → radnn-0.0.8.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,10 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
1
7
|
# ......................................................................................
|
|
2
|
-
# MIT License
|
|
3
8
|
|
|
4
9
|
# Copyright (c) 2022-2025 Pantelis I. Kaplanoglou
|
|
5
10
|
|
|
@@ -21,8 +26,7 @@
|
|
|
21
26
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
27
|
# SOFTWARE.
|
|
23
28
|
|
|
24
|
-
#
|
|
25
|
-
|
|
29
|
+
# .......................................................................................
|
|
26
30
|
import numpy as np
|
|
27
31
|
import matplotlib.pyplot as plt
|
|
28
32
|
|
|
@@ -44,9 +48,9 @@ class AutoMultiImagePlot(object):
|
|
|
44
48
|
self.row_count = len(self.rows)
|
|
45
49
|
self.row_titles.append(row_title)
|
|
46
50
|
|
|
47
|
-
def add_column(self,
|
|
51
|
+
def add_column(self, image, image_title=None, color_map=None, aspect=None, extent=None):
|
|
48
52
|
oRowColumns = self.rows[self.current_row]
|
|
49
|
-
dImage = {"image":
|
|
53
|
+
dImage = {"image": image, "title": image_title
|
|
50
54
|
, "cmap": color_map, "aspect": aspect
|
|
51
55
|
, "extend": extent}
|
|
52
56
|
|
|
@@ -63,7 +67,7 @@ class AutoMultiImagePlot(object):
|
|
|
63
67
|
nColumns = restrict_columns
|
|
64
68
|
if nColumns is None:
|
|
65
69
|
nColumns = self.max_col_count
|
|
66
|
-
fig, oSubplotGrid = plt.subplots(nrows=self.row_count, ncols=nColumns
|
|
70
|
+
fig, oSubplotGrid = plt.subplots( nrows=self.row_count, ncols=nColumns
|
|
67
71
|
, figsize=figure_size
|
|
68
72
|
, subplot_kw={'xticks': [], 'yticks': []})
|
|
69
73
|
bIsSingleRow = self.row_count == 1
|
|
@@ -73,32 +77,38 @@ class AutoMultiImagePlot(object):
|
|
|
73
77
|
if title is None:
|
|
74
78
|
title = self.title
|
|
75
79
|
fig.suptitle(title)
|
|
76
|
-
|
|
77
80
|
for nRowIndex, oRowColumns in enumerate(self.rows):
|
|
78
81
|
if len(oRowColumns) > 0:
|
|
79
82
|
sRowTitle = self.row_titles[nRowIndex]
|
|
80
|
-
|
|
81
|
-
nIncr = nImageCount //
|
|
83
|
+
nRowImageCount = len(oRowColumns)
|
|
84
|
+
#nIncr = nImageCount // nRowColumnCount
|
|
85
|
+
nIncr = 1
|
|
82
86
|
nImageIndex = 0
|
|
83
87
|
for nColIndex in range(nColumns):
|
|
84
|
-
bMustPlot =
|
|
85
|
-
if (nIncr == 0) and (nColIndex > 0):
|
|
86
|
-
|
|
88
|
+
bMustPlot = nColIndex < nRowImageCount
|
|
89
|
+
#if (nIncr == 0) and (nColIndex > 0):
|
|
90
|
+
# bMustPlot = False
|
|
87
91
|
|
|
88
92
|
if bMustPlot:
|
|
89
93
|
dImage = oRowColumns[nImageIndex]
|
|
90
94
|
oSubPlot = oSubplotGrid[nRowIndex, nColIndex]
|
|
91
|
-
|
|
95
|
+
sTitle = dImage['title']
|
|
96
|
+
if sTitle is not None:
|
|
97
|
+
oSubPlot.title.set_text(sTitle)
|
|
98
|
+
oSubPlot.set_xticks([])
|
|
99
|
+
oSubPlot.set_yticks([])
|
|
92
100
|
oSubPlot.imshow(dImage["image"], cmap=dImage["cmap"],
|
|
93
101
|
aspect=dImage["aspect"], extent=dImage["extend"],
|
|
94
102
|
vmin=self.min, vmax=self.max
|
|
95
103
|
)
|
|
104
|
+
|
|
96
105
|
if nColIndex == 0:
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
106
|
+
if sRowTitle is not None:
|
|
107
|
+
oSubPlot.text(0.0, 0.5, sRowTitle, transform=oSubPlot.transAxes,
|
|
108
|
+
horizontalalignment='right', verticalalignment='center',
|
|
109
|
+
fontsize=9, fontweight='bold')
|
|
100
110
|
nImageIndex += nIncr
|
|
101
|
-
|
|
111
|
+
fig.subplots_adjust(wspace=0.1, hspace=0.6)
|
|
102
112
|
return self
|
|
103
113
|
|
|
104
114
|
# --------------------------------------------------------------------------------------
|
|
@@ -1,5 +1,10 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
1
7
|
# ......................................................................................
|
|
2
|
-
# MIT License
|
|
3
8
|
|
|
4
9
|
# Copyright (c) 2020-2025 Pantelis I. Kaplanoglou
|
|
5
10
|
|
|
@@ -21,9 +26,7 @@
|
|
|
21
26
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
27
|
# SOFTWARE.
|
|
23
28
|
|
|
24
|
-
#
|
|
25
|
-
|
|
26
|
-
|
|
29
|
+
# .......................................................................................
|
|
27
30
|
import matplotlib.pyplot as plt
|
|
28
31
|
|
|
29
32
|
class PlotConfusionMatrix(object):
|
|
@@ -1,5 +1,10 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
1
7
|
# ......................................................................................
|
|
2
|
-
# MIT License
|
|
3
8
|
|
|
4
9
|
# Copyright (c) 2020-2025 Pantelis I. Kaplanoglou
|
|
5
10
|
|
|
@@ -21,8 +26,7 @@
|
|
|
21
26
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
27
|
# SOFTWARE.
|
|
23
28
|
|
|
24
|
-
#
|
|
25
|
-
|
|
29
|
+
# .......................................................................................
|
|
26
30
|
import matplotlib.pyplot as plt
|
|
27
31
|
|
|
28
32
|
class PlotLearningCurve(object):
|
|
@@ -1,5 +1,10 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
1
7
|
# ......................................................................................
|
|
2
|
-
# MIT License
|
|
3
8
|
|
|
4
9
|
# Copyright (c) 2022-2025 Pantelis I. Kaplanoglou
|
|
5
10
|
|
|
@@ -21,8 +26,7 @@
|
|
|
21
26
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
27
|
# SOFTWARE.
|
|
23
28
|
|
|
24
|
-
#
|
|
25
|
-
|
|
29
|
+
# .......................................................................................
|
|
26
30
|
import numpy as np
|
|
27
31
|
import matplotlib.pyplot as plt
|
|
28
32
|
from matplotlib import cm
|
radnn/plots/plot_roc.py
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
1
7
|
# ......................................................................................
|
|
2
|
-
# MIT License
|
|
3
8
|
|
|
4
|
-
# Copyright (c)
|
|
9
|
+
# Copyright (c) 2023-2025 Pantelis I. Kaplanoglou
|
|
5
10
|
|
|
6
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
@@ -21,8 +26,7 @@
|
|
|
21
26
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
27
|
# SOFTWARE.
|
|
23
28
|
|
|
24
|
-
#
|
|
25
|
-
|
|
29
|
+
# .......................................................................................
|
|
26
30
|
from sklearn import metrics
|
|
27
31
|
import matplotlib.pyplot as plt
|
|
28
32
|
|
radnn/plots/plot_voronoi_2d.py
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
1
7
|
# ......................................................................................
|
|
2
|
-
# MIT License
|
|
3
8
|
|
|
4
|
-
# Copyright (c)
|
|
9
|
+
# Copyright (c) 2023-2025 Pantelis I. Kaplanoglou
|
|
5
10
|
|
|
6
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
@@ -21,9 +26,7 @@
|
|
|
21
26
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
27
|
# SOFTWARE.
|
|
23
28
|
|
|
24
|
-
#
|
|
25
|
-
|
|
26
|
-
|
|
29
|
+
# .......................................................................................
|
|
27
30
|
import matplotlib.pyplot as plt
|
|
28
31
|
import numpy as np
|
|
29
32
|
from matplotlib import cm
|
radnn/stats/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .descriptive_stats import DescriptiveStats
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
class DescriptiveStats(object):
|
|
4
|
+
INNER_FENCE_RATIO = 1.5
|
|
5
|
+
OUTER_FENCE_RATIO = 3.0
|
|
6
|
+
|
|
7
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
8
|
+
def __init__(self, data):
|
|
9
|
+
if (data.ndim == 2) and (np.prod(data.shape) == np.max(data.shape)):
|
|
10
|
+
data = data.reshape(-1)
|
|
11
|
+
|
|
12
|
+
self.min = np.min(data)
|
|
13
|
+
self.max = np.max(data)
|
|
14
|
+
|
|
15
|
+
self.mean = np.mean(data)
|
|
16
|
+
self.std = np.std(data)
|
|
17
|
+
|
|
18
|
+
self.q1 = np.percentile(data, q=25)
|
|
19
|
+
self.median = np.median(data)
|
|
20
|
+
self.q2 = self.median
|
|
21
|
+
self.q3 = np.percentile(data, q=75)
|
|
22
|
+
self.iq_range = self.q3 - self.q1
|
|
23
|
+
|
|
24
|
+
self.inner_fence_low = self.q1 - type(self).INNER_FENCE_RATIO * self.iq_range
|
|
25
|
+
self.inner_fence_high = self.q3 + type(self).INNER_FENCE_RATIO * self.iq_range
|
|
26
|
+
|
|
27
|
+
self.outer_fence_low = self.q1 - type(self).OUTER_FENCE_RATIO * self.iq_range
|
|
28
|
+
self.outer_fence_high = self.q3 + type(self).OUTER_FENCE_RATIO * self.iq_range
|
|
29
|
+
|
|
30
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
31
|
+
def outliers(self, data, is_using_outer_fence=False, dtype=np.float32):
|
|
32
|
+
if is_using_outer_fence:
|
|
33
|
+
nResult = np.asarray([x for x in data if x < self.outer_fence_low or x > self.outer_fence_high], dtype=dtype)
|
|
34
|
+
else:
|
|
35
|
+
nResult = np.asarray([x for x in data if x < self.inner_fence_low or x > self.inner_fence_high], dtype=dtype)
|
|
36
|
+
return nResult
|
|
37
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
38
|
+
def __str__(self):
|
|
39
|
+
sResult = "range=[%.6f,%.6f] mean=%6f std=%.6f iqrange=%.6f Q1=%.6f median (Q2)=%.6f Q3=%.6f max (Q4)=%.6f" % (self.min, self.max,
|
|
40
|
+
self.mean, self.std,
|
|
41
|
+
self.iq_range,
|
|
42
|
+
self.q1, self.median,
|
|
43
|
+
self.q3, self.max)
|
|
44
|
+
return sResult
|
|
45
|
+
# --------------------------------------------------------------------------------------------------------------------
|
radnn/system/files/__init__.py
CHANGED
radnn/system/files/csvfile.py
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
1
7
|
# ......................................................................................
|
|
2
|
-
# MIT License
|
|
3
8
|
|
|
4
|
-
# Copyright (c)
|
|
9
|
+
# Copyright (c) 2018-2025 Pantelis I. Kaplanoglou
|
|
5
10
|
|
|
6
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
@@ -21,9 +26,7 @@
|
|
|
21
26
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
27
|
# SOFTWARE.
|
|
23
28
|
|
|
24
|
-
#
|
|
25
|
-
|
|
26
|
-
|
|
29
|
+
# .......................................................................................
|
|
27
30
|
import pandas as pd
|
|
28
31
|
|
|
29
32
|
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
# ======================================================================================================================
|
|
5
|
+
class FileListFullPathIterator(object):
|
|
6
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
7
|
+
def __init__(self, filelist):
|
|
8
|
+
self.filelist = filelist
|
|
9
|
+
self.index = 0
|
|
10
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
11
|
+
def __iter__(self):
|
|
12
|
+
return self
|
|
13
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
14
|
+
def __next__(self):
|
|
15
|
+
if self.index >= len(self.filelist):
|
|
16
|
+
raise StopIteration
|
|
17
|
+
sFileFullPath = os.path.join(self.filelist.parent_folder_path, self.filelist[self.index])
|
|
18
|
+
self.index += 1
|
|
19
|
+
return sFileFullPath
|
|
20
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
21
|
+
# ======================================================================================================================
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# ======================================================================================================================
|
|
28
|
+
class FileList(list):
|
|
29
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
30
|
+
def __init__(self, parent_folder_path=None):
|
|
31
|
+
self.parent_folder_path = parent_folder_path
|
|
32
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
33
|
+
@property
|
|
34
|
+
def full_paths(self):
|
|
35
|
+
if self.parent_folder_path is not None:
|
|
36
|
+
return FileListFullPathIterator(self)
|
|
37
|
+
else:
|
|
38
|
+
return None
|
|
39
|
+
# --------------------------------------------------------------------------------------------------------------------
|
|
40
|
+
# ======================================================================================================================
|
radnn/system/files/fileobject.py
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
1
7
|
# ......................................................................................
|
|
2
|
-
# MIT License
|
|
3
8
|
|
|
4
|
-
# Copyright (c)
|
|
9
|
+
# Copyright (c) 2018-2025 Pantelis I. Kaplanoglou
|
|
5
10
|
|
|
6
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
@@ -21,10 +26,10 @@
|
|
|
21
26
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
27
|
# SOFTWARE.
|
|
23
28
|
|
|
24
|
-
#
|
|
25
|
-
|
|
29
|
+
# .......................................................................................
|
|
26
30
|
import os
|
|
27
31
|
import glob
|
|
32
|
+
|
|
28
33
|
class FileObject(object):
|
|
29
34
|
# ----------------------------------------------------------------------------------
|
|
30
35
|
def __init__(self, filename, parent_folder=None, error_template=None, default_file_extension=None):
|
radnn/system/files/imgfile.py
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
1
7
|
# ......................................................................................
|
|
2
|
-
# MIT License
|
|
3
8
|
|
|
4
|
-
# Copyright (c)
|
|
9
|
+
# Copyright (c) 2018-2025 Pantelis I. Kaplanoglou
|
|
5
10
|
|
|
6
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
@@ -21,8 +26,7 @@
|
|
|
21
26
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
27
|
# SOFTWARE.
|
|
23
28
|
|
|
24
|
-
#
|
|
25
|
-
|
|
29
|
+
# .......................................................................................
|
|
26
30
|
import cv2
|
|
27
31
|
|
|
28
32
|
|
radnn/system/files/jsonfile.py
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
1
7
|
# ......................................................................................
|
|
2
|
-
# MIT License
|
|
3
8
|
|
|
4
|
-
# Copyright (c)
|
|
9
|
+
# Copyright (c) 2018-2025 Pantelis I. Kaplanoglou
|
|
5
10
|
|
|
6
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
@@ -21,8 +26,7 @@
|
|
|
21
26
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
27
|
# SOFTWARE.
|
|
23
28
|
|
|
24
|
-
#
|
|
25
|
-
|
|
29
|
+
# .......................................................................................
|
|
26
30
|
import os
|
|
27
31
|
import json
|
|
28
32
|
import glob
|
radnn/system/files/picklefile.py
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
1
7
|
# ......................................................................................
|
|
2
|
-
# MIT License
|
|
3
8
|
|
|
4
|
-
# Copyright (c)
|
|
9
|
+
# Copyright (c) 2018-2025 Pantelis I. Kaplanoglou
|
|
5
10
|
|
|
6
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
@@ -21,8 +26,7 @@
|
|
|
21
26
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
27
|
# SOFTWARE.
|
|
23
28
|
|
|
24
|
-
#
|
|
25
|
-
|
|
29
|
+
# .......................................................................................
|
|
26
30
|
import os
|
|
27
31
|
import sys
|
|
28
32
|
import shutil
|
radnn/system/files/textfile.py
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
1
7
|
# ......................................................................................
|
|
2
|
-
# MIT License
|
|
3
8
|
|
|
4
|
-
# Copyright (c)
|
|
9
|
+
# Copyright (c) 2018-2025 Pantelis I. Kaplanoglou
|
|
5
10
|
|
|
6
11
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
12
|
# of this software and associated documentation files (the "Software"), to deal
|
|
@@ -21,10 +26,10 @@
|
|
|
21
26
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
27
|
# SOFTWARE.
|
|
23
28
|
|
|
24
|
-
#
|
|
25
|
-
|
|
29
|
+
# .......................................................................................
|
|
26
30
|
import os
|
|
27
31
|
import numpy as np
|
|
32
|
+
import locale
|
|
28
33
|
from .fileobject import FileObject
|
|
29
34
|
|
|
30
35
|
|
|
@@ -33,8 +38,33 @@ class TextFile(FileObject):
|
|
|
33
38
|
def __init__(self, filename, parent_folder=None, error_template=None, is_verbose=False):
|
|
34
39
|
super(TextFile, self).__init__(filename, parent_folder, error_template)
|
|
35
40
|
self.is_verbose = is_verbose
|
|
41
|
+
|
|
42
|
+
# ----------------------------------------------------------------------------------
|
|
43
|
+
def load(self, filename=None, encoding=None):
|
|
44
|
+
filename = self._useFileName(filename)
|
|
45
|
+
|
|
46
|
+
oEncodingToTry = ["utf-8", "utf-16", "latin1", "ascii"] # Add more if needed
|
|
47
|
+
|
|
48
|
+
sText = None
|
|
49
|
+
if encoding is None:
|
|
50
|
+
bIsLoaded = False
|
|
51
|
+
for sEnc in oEncodingToTry:
|
|
52
|
+
try:
|
|
53
|
+
with open(filename, "r", encoding=sEnc) as oFile:
|
|
54
|
+
sText = oFile.read()
|
|
55
|
+
bIsLoaded = True
|
|
56
|
+
break
|
|
57
|
+
except (UnicodeDecodeError, UnicodeError):
|
|
58
|
+
continue
|
|
59
|
+
if not bIsLoaded:
|
|
60
|
+
raise ValueError("Unsupported encoding")
|
|
61
|
+
else:
|
|
62
|
+
with open(filename, "r", encoding=encoding) as oFile:
|
|
63
|
+
sText = oFile.read()
|
|
64
|
+
|
|
65
|
+
return sText
|
|
36
66
|
# --------------------------------------------------------------------------------------------------------------------
|
|
37
|
-
def save(self, text_obj, filename=None):
|
|
67
|
+
def save(self, text_obj, filename=None, encoding="utf-8"):
|
|
38
68
|
sFilename = self._useFileName(filename)
|
|
39
69
|
|
|
40
70
|
"""
|
|
@@ -57,12 +87,12 @@ class TextFile(FileObject):
|
|
|
57
87
|
bIsIterable = text_obj.dtype = str
|
|
58
88
|
|
|
59
89
|
if bIsIterable:
|
|
60
|
-
with open(sFilename, "w") as oFile:
|
|
90
|
+
with open(sFilename, "w", encoding=encoding) as oFile:
|
|
61
91
|
for sLine in text_obj:
|
|
62
92
|
print(sLine, file=oFile)
|
|
63
93
|
oFile.close()
|
|
64
94
|
else:
|
|
65
|
-
with open(sFilename, "w") as oFile:
|
|
95
|
+
with open(sFilename, "w", encoding=encoding) as oFile:
|
|
66
96
|
print(text_obj, file=oFile)
|
|
67
97
|
oFile.close()
|
|
68
98
|
return True
|
radnn/system/filestore.py
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
|
+
# ======================================================================================
|
|
2
|
+
#
|
|
3
|
+
# Rapid Deep Neural Networks
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the MIT License
|
|
6
|
+
# ______________________________________________________________________________________
|
|
1
7
|
# ......................................................................................
|
|
2
|
-
# MIT License
|
|
3
8
|
|
|
4
9
|
# Copyright (c) 2018-2025 Pantelis I. Kaplanoglou
|
|
5
10
|
|
|
@@ -21,9 +26,7 @@
|
|
|
21
26
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
27
|
# SOFTWARE.
|
|
23
28
|
|
|
24
|
-
#
|
|
25
|
-
|
|
26
|
-
|
|
29
|
+
# .......................................................................................
|
|
27
30
|
import os
|
|
28
31
|
import shutil
|
|
29
32
|
import glob
|
|
@@ -32,14 +35,13 @@ if (sys.version_info.major == 3) and (sys.version_info.minor <= 7):
|
|
|
32
35
|
import pickle5 as pickle
|
|
33
36
|
else:
|
|
34
37
|
import pickle
|
|
35
|
-
|
|
38
|
+
from radnn.system.files import FileList
|
|
36
39
|
from radnn.system.files import JSONFile
|
|
37
40
|
from radnn.system.files import PickleFile
|
|
38
41
|
from radnn.system.files import TextFile
|
|
39
42
|
from radnn.system.files import CSVFile
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
if (is_opencv_installed()):
|
|
43
|
+
from radnn.ml_system import mlsys
|
|
44
|
+
if mlsys.is_opencv_installed:
|
|
43
45
|
from radnn.system.files.imgfile import PNGFile
|
|
44
46
|
|
|
45
47
|
|
|
@@ -51,6 +53,7 @@ class FileStore(object):
|
|
|
51
53
|
def __init__(self, base_folder, is_verbose=False, must_exist=False):
|
|
52
54
|
#.......................... | Instance Attributes | ............................
|
|
53
55
|
self.base_folder = base_folder
|
|
56
|
+
self.absolute_path = os.path.abspath(base_folder)
|
|
54
57
|
if not os.path.exists(self.base_folder):
|
|
55
58
|
if must_exist:
|
|
56
59
|
raise Exception(f"File store folder {self.base_folder} does not exist.")
|
|
@@ -62,7 +65,7 @@ class FileStore(object):
|
|
|
62
65
|
self.obj = PickleFile(None, parent_folder=self.base_folder)
|
|
63
66
|
self.text = TextFile(None, parent_folder=self.base_folder)
|
|
64
67
|
self.csv = CSVFile(None, parent_folder=self.base_folder)
|
|
65
|
-
if
|
|
68
|
+
if mlsys.is_opencv_installed:
|
|
66
69
|
self.img = PNGFile(None, parent_folder=base_folder)
|
|
67
70
|
self.donefs = None
|
|
68
71
|
#................................................................................
|
|
@@ -96,7 +99,7 @@ class FileStore(object):
|
|
|
96
99
|
return self.folder(subfolder_name)
|
|
97
100
|
# --------------------------------------------------------------------------------------------------------
|
|
98
101
|
def folder(self, folder_name):
|
|
99
|
-
sFolder = os.path.join(self.
|
|
102
|
+
sFolder = os.path.join(self.absolute_path, folder_name)
|
|
100
103
|
if not os.path.exists(sFolder):
|
|
101
104
|
os.makedirs(sFolder)
|
|
102
105
|
|
|
@@ -108,13 +111,19 @@ class FileStore(object):
|
|
|
108
111
|
file_name += "." + file_ext
|
|
109
112
|
else:
|
|
110
113
|
file_name += file_ext
|
|
111
|
-
return os.path.join(self.
|
|
114
|
+
return os.path.join(self.absolute_path, file_name)
|
|
112
115
|
# --------------------------------------------------------------------------------------------------------
|
|
113
116
|
def entries(self):
|
|
114
117
|
return os.listdir(self.base_folder)
|
|
115
118
|
# --------------------------------------------------------------------------------------------------------
|
|
116
|
-
def
|
|
117
|
-
|
|
119
|
+
def _ls(self, file_matching_pattern, is_removing_extension, sort_filename_key):
|
|
120
|
+
if ";" in file_matching_pattern:
|
|
121
|
+
sEntries = []
|
|
122
|
+
for sExt in file_matching_pattern.split(";"):
|
|
123
|
+
sEntries += glob.glob1(self.base_folder, sExt.strip())
|
|
124
|
+
else:
|
|
125
|
+
sEntries = glob.glob1(self.base_folder, file_matching_pattern)
|
|
126
|
+
|
|
118
127
|
if is_removing_extension:
|
|
119
128
|
oFileNamesOnly = []
|
|
120
129
|
for sEntry in sEntries:
|
|
@@ -122,15 +131,24 @@ class FileStore(object):
|
|
|
122
131
|
oFileNamesOnly.append(sFileNameOnly)
|
|
123
132
|
sEntries = sorted(oFileNamesOnly, key=sort_filename_key)
|
|
124
133
|
|
|
134
|
+
return sEntries
|
|
135
|
+
# --------------------------------------------------------------------------------------------------------
|
|
136
|
+
def list_files(self, file_matching_pattern, is_full_path=True, is_removing_extension=False, sort_filename_key=None):
|
|
137
|
+
sEntries = self._ls(file_matching_pattern, is_removing_extension, sort_filename_key)
|
|
125
138
|
if is_full_path:
|
|
126
139
|
oResult = [os.path.join(self.base_folder, x) for x in sEntries]
|
|
127
140
|
else:
|
|
128
141
|
oResult = [x for x in sEntries]
|
|
129
|
-
|
|
130
142
|
return oResult
|
|
131
143
|
# --------------------------------------------------------------------------------------------------------
|
|
132
|
-
|
|
133
|
-
|
|
144
|
+
def filelist(self, file_matching_pattern, is_removing_extension=False, sort_filename_key=None):
|
|
145
|
+
sEntries = self._ls(file_matching_pattern, is_removing_extension, sort_filename_key)
|
|
146
|
+
oFileList = FileList(self.base_folder)
|
|
147
|
+
for x in sEntries:
|
|
148
|
+
oFileList.append(x)
|
|
149
|
+
return oFileList
|
|
150
|
+
# --------------------------------------------------------------------------------------------------------
|
|
151
|
+
def list_folders(self, is_full_path=True):
|
|
134
152
|
sResult = []
|
|
135
153
|
for sFolder in os.listdir(self.base_folder):
|
|
136
154
|
sFullPath = os.path.join(self.base_folder, sFolder)
|
|
@@ -161,10 +179,10 @@ class FileStore(object):
|
|
|
161
179
|
pass
|
|
162
180
|
# ----------------------------------------------------------------------------------
|
|
163
181
|
def __repr__(self)->str:
|
|
164
|
-
return self.
|
|
182
|
+
return self.absolute_path
|
|
165
183
|
# --------------------------------------------------------------------------------------------------------
|
|
166
184
|
def __str__(self)->str:
|
|
167
|
-
return self.
|
|
185
|
+
return self.absolute_path
|
|
168
186
|
# --------------------------------------------------------------------------------------------------------
|
|
169
187
|
# ======================================================================================================================
|
|
170
188
|
|