pyerualjetwork 4.3.10b2__py3-none-any.whl → 4.3.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyerualjetwork/__init__.py +29 -7
- pyerualjetwork/activation_functions.py +0 -4
- pyerualjetwork/fitness_functions.py +14 -76
- pyerualjetwork/loss_functions.py +21 -0
- pyerualjetwork/loss_functions_cuda.py +21 -0
- pyerualjetwork/plan.py +106 -80
- pyerualjetwork/plan_cuda.py +122 -84
- pyerualjetwork/planeat.py +2 -2
- pyerualjetwork/planeat_cuda.py +2 -2
- pyerualjetwork/visualizations.py +3 -3
- pyerualjetwork/visualizations_cuda.py +4 -3
- {pyerualjetwork-4.3.10b2.dist-info → pyerualjetwork-4.3.11.dist-info}/METADATA +2 -2
- pyerualjetwork-4.3.11.dist-info/RECORD +25 -0
- pyerualjetwork/fitness_functions_cuda.py +0 -76
- pyerualjetwork-4.3.10b2.dist-info/RECORD +0 -24
- {pyerualjetwork-4.3.10b2.dist-info → pyerualjetwork-4.3.11.dist-info}/WHEEL +0 -0
- {pyerualjetwork-4.3.10b2.dist-info → pyerualjetwork-4.3.11.dist-info}/top_level.txt +0 -0
pyerualjetwork/__init__.py
CHANGED
@@ -1,11 +1,33 @@
|
|
1
|
-
__version__ = "4.3.
|
2
|
-
__update__ = "* Changes: https://github.com/HCB06/PyerualJetwork/blob/main/CHANGES
|
1
|
+
__version__ = "4.3.11"
|
2
|
+
__update__ = """* Changes: https://github.com/HCB06/PyerualJetwork/blob/main/CHANGES
|
3
|
+
* PyerualJetwork Homepage: https://github.com/HCB06/PyerualJetwork/tree/main
|
4
|
+
* PyerualJetwork document: https://github.com/HCB06/PyerualJetwork/blob/main/Welcome_to_PyerualJetwork/PYERUALJETWORK_USER_MANUEL_AND_LEGAL_INFORMATION(EN).pdf
|
5
|
+
* YouTube tutorials: https://www.youtube.com/@HasanCanBeydili"""
|
3
6
|
|
4
|
-
def print_version(
|
5
|
-
print(f"PyerualJetwork Version {
|
7
|
+
def print_version(version):
|
8
|
+
print(f"PyerualJetwork Version {version}\n")
|
6
9
|
|
7
|
-
def print_update_notes(
|
8
|
-
print(f"Notes:\n{
|
10
|
+
def print_update_notes(update):
|
11
|
+
print(f"Notes:\n{update}")
|
9
12
|
|
10
13
|
print_version(__version__)
|
11
|
-
print_update_notes(__update__)
|
14
|
+
print_update_notes(__update__)
|
15
|
+
|
16
|
+
required_modules = ["scipy", "tqdm", "pandas", "numpy", "colorama", "cupy", "psutil"]
|
17
|
+
|
18
|
+
missing_modules = []
|
19
|
+
for module in required_modules:
|
20
|
+
try:
|
21
|
+
__import__(module)
|
22
|
+
except ModuleNotFoundError:
|
23
|
+
missing_modules.append(module)
|
24
|
+
|
25
|
+
if missing_modules:
|
26
|
+
raise ImportError(
|
27
|
+
f"Missing modules detected: {', '.join(missing_modules)}\n"
|
28
|
+
"Please run the following command to install the missing packages:\n\n"
|
29
|
+
f" pip install {' '.join(missing_modules)}\n\n"
|
30
|
+
"For more information, visit the PyerualJetwork GitHub README.md file:\n"
|
31
|
+
"https://github.com/HCB06/PyerualJetwork/blob/main/README.md"
|
32
|
+
|
33
|
+
)
|
@@ -203,9 +203,6 @@ def square_quartic(x):
|
|
203
203
|
def cubic_quadratic(x):
|
204
204
|
return x**3 * (x**2)
|
205
205
|
|
206
|
-
def exp_cubic(x):
|
207
|
-
return np.exp(x**3)
|
208
|
-
|
209
206
|
def sine_square(x):
|
210
207
|
return np.sin(x)**2
|
211
208
|
|
@@ -267,7 +264,6 @@ def apply_activation(Input, activation_list):
|
|
267
264
|
'quartic': quartic,
|
268
265
|
'square_quartic': square_quartic,
|
269
266
|
'cubic_quadratic': cubic_quadratic,
|
270
|
-
'exp_cubic': exp_cubic,
|
271
267
|
'sine_square': sine_square,
|
272
268
|
'logarithmic': logarithmic,
|
273
269
|
'scaled_cubic': lambda x: scaled_cubic(x, 1.0),
|
@@ -1,78 +1,16 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
def pairwise_distances(X):
|
1
|
+
def wals(acc, loss, acc_impact, loss_impact):
|
4
2
|
"""
|
5
|
-
|
6
|
-
|
7
|
-
:
|
3
|
+
The WALS(weighted accuracy-loss score) function calculates a weighted sum of accuracy and loss based on their respective impacts.
|
4
|
+
|
5
|
+
:param acc: The `acc` parameter represents the accuracy of a model or system
|
6
|
+
:param loss: The `loss` parameter in the `wals` function represents the amount of loss incurred. It
|
7
|
+
is used in the calculation to determine the overall impact based on the accuracy and loss impacts
|
8
|
+
provided
|
9
|
+
:param acc_impact: The `acc_impact` parameter represents the impact of accuracy on the overall score
|
10
|
+
calculation in the `wals` function. It is a multiplier that determines how much the accuracy
|
11
|
+
contributes to the final result
|
12
|
+
:param loss_impact: The `loss_impact` parameter in the `wals` function represents the weight of loss value when calculating the overall impact. It is used to determine how
|
13
|
+
much the loss affects the final result compared to the accuracy impact
|
14
|
+
:return: the weighted sum of accuracy and loss based on their respective impacts.
|
8
15
|
"""
|
9
|
-
|
10
|
-
r = np.sum(X**2, axis=1, keepdims=True)
|
11
|
-
|
12
|
-
distances = np.maximum(r - 2 * np.dot(X, X.T) + r.T, 0.0)
|
13
|
-
|
14
|
-
return np.sqrt(distances)
|
15
|
-
|
16
|
-
def diversity_score(population):
|
17
|
-
"""
|
18
|
-
Calculates the diversity score of a population based on the
|
19
|
-
Euclidean distances between individuals in the population.
|
20
|
-
|
21
|
-
:param population: A NumPy array of shape (n_samples, n_features),
|
22
|
-
where each element is an individual weight matrix (flattened).
|
23
|
-
:return: A NumPy array of diversity scores for each weight matrix.
|
24
|
-
"""
|
25
|
-
if len(population) < 2:
|
26
|
-
return np.zeros(len(population))
|
27
|
-
|
28
|
-
population = np.nan_to_num(population, nan=0.0)
|
29
|
-
|
30
|
-
distances = pairwise_distances(population)
|
31
|
-
distances = np.maximum(distances, 1e-10)
|
32
|
-
|
33
|
-
n_samples = len(population)
|
34
|
-
|
35
|
-
mask = np.ones((n_samples, n_samples), dtype=bool)
|
36
|
-
np.fill_diagonal(mask, 0)
|
37
|
-
|
38
|
-
avg_distances = np.sum(distances * mask, axis=1) / (n_samples - 1)
|
39
|
-
|
40
|
-
if population.shape[1] == 0:
|
41
|
-
return np.zeros(len(population))
|
42
|
-
|
43
|
-
max_possible_distance = np.sqrt(population.shape[1])
|
44
|
-
max_possible_distance = float(max(max_possible_distance, 1e-10))
|
45
|
-
|
46
|
-
diversity_scores = avg_distances / max_possible_distance
|
47
|
-
|
48
|
-
return diversity_scores
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
def multi_head_fitness(y_true, y_pred, diversity_score, accuracy, alpha=2, beta=1.5, lambda_div=0.05):
|
53
|
-
"""
|
54
|
-
Computes a fitness score based on accuracy, margin loss, and diversity score.
|
55
|
-
|
56
|
-
:param y_true: Ground truth labels (NumPy array).
|
57
|
-
:param y_pred: Predicted labels (NumPy array).
|
58
|
-
:param diversity_score: Diversity score for a single weight matrix.
|
59
|
-
:param accuracy: Model accuracy.
|
60
|
-
:param alpha: Exponent for accuracy impact.
|
61
|
-
:param beta: Exponent for margin loss impact.
|
62
|
-
:param lambda_div: Weight for diversity score influence.
|
63
|
-
:return: Computed fitness value.
|
64
|
-
"""
|
65
|
-
incorrect = y_true != y_pred
|
66
|
-
|
67
|
-
if np.any(incorrect):
|
68
|
-
margin_loss = np.abs(y_true[incorrect] - y_pred[incorrect]).mean()
|
69
|
-
else:
|
70
|
-
margin_loss = 0.0
|
71
|
-
|
72
|
-
margin_loss = np.nan_to_num(margin_loss, nan=0.0)
|
73
|
-
accuracy = max(accuracy, 1e-10)
|
74
|
-
|
75
|
-
fitness = (accuracy ** alpha) * ((1 - margin_loss) ** beta) * (1 + lambda_div * diversity_score)
|
76
|
-
fitness = np.nan_to_num(fitness, nan=0.0, posinf=1e10, neginf=-1e10)
|
77
|
-
|
78
|
-
return fitness
|
16
|
+
return (acc * acc_impact) + (-loss * loss_impact)
|
@@ -0,0 +1,21 @@
|
|
1
|
+
|
2
|
+
import numpy as np
|
3
|
+
|
4
|
+
def categorical_crossentropy(y_true_batch, y_pred_batch):
|
5
|
+
epsilon = 1e-7
|
6
|
+
y_pred_batch = np.clip(y_pred_batch, epsilon, 1. - epsilon)
|
7
|
+
|
8
|
+
losses = -np.sum(y_true_batch * np.log(y_pred_batch), axis=1)
|
9
|
+
|
10
|
+
mean_loss = np.mean(losses)
|
11
|
+
return mean_loss
|
12
|
+
|
13
|
+
|
14
|
+
def binary_crossentropy(y_true_batch, y_pred_batch):
|
15
|
+
epsilon = 1e-7
|
16
|
+
y_pred_batch = np.clip(y_pred_batch, epsilon, 1. - epsilon)
|
17
|
+
|
18
|
+
losses = -np.mean(y_true_batch * np.log(y_pred_batch) + (1 - y_true_batch) * np.log(1 - y_pred_batch), axis=1)
|
19
|
+
|
20
|
+
mean_loss = np.mean(losses)
|
21
|
+
return mean_loss
|
@@ -0,0 +1,21 @@
|
|
1
|
+
|
2
|
+
import cupy as cp
|
3
|
+
|
4
|
+
def categorical_crossentropy(y_true_batch, y_pred_batch):
|
5
|
+
epsilon = 1e-7
|
6
|
+
y_pred_batch = cp.clip(y_pred_batch, epsilon, 1. - epsilon)
|
7
|
+
|
8
|
+
losses = -cp.sum(y_true_batch * cp.log(y_pred_batch), axis=1)
|
9
|
+
|
10
|
+
mean_loss = cp.mean(losses)
|
11
|
+
return mean_loss
|
12
|
+
|
13
|
+
|
14
|
+
def binary_crossentropy(y_true_batch, y_pred_batch):
|
15
|
+
epsilon = 1e-7
|
16
|
+
y_pred_batch = cp.clip(y_pred_batch, epsilon, 1. - epsilon)
|
17
|
+
|
18
|
+
losses = -cp.mean(y_true_batch * cp.log(y_pred_batch) + (1 - y_true_batch) * cp.log(1 - y_pred_batch), axis=1)
|
19
|
+
|
20
|
+
mean_loss = cp.mean(losses)
|
21
|
+
return mean_loss
|
pyerualjetwork/plan.py
CHANGED
@@ -6,7 +6,7 @@ MAIN MODULE FOR PLAN
|
|
6
6
|
Examples: https://github.com/HCB06/PyerualJetwork/tree/main/Welcome_to_PyerualJetwork/ExampleCodes
|
7
7
|
|
8
8
|
PLAN document: https://github.com/HCB06/PyerualJetwork/blob/main/Welcome_to_PLAN/PLAN.pdf
|
9
|
-
|
9
|
+
PyerualJetwork document: https://github.com/HCB06/PyerualJetwork/blob/main/Welcome_to_PyerualJetwork/PYERUALJETWORK_USER_MANUEL_AND_LEGAL_INFORMATION(EN).pdf
|
10
10
|
|
11
11
|
@author: Hasan Can Beydili
|
12
12
|
@YouTube: https://www.youtube.com/@HasanCanBeydili
|
@@ -21,8 +21,10 @@ import numpy as np
|
|
21
21
|
from .ui import loading_bars, initialize_loading_bar
|
22
22
|
from .data_operations import normalization, batcher
|
23
23
|
from .activation_functions import apply_activation, all_activations
|
24
|
-
from .model_operations import get_acc,
|
24
|
+
from .model_operations import get_acc, get_preds_softmax
|
25
25
|
from .memory_operations import optimize_labels
|
26
|
+
from .loss_functions import categorical_crossentropy, binary_crossentropy
|
27
|
+
from .fitness_functions import wals
|
26
28
|
from .visualizations import (
|
27
29
|
draw_neural_web,
|
28
30
|
display_visualizations_for_learner,
|
@@ -81,10 +83,10 @@ def fit(
|
|
81
83
|
return normalization(weight, dtype=dtype)
|
82
84
|
|
83
85
|
|
84
|
-
def learner(x_train, y_train, optimizer,
|
86
|
+
def learner(x_train, y_train, optimizer, fit_start=True, gen=None, batch_size=1, pop_size=None,
|
85
87
|
neural_web_history=False, show_current_activations=False, auto_normalization=False,
|
86
|
-
neurons_history=False, early_stop=False, show_history=False,
|
87
|
-
interval=33.33, target_acc=None,
|
88
|
+
neurons_history=False, early_stop=False, show_history=False, target_loss=None,
|
89
|
+
interval=33.33, target_acc=None, loss='categorical_crossentropy', acc_impact=0.9, loss_impact=0.1,
|
88
90
|
start_this_act=None, start_this_W=None, dtype=np.float32):
|
89
91
|
"""
|
90
92
|
Optimizes the activation functions for a neural network by leveraging train data to find
|
@@ -103,37 +105,26 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
103
105
|
optimizer (function): PLAN optimization technique with hyperparameters. (PLAN using NEAT(PLANEAT) for optimization.) Please use this: from pyerualjetwork import planeat (and) optimizer = lambda *args, **kwargs: planeat.evolve(*args, 'here give your neat hyperparameters for example: activation_add_prob=0.85', **kwargs) Example:
|
104
106
|
```python
|
105
107
|
optimizer = lambda *args, **kwargs: planeat.evolver(*args,
|
106
|
-
activation_add_prob=0.
|
108
|
+
activation_add_prob=0.05,
|
107
109
|
strategy='aggressive',
|
110
|
+
policy='more_selective',
|
108
111
|
**kwargs)
|
109
112
|
|
110
113
|
model = plan.learner(x_train,
|
111
114
|
y_train,
|
112
115
|
optimizer,
|
113
|
-
fitness,
|
114
116
|
fit_start=True,
|
115
|
-
strategy='accuracy',
|
116
117
|
show_history=True,
|
117
118
|
gen=15,
|
118
119
|
batch_size=0.05,
|
119
120
|
interval=16.67)
|
120
121
|
```
|
121
122
|
|
122
|
-
|
123
|
-
```python
|
124
|
-
fitness = lambda *args, **kwargs: fitness_functions.hybrid_accuracy_confidence(*args, alpha=2, beta=1.5, lambda_div=0.05, **kwargs)
|
123
|
+
loss (str, optional): For visualizing and monitoring. PLAN neural networks doesn't need any loss function in training. options: ('categorical_crossentropy' or 'binary_crossentropy') Default is 'categorical_crossentropy'.
|
125
124
|
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
fitness
|
130
|
-
fit_start=True,
|
131
|
-
strategy='accuracy',
|
132
|
-
show_history=True,
|
133
|
-
gen=15,
|
134
|
-
batch_size=0.05,
|
135
|
-
interval=16.67)
|
136
|
-
```
|
125
|
+
target_acc (float, optional): The target accuracy to stop training early when achieved. Default is None.
|
126
|
+
|
127
|
+
target_loss (float, optional): The target loss to stop training early when achieved. Default is None.
|
137
128
|
|
138
129
|
fit_start (bool, optional): If the fit_start parameter is set to True, the initial generation population undergoes a simple short training process using the PLAN algorithm. This allows for a very robust starting point, especially for large and complex datasets. However, for small or relatively simple datasets, it may result in unnecessary computational overhead. When fit_start is True, completing the first generation may take slightly longer (this increase in computational cost applies only to the first generation and does not affect subsequent generations). If fit_start is set to False, the initial population will be entirely random. Options: True or False. Default: True
|
139
130
|
|
@@ -170,7 +161,6 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
170
161
|
|
171
162
|
"""
|
172
163
|
|
173
|
-
from .fitness_functions import diversity_score
|
174
164
|
from .planeat import define_genomes
|
175
165
|
|
176
166
|
data = 'Train'
|
@@ -188,8 +178,7 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
188
178
|
|
189
179
|
if pop_size < activation_potentiation_len: raise ValueError(f"pop_size must be higher or equal to {activation_potentiation_len}")
|
190
180
|
|
191
|
-
if gen is None:
|
192
|
-
gen = activation_potentiation_len
|
181
|
+
if gen is None: gen = activation_potentiation_len
|
193
182
|
|
194
183
|
if target_acc is not None and (target_acc < 0 or target_acc > 1): raise ValueError('target_acc must be in range 0 and 1')
|
195
184
|
if fit_start is not True and fit_start is not False: raise ValueError('fit_start parameter only be True or False. Please read doc-string')
|
@@ -199,26 +188,27 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
199
188
|
|
200
189
|
# Initialize variables
|
201
190
|
best_acc = 0
|
202
|
-
|
191
|
+
best_loss = float('inf')
|
192
|
+
best_fitness = float('-inf')
|
203
193
|
best_acc_per_gen_list = []
|
204
194
|
postfix_dict = {}
|
205
|
-
|
195
|
+
loss_list = []
|
206
196
|
target_pop = []
|
207
197
|
|
208
|
-
progress = initialize_loading_bar(total=activation_potentiation_len, desc="", ncols=
|
198
|
+
progress = initialize_loading_bar(total=activation_potentiation_len, desc="", ncols=77, bar_format=bar_format_learner)
|
209
199
|
|
210
200
|
if fit_start is False or pop_size > activation_potentiation_len:
|
211
201
|
weight_pop, act_pop = define_genomes(input_shape=len(x_train[0]), output_shape=len(y_train[0]), population_size=pop_size, dtype=dtype)
|
212
|
-
div_score = diversity_score(weight_pop)
|
213
202
|
|
214
203
|
else:
|
215
|
-
weight_pop = [
|
204
|
+
weight_pop = [0] * pop_size
|
216
205
|
act_pop = [0] * pop_size
|
217
206
|
|
218
207
|
if start_this_act is not None and start_this_W is not None:
|
219
208
|
weight_pop[0] = start_this_W
|
220
209
|
act_pop[0] = start_this_act
|
221
210
|
|
211
|
+
# LEARNING STARTED
|
222
212
|
for i in range(gen):
|
223
213
|
postfix_dict["Gen"] = str(i+1) + '/' + str(gen)
|
224
214
|
progress.set_postfix(postfix_dict)
|
@@ -238,43 +228,43 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
238
228
|
act_pop[j] = activation_potentiation[j]
|
239
229
|
W = fit(x_train_batch, y_train_batch, activation_potentiation=act_pop[j], auto_normalization=auto_normalization, dtype=dtype)
|
240
230
|
weight_pop[j] = W
|
241
|
-
div_score = diversity_score(weight_pop)
|
242
231
|
|
243
232
|
model = evaluate(x_train_batch, y_train_batch, W=weight_pop[j], activation_potentiation=act_pop[j])
|
244
233
|
acc = model[get_acc()]
|
245
234
|
|
246
|
-
|
235
|
+
if loss == 'categorical_crossentropy':
|
236
|
+
train_loss = categorical_crossentropy(y_true_batch=y_train_batch,
|
237
|
+
y_pred_batch=model[get_preds_softmax()])
|
238
|
+
else:
|
239
|
+
train_loss = binary_crossentropy(y_true_batch=y_train_batch,
|
240
|
+
y_pred_batch=model[get_preds_softmax()])
|
247
241
|
|
248
|
-
|
242
|
+
fitness = wals(acc, train_loss, acc_impact, loss_impact)
|
243
|
+
target_pop.append(fitness)
|
249
244
|
|
250
|
-
if
|
245
|
+
if fitness >= best_fitness:
|
251
246
|
|
247
|
+
best_fitness = fitness
|
252
248
|
best_acc = acc
|
253
|
-
|
254
|
-
best_div_score = div_score[j]
|
249
|
+
best_loss = train_loss
|
255
250
|
best_weight = np.copy(weight_pop[j])
|
256
|
-
final_activations = act_pop[j].copy() if isinstance(act_pop[j], list) else act_pop[j]
|
257
|
-
|
258
251
|
best_model = model
|
259
252
|
|
253
|
+
final_activations = act_pop[j].copy() if isinstance(act_pop[j], list) else act_pop[j]
|
260
254
|
final_activations = [final_activations[0]] if len(set(final_activations)) == 1 else final_activations # removing if all same
|
261
255
|
|
262
256
|
if batch_size == 1:
|
263
257
|
postfix_dict[f"{data} Accuracy"] = np.round(best_acc, 4)
|
258
|
+
postfix_dict[f"{data} Loss"] = np.round(train_loss, 4)
|
264
259
|
progress.set_postfix(postfix_dict)
|
265
260
|
|
266
261
|
if show_current_activations:
|
267
262
|
print(f", Current Activations={final_activations}", end='')
|
268
263
|
|
269
|
-
if batch_size == 1:
|
270
|
-
postfix_dict["Fitness"] = np.round(fit_score, 4)
|
271
|
-
progress.set_postfix(postfix_dict)
|
272
|
-
best_fit = fit_score
|
273
|
-
|
274
264
|
# Update visualizations during training
|
275
265
|
if show_history:
|
276
266
|
gen_list = range(1, len(best_acc_per_gen_list) + 2)
|
277
|
-
update_history_plots_for_learner(viz_objects, gen_list,
|
267
|
+
update_history_plots_for_learner(viz_objects, gen_list, loss_list + [train_loss],
|
278
268
|
best_acc_per_gen_list + [best_acc], x_train, final_activations)
|
279
269
|
|
280
270
|
if neurons_history:
|
@@ -283,7 +273,7 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
283
273
|
viz_objects['neurons']['row'], viz_objects['neurons']['col'],
|
284
274
|
y_train[0], viz_objects['neurons']['artists'],
|
285
275
|
data=data, fig1=viz_objects['neurons']['fig'],
|
286
|
-
acc=best_acc, loss=
|
276
|
+
acc=best_acc, loss=train_loss)
|
287
277
|
)
|
288
278
|
|
289
279
|
if neural_web_history:
|
@@ -297,45 +287,68 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
297
287
|
progress.close()
|
298
288
|
train_model = evaluate(x_train, y_train, W=best_weight,
|
299
289
|
activation_potentiation=final_activations)
|
300
|
-
|
301
|
-
|
290
|
+
if loss == 'categorical_crossentropy':
|
291
|
+
train_loss = categorical_crossentropy(y_true_batch=y_train,
|
292
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
293
|
+
else:
|
294
|
+
train_loss = binary_crossentropy(y_true_batch=y_train,
|
295
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
302
296
|
|
303
297
|
print('\nActivations: ', final_activations)
|
304
298
|
print(f'Train Accuracy:', train_model[get_acc()])
|
305
|
-
print(f'
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
299
|
+
print(f'Train Loss: ', train_loss, '\n')
|
300
|
+
|
301
|
+
display_visualizations_for_learner(viz_objects, best_weight, data, best_acc,
|
302
|
+
best_loss, y_train, interval)
|
303
|
+
return best_weight, best_model[get_preds_softmax()], best_acc, final_activations
|
304
|
+
|
305
|
+
# Check target loss
|
306
|
+
if target_loss is not None and best_loss <= target_loss:
|
307
|
+
progress.close()
|
308
|
+
train_model = evaluate(x_train, y_train, W=best_weight,
|
309
|
+
activation_potentiation=final_activations)
|
310
|
+
|
311
|
+
if loss == 'categorical_crossentropy':
|
312
|
+
train_loss = categorical_crossentropy(y_true_batch=y_train,
|
313
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
314
|
+
else:
|
315
|
+
train_loss = binary_crossentropy(y_true_batch=y_train,
|
316
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
317
|
+
|
318
|
+
print('\nActivations: ', final_activations)
|
319
|
+
print(f'Train Accuracy :', train_model[get_acc()])
|
320
|
+
print(f'Train Loss : ', train_loss, '\n')
|
321
|
+
|
310
322
|
# Display final visualizations
|
311
323
|
display_visualizations_for_learner(viz_objects, best_weight, data, best_acc,
|
312
|
-
|
324
|
+
train_loss, y_train, interval)
|
313
325
|
return best_weight, best_model[get_preds_softmax()], best_acc, final_activations
|
314
326
|
|
327
|
+
|
315
328
|
progress.update(1)
|
316
329
|
|
317
330
|
if batch_size != 1:
|
318
331
|
train_model = evaluate(x_train, y_train, best_weight, final_activations)
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
332
|
+
|
333
|
+
if loss == 'categorical_crossentropy':
|
334
|
+
train_loss = categorical_crossentropy(y_true_batch=y_train,
|
335
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
336
|
+
else:
|
337
|
+
train_loss = binary_crossentropy(y_true_batch=y_train,
|
338
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
325
339
|
|
326
340
|
postfix_dict[f"{data} Accuracy"] = np.round(train_model[get_acc()], 4)
|
327
|
-
postfix_dict["
|
341
|
+
postfix_dict[f"{data} Loss"] = np.round(train_loss, 4)
|
328
342
|
progress.set_postfix(postfix_dict)
|
329
343
|
|
330
344
|
best_acc_per_gen_list.append(train_model[get_acc()])
|
331
|
-
|
345
|
+
loss_list.append(train_loss)
|
332
346
|
|
333
347
|
else:
|
334
348
|
best_acc_per_gen_list.append(best_acc)
|
335
|
-
|
349
|
+
loss_list.append(best_loss)
|
336
350
|
|
337
351
|
weight_pop, act_pop = optimizer(np.array(weight_pop, copy=False, dtype=dtype), act_pop, i, np.array(target_pop, dtype=dtype, copy=False), bar_status=False)
|
338
|
-
div_score = diversity_score(weight_pop)
|
339
352
|
target_pop = []
|
340
353
|
|
341
354
|
# Early stopping check
|
@@ -345,30 +358,40 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
345
358
|
train_model = evaluate(x_train, y_train, W=best_weight,
|
346
359
|
activation_potentiation=final_activations)
|
347
360
|
|
348
|
-
|
361
|
+
if loss == 'categorical_crossentropy':
|
362
|
+
train_loss = categorical_crossentropy(y_true_batch=y_train,
|
363
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
364
|
+
else:
|
365
|
+
train_loss = binary_crossentropy(y_true_batch=y_train,
|
366
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
349
367
|
|
350
|
-
|
351
|
-
|
352
|
-
|
368
|
+
print('\nActivations: ', final_activations)
|
369
|
+
print(f'Train Accuracy:', train_model[get_acc()])
|
370
|
+
print(f'Train Loss: ', train_loss, '\n')
|
353
371
|
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
372
|
+
# Display final visualizations
|
373
|
+
display_visualizations_for_learner(viz_objects, best_weight, data, best_acc,
|
374
|
+
train_loss, y_train, interval)
|
375
|
+
return best_weight, best_model[get_preds_softmax()], best_acc, final_activations
|
358
376
|
|
359
377
|
# Final evaluation
|
360
378
|
progress.close()
|
361
|
-
train_model = evaluate(x_train, y_train, W=best_weight,
|
379
|
+
train_model = evaluate(x_train, y_train, W=best_weight,
|
362
380
|
activation_potentiation=final_activations)
|
363
381
|
|
364
|
-
|
365
|
-
|
382
|
+
if loss == 'categorical_crossentropy':
|
383
|
+
train_loss = categorical_crossentropy(y_true_batch=y_train,
|
384
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
385
|
+
else:
|
386
|
+
train_loss = binary_crossentropy(y_true_batch=y_train,
|
387
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
388
|
+
|
366
389
|
print('\nActivations: ', final_activations)
|
367
390
|
print(f'Train Accuracy:', train_model[get_acc()])
|
368
|
-
print(f'
|
391
|
+
print(f'Train Loss: ', train_loss, '\n')
|
369
392
|
|
370
393
|
# Display final visualizations
|
371
|
-
display_visualizations_for_learner(viz_objects, best_weight, data, best_acc,
|
394
|
+
display_visualizations_for_learner(viz_objects, best_weight, data, best_acc, train_loss, y_train, interval)
|
372
395
|
return best_weight, best_model[get_preds_softmax()], best_acc, final_activations
|
373
396
|
|
374
397
|
|
@@ -395,9 +418,12 @@ def evaluate(
|
|
395
418
|
"""
|
396
419
|
|
397
420
|
x_test = apply_activation(x_test, activation_potentiation)
|
421
|
+
|
398
422
|
result = x_test @ W.T
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
423
|
+
|
424
|
+
max_vals = np.max(result, axis=1, keepdims=True)
|
425
|
+
|
426
|
+
softmax_preds = np.exp(result - max_vals) / np.sum(np.exp(result - max_vals), axis=1, keepdims=True)
|
427
|
+
accuracy = (np.argmax(softmax_preds, axis=1) == np.argmax(y_test, axis=1)).mean()
|
428
|
+
|
403
429
|
return W, result, accuracy, None, None, softmax_preds
|
pyerualjetwork/plan_cuda.py
CHANGED
@@ -6,7 +6,7 @@ MAIN MODULE FOR PLAN_CUDA
|
|
6
6
|
Examples: https://github.com/HCB06/PyerualJetwork/tree/main/Welcome_to_PyerualJetwork/ExampleCodes
|
7
7
|
|
8
8
|
PLAN document: https://github.com/HCB06/PyerualJetwork/blob/main/Welcome_to_PLAN/PLAN.pdf
|
9
|
-
|
9
|
+
PyerualJetwork document: https://github.com/HCB06/PyerualJetwork/blob/main/Welcome_to_PyerualJetwork/PYERUALJETWORK_USER_MANUEL_AND_LEGAL_INFORMATION(EN).pdf
|
10
10
|
|
11
11
|
@author: Hasan Can Beydili
|
12
12
|
@YouTube: https://www.youtube.com/@HasanCanBeydili
|
@@ -21,8 +21,10 @@ import cupy as cp
|
|
21
21
|
from .ui import loading_bars, initialize_loading_bar
|
22
22
|
from .data_operations_cuda import normalization
|
23
23
|
from .activation_functions_cuda import apply_activation, all_activations
|
24
|
-
from .model_operations_cuda import get_acc,
|
24
|
+
from .model_operations_cuda import get_acc, get_preds_softmax
|
25
25
|
from .memory_operations import transfer_to_gpu, transfer_to_cpu, optimize_labels
|
26
|
+
from .loss_functions_cuda import categorical_crossentropy, binary_crossentropy
|
27
|
+
from .fitness_functions import wals
|
26
28
|
from .visualizations_cuda import (
|
27
29
|
draw_neural_web,
|
28
30
|
display_visualizations_for_learner,
|
@@ -80,10 +82,10 @@ def fit(
|
|
80
82
|
return normalization(weight, dtype=dtype)
|
81
83
|
|
82
84
|
|
83
|
-
def learner(x_train, y_train, optimizer,
|
84
|
-
neural_web_history=False, show_current_activations=False, auto_normalization=False,
|
85
|
-
neurons_history=False, early_stop=False, show_history=False,
|
86
|
-
interval=33.33,
|
85
|
+
def learner(x_train, y_train, optimizer, fit_start=True, gen=None, batch_size=1, pop_size=None,
|
86
|
+
neural_web_history=False, show_current_activations=False, auto_normalization=False, target_acc=None,
|
87
|
+
neurons_history=False, early_stop=False, show_history=False, loss='categorical_crossentropy',
|
88
|
+
interval=33.33, target_loss=None, loss_impact=0.1, acc_impact=0.9,
|
87
89
|
start_this_act=None, start_this_W=None, dtype=cp.float32, memory='gpu'):
|
88
90
|
"""
|
89
91
|
Optimizes the activation functions for a neural network by leveraging train data to find
|
@@ -101,37 +103,26 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
101
103
|
|
102
104
|
optimizer (function): PLAN optimization technique with hyperparameters. (PLAN using NEAT(PLANEAT) for optimization.) Please use this: from pyerualjetwork import planeat_cuda (and) optimizer = lambda *args, **kwargs: planeat_cuda.evolve(*args, 'here give your neat hyperparameters for example: activation_add_prob=0.85', **kwargs) Example:
|
103
105
|
```python
|
104
|
-
|
105
|
-
activation_add_prob=0.
|
106
|
+
optimizer = lambda *args, **kwargs: planeat_cuda.evolver(*args,
|
107
|
+
activation_add_prob=0.05,
|
106
108
|
strategy='aggressive',
|
109
|
+
policy='more_selective',
|
107
110
|
**kwargs)
|
108
111
|
|
109
112
|
model = plan_cuda.learner(x_train,
|
110
113
|
y_train,
|
111
|
-
optimizer
|
114
|
+
optimizer,
|
112
115
|
fit_start=True,
|
113
|
-
strategy='accuracy',
|
114
116
|
show_history=True,
|
115
117
|
gen=15,
|
116
118
|
batch_size=0.05,
|
117
119
|
interval=16.67)
|
118
120
|
```
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
model = plan.learner(x_train,
|
125
|
-
y_train,
|
126
|
-
optimizer,
|
127
|
-
fitness
|
128
|
-
fit_start=True,
|
129
|
-
strategy='accuracy',
|
130
|
-
show_history=True,
|
131
|
-
gen=15,
|
132
|
-
batch_size=0.05,
|
133
|
-
interval=16.67)
|
134
|
-
```
|
121
|
+
loss (str, optional): For visualizing and monitoring. PLAN neural networks doesn't need any loss function in training. options: ('categorical_crossentropy' or 'binary_crossentropy') Default is 'categorical_crossentropy'.
|
122
|
+
|
123
|
+
target_acc (float, optional): The target accuracy to stop training early when achieved. Default is None.
|
124
|
+
|
125
|
+
target_loss (float, optional): The target loss to stop training early when achieved. Default is None.
|
135
126
|
|
136
127
|
fit_start (bool, optional): If the fit_start parameter is set to True, the initial generation population undergoes a simple short training process using the PLAN algorithm. This allows for a very robust starting point, especially for large and complex datasets. However, for small or relatively simple datasets, it may result in unnecessary computational overhead. When fit_start is True, completing the first generation may take slightly longer (this increase in computational cost applies only to the first generation and does not affect subsequent generations). If fit_start is set to False, the initial population will be entirely random. Options: True or False. Default: True
|
137
128
|
|
@@ -139,6 +130,10 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
139
130
|
|
140
131
|
batch_size (float, optional): Batch size is used in the prediction process to receive train feedback by dividing the train data into chunks and selecting activations based on randomly chosen partitions. This process reduces computational cost and time while still covering the entire test set due to random selection, so it doesn't significantly impact accuracy. For example, a batch size of 0.08 means each train batch represents 8% of the train set. Default is 1. (%100 of train)
|
141
132
|
|
133
|
+
loss_impact (float, optional): Impact of loss during evolve. [0-1] Default: 0.1
|
134
|
+
|
135
|
+
acc_impact (float, optional): Impact of accuracy during evolve. [0-1] Default: 0.9
|
136
|
+
|
142
137
|
pop_size (int, optional): Population size of each generation. Default: count of activation functions
|
143
138
|
|
144
139
|
early_stop (bool, optional): If True, implements early stopping during training.(If train accuracy not improves in two gen stops learning.) Default is False.
|
@@ -170,7 +165,6 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
170
165
|
|
171
166
|
"""
|
172
167
|
|
173
|
-
from .fitness_functions_cuda import diversity_score
|
174
168
|
from .planeat_cuda import define_genomes
|
175
169
|
|
176
170
|
data = 'Train'
|
@@ -184,6 +178,8 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
184
178
|
|
185
179
|
if pop_size < activation_potentiation_len: raise ValueError(f"pop_size must be higher or equal to {activation_potentiation_len}")
|
186
180
|
|
181
|
+
if gen is None: gen = activation_potentiation_len
|
182
|
+
|
187
183
|
if memory == 'gpu':
|
188
184
|
x_train = transfer_to_gpu(x_train, dtype=dtype)
|
189
185
|
y_train = transfer_to_gpu(y_train, dtype=y_train.dtype)
|
@@ -207,26 +203,27 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
207
203
|
|
208
204
|
# Initialize variables
|
209
205
|
best_acc = 0
|
210
|
-
|
206
|
+
best_loss = float('inf')
|
207
|
+
best_fitness = float('-inf')
|
211
208
|
best_acc_per_gen_list = []
|
212
209
|
postfix_dict = {}
|
213
|
-
|
210
|
+
loss_list = []
|
214
211
|
target_pop = []
|
215
212
|
|
216
|
-
progress = initialize_loading_bar(total=activation_potentiation_len, desc="", ncols=
|
213
|
+
progress = initialize_loading_bar(total=activation_potentiation_len, desc="", ncols=79, bar_format=bar_format_learner)
|
217
214
|
|
218
215
|
if fit_start is False or pop_size > activation_potentiation_len:
|
219
216
|
weight_pop, act_pop = define_genomes(input_shape=len(x_train[0]), output_shape=len(y_train[0]), population_size=pop_size, dtype=dtype)
|
220
|
-
|
221
|
-
|
217
|
+
|
222
218
|
else:
|
223
|
-
weight_pop = [
|
219
|
+
weight_pop = [0] * pop_size
|
224
220
|
act_pop = [0] * pop_size
|
225
221
|
|
226
222
|
if start_this_act is not None and start_this_W is not None:
|
227
223
|
weight_pop[0] = start_this_W
|
228
224
|
act_pop[0] = start_this_act
|
229
225
|
|
226
|
+
# LEARNING STARTED
|
230
227
|
for i in range(gen):
|
231
228
|
postfix_dict["Gen"] = str(i+1) + '/' + str(gen)
|
232
229
|
progress.set_postfix(postfix_dict)
|
@@ -240,7 +237,7 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
240
237
|
x_train_batch, y_train_batch = batcher(x_train, y_train, batch_size=batch_size)
|
241
238
|
|
242
239
|
x_train_batch = cp.array(x_train_batch, dtype=dtype, copy=False)
|
243
|
-
y_train_batch = cp.array(y_train_batch, dtype=dtype
|
240
|
+
y_train_batch = cp.array(y_train_batch, dtype=y_train.dtype)
|
244
241
|
|
245
242
|
if fit_start is True and i == 0 and j < activation_potentiation_len:
|
246
243
|
if start_this_act is not None and j == 0:
|
@@ -249,43 +246,44 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
249
246
|
act_pop[j] = activation_potentiation[j]
|
250
247
|
W = fit(x_train_batch, y_train_batch, activation_potentiation=act_pop[j], auto_normalization=auto_normalization, dtype=dtype)
|
251
248
|
weight_pop[j] = W
|
252
|
-
|
253
|
-
|
249
|
+
|
254
250
|
model = evaluate(x_train_batch, y_train_batch, W=weight_pop[j], activation_potentiation=act_pop[j])
|
255
251
|
acc = model[get_acc()]
|
256
252
|
|
257
|
-
|
258
|
-
|
259
|
-
|
253
|
+
if loss == 'categorical_crossentropy':
|
254
|
+
train_loss = categorical_crossentropy(y_true_batch=y_train_batch,
|
255
|
+
y_pred_batch=model[get_preds_softmax()])
|
256
|
+
else:
|
257
|
+
train_loss = binary_crossentropy(y_true_batch=y_train_batch,
|
258
|
+
y_pred_batch=model[get_preds_softmax()])
|
260
259
|
|
261
|
-
if fit_score >= best_fit:
|
262
260
|
|
261
|
+
fitness = wals(acc, train_loss, acc_impact, loss_impact)
|
262
|
+
target_pop.append(fitness)
|
263
|
+
|
264
|
+
if fitness >= best_fitness:
|
265
|
+
|
266
|
+
best_fitness = fitness
|
263
267
|
best_acc = acc
|
264
|
-
|
265
|
-
best_div_score = div_score[j]
|
268
|
+
best_loss = train_loss
|
266
269
|
best_weight = cp.copy(weight_pop[j])
|
267
|
-
final_activations = act_pop[j].copy() if isinstance(act_pop[j], list) else act_pop[j]
|
268
|
-
|
269
270
|
best_model = model
|
270
271
|
|
272
|
+
final_activations = act_pop[j].copy() if isinstance(act_pop[j], list) else act_pop[j]
|
271
273
|
final_activations = [final_activations[0]] if len(set(final_activations)) == 1 else final_activations # removing if all same
|
272
274
|
|
273
275
|
if batch_size == 1:
|
274
276
|
postfix_dict[f"{data} Accuracy"] = cp.round(best_acc, 4)
|
277
|
+
postfix_dict[f"{data} Loss"] = cp.round(train_loss, 4)
|
275
278
|
progress.set_postfix(postfix_dict)
|
276
279
|
|
277
280
|
if show_current_activations:
|
278
281
|
print(f", Current Activations={final_activations}", end='')
|
279
282
|
|
280
|
-
if batch_size == 1:
|
281
|
-
postfix_dict["Fitness"] = cp.round(fit_score, 4)
|
282
|
-
progress.set_postfix(postfix_dict)
|
283
|
-
best_fit = fit_score
|
284
|
-
|
285
283
|
# Update visualizations during training
|
286
284
|
if show_history:
|
287
285
|
gen_list = range(1, len(best_acc_per_gen_list) + 2)
|
288
|
-
update_history_plots_for_learner(viz_objects, gen_list,
|
286
|
+
update_history_plots_for_learner(viz_objects, gen_list, loss_list + [train_loss],
|
289
287
|
best_acc_per_gen_list + [best_acc], x_train, final_activations)
|
290
288
|
|
291
289
|
if neurons_history:
|
@@ -294,7 +292,7 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
294
292
|
viz_objects['neurons']['row'], viz_objects['neurons']['col'],
|
295
293
|
y_train[0], viz_objects['neurons']['artists'],
|
296
294
|
data=data, fig1=viz_objects['neurons']['fig'],
|
297
|
-
acc=best_acc, loss=
|
295
|
+
acc=best_acc, loss=train_loss)
|
298
296
|
)
|
299
297
|
|
300
298
|
if neural_web_history:
|
@@ -308,41 +306,68 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
308
306
|
progress.close()
|
309
307
|
train_model = evaluate(x_train, y_train, W=best_weight,
|
310
308
|
activation_potentiation=final_activations)
|
311
|
-
|
312
|
-
|
309
|
+
if loss == 'categorical_crossentropy':
|
310
|
+
train_loss = categorical_crossentropy(y_true_batch=y_train,
|
311
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
312
|
+
else:
|
313
|
+
train_loss = binary_crossentropy(y_true_batch=y_train,
|
314
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
313
315
|
|
314
316
|
print('\nActivations: ', final_activations)
|
315
317
|
print(f'Train Accuracy:', train_model[get_acc()])
|
316
|
-
print(f'
|
318
|
+
print(f'Train Loss: ', train_loss, '\n')
|
317
319
|
|
318
320
|
display_visualizations_for_learner(viz_objects, best_weight, data, best_acc,
|
319
|
-
|
321
|
+
best_loss, y_train, interval)
|
320
322
|
return best_weight, best_model[get_preds_softmax()], best_acc, final_activations
|
321
323
|
|
324
|
+
# Check target loss
|
325
|
+
if target_loss is not None and best_loss <= target_loss:
|
326
|
+
progress.close()
|
327
|
+
train_model = evaluate(x_train, y_train, W=best_weight,
|
328
|
+
activation_potentiation=final_activations)
|
329
|
+
|
330
|
+
if loss == 'categorical_crossentropy':
|
331
|
+
train_loss = categorical_crossentropy(y_true_batch=y_train,
|
332
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
333
|
+
else:
|
334
|
+
train_loss = binary_crossentropy(y_true_batch=y_train,
|
335
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
336
|
+
|
337
|
+
print('\nActivations: ', final_activations)
|
338
|
+
print(f'Train Accuracy :', train_model[get_acc()])
|
339
|
+
print(f'Train Loss : ', train_loss, '\n')
|
340
|
+
|
341
|
+
# Display final visualizations
|
342
|
+
display_visualizations_for_learner(viz_objects, best_weight, data, best_acc,
|
343
|
+
train_loss, y_train, interval)
|
344
|
+
return best_weight, best_model[get_preds_softmax()], best_acc, final_activations
|
345
|
+
|
346
|
+
|
322
347
|
progress.update(1)
|
323
348
|
|
324
349
|
if batch_size != 1:
|
325
350
|
train_model = evaluate(x_train, y_train, best_weight, final_activations)
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
351
|
+
|
352
|
+
if loss == 'categorical_crossentropy':
|
353
|
+
train_loss = categorical_crossentropy(y_true_batch=y_train,
|
354
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
355
|
+
else:
|
356
|
+
train_loss = binary_crossentropy(y_true_batch=y_train,
|
357
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
358
|
+
|
333
359
|
postfix_dict[f"{data} Accuracy"] = cp.round(train_model[get_acc()], 4)
|
334
|
-
postfix_dict["
|
360
|
+
postfix_dict[f"{data} Loss"] = cp.round(train_loss, 4)
|
335
361
|
progress.set_postfix(postfix_dict)
|
336
362
|
|
337
363
|
best_acc_per_gen_list.append(train_model[get_acc()])
|
338
|
-
|
364
|
+
loss_list.append(train_loss)
|
339
365
|
|
340
366
|
else:
|
341
367
|
best_acc_per_gen_list.append(best_acc)
|
342
|
-
|
368
|
+
loss_list.append(best_loss)
|
343
369
|
|
344
370
|
weight_pop, act_pop = optimizer(cp.array(weight_pop, copy=False, dtype=dtype), act_pop, i, cp.array(target_pop, dtype=dtype, copy=False), bar_status=False)
|
345
|
-
div_score = diversity_score(weight_pop)
|
346
371
|
target_pop = []
|
347
372
|
|
348
373
|
# Early stopping check
|
@@ -351,31 +376,41 @@ def learner(x_train, y_train, optimizer, fitness, fit_start=True, gen=None, batc
|
|
351
376
|
progress.close()
|
352
377
|
train_model = evaluate(x_train, y_train, W=best_weight,
|
353
378
|
activation_potentiation=final_activations)
|
379
|
+
|
380
|
+
if loss == 'categorical_crossentropy':
|
381
|
+
train_loss = categorical_crossentropy(y_true_batch=y_train,
|
382
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
383
|
+
else:
|
384
|
+
train_loss = binary_crossentropy(y_true_batch=y_train,
|
385
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
354
386
|
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
print(f'Train Accuracy:', train_model[get_acc()])
|
359
|
-
print(f'Fitness Value: ', fit_score, '\n')
|
387
|
+
print('\nActivations: ', final_activations)
|
388
|
+
print(f'Train Accuracy:', train_model[get_acc()])
|
389
|
+
print(f'Train Loss: ', train_loss, '\n')
|
360
390
|
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
391
|
+
# Display final visualizations
|
392
|
+
display_visualizations_for_learner(viz_objects, best_weight, data, best_acc,
|
393
|
+
train_loss, y_train, interval)
|
394
|
+
return best_weight, best_model[get_preds_softmax()], best_acc, final_activations
|
365
395
|
|
366
396
|
# Final evaluation
|
367
397
|
progress.close()
|
368
398
|
train_model = evaluate(x_train, y_train, W=best_weight,
|
369
399
|
activation_potentiation=final_activations)
|
370
400
|
|
371
|
-
|
372
|
-
|
401
|
+
if loss == 'categorical_crossentropy':
|
402
|
+
train_loss = categorical_crossentropy(y_true_batch=y_train,
|
403
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
404
|
+
else:
|
405
|
+
train_loss = binary_crossentropy(y_true_batch=y_train,
|
406
|
+
y_pred_batch=train_model[get_preds_softmax()])
|
407
|
+
|
373
408
|
print('\nActivations: ', final_activations)
|
374
409
|
print(f'Train Accuracy:', train_model[get_acc()])
|
375
|
-
print(f'
|
410
|
+
print(f'Train Loss: ', train_loss, '\n')
|
376
411
|
|
377
412
|
# Display final visualizations
|
378
|
-
display_visualizations_for_learner(viz_objects, best_weight, data, best_acc,
|
413
|
+
display_visualizations_for_learner(viz_objects, best_weight, data, best_acc, train_loss, y_train, interval)
|
379
414
|
return best_weight, best_model[get_preds_softmax()], best_acc, final_activations
|
380
415
|
|
381
416
|
def evaluate(
|
@@ -389,9 +424,9 @@ def evaluate(
|
|
389
424
|
|
390
425
|
Args:
|
391
426
|
x_test (cp.ndarray): Test data.
|
392
|
-
|
427
|
+
|
393
428
|
y_test (cp.ndarray): Test labels (one-hot encoded).
|
394
|
-
|
429
|
+
|
395
430
|
W (cp.ndarray): Neural net weight matrix.
|
396
431
|
|
397
432
|
activation_potentiation (list): Activation list. Default = ['linear'].
|
@@ -401,9 +436,12 @@ def evaluate(
|
|
401
436
|
"""
|
402
437
|
|
403
438
|
x_test = apply_activation(x_test, activation_potentiation)
|
439
|
+
|
404
440
|
result = x_test @ W.T
|
405
|
-
|
406
|
-
|
407
|
-
|
441
|
+
|
442
|
+
max_vals = cp.max(result, axis=1, keepdims=True)
|
443
|
+
|
444
|
+
softmax_preds = cp.exp(result - max_vals) / cp.sum(cp.exp(result - max_vals), axis=1, keepdims=True)
|
445
|
+
accuracy = (cp.argmax(softmax_preds, axis=1) == cp.argmax(y_test, axis=1)).mean()
|
408
446
|
|
409
447
|
return W, result, accuracy, None, None, softmax_preds
|
pyerualjetwork/planeat.py
CHANGED
@@ -3,12 +3,12 @@ MAIN MODULE FOR PLANEAT
|
|
3
3
|
|
4
4
|
Examples: https://github.com/HCB06/PyerualJetwork/tree/main/Welcome_to_PyerualJetwork/ExampleCodes
|
5
5
|
|
6
|
-
|
6
|
+
PyerualJetwork document: https://github.com/HCB06/Anaplan/blob/main/Welcome_to_Anaplan/ANAPLAN_USER_MANUEL_AND_LEGAL_INFORMATION(EN).pdf
|
7
7
|
|
8
8
|
@author: Hasan Can Beydili
|
9
9
|
@YouTube: https://www.youtube.com/@HasanCanBeydili
|
10
10
|
@Linkedin: https://www.linkedin.com/in/hasan-can-beydili-77a1b9270/
|
11
|
-
@Instagram: https://www.instagram.com/
|
11
|
+
@Instagram: https://www.instagram.com/canbeydilj
|
12
12
|
@contact: tchasancan@gmail.com
|
13
13
|
"""
|
14
14
|
|
pyerualjetwork/planeat_cuda.py
CHANGED
@@ -3,12 +3,12 @@ MAIN MODULE FOR PLANEAT
|
|
3
3
|
|
4
4
|
Examples: https://github.com/HCB06/PyerualJetwork/tree/main/Welcome_to_PyerualJetwork/ExampleCodes
|
5
5
|
|
6
|
-
|
6
|
+
PyerualJetwork document: https://github.com/HCB06/Anaplan/blob/main/Welcome_to_Anaplan/ANAPLAN_USER_MANUEL_AND_LEGAL_INFORMATION(EN).pdf
|
7
7
|
|
8
8
|
@author: Hasan Can Beydili
|
9
9
|
@YouTube: https://www.youtube.com/@HasanCanBeydili
|
10
10
|
@Linkedin: https://www.linkedin.com/in/hasan-can-beydili-77a1b9270/
|
11
|
-
@Instagram: https://www.instagram.com/
|
11
|
+
@Instagram: https://www.instagram.com/canbeydilj
|
12
12
|
@contact: tchasancan@gmail.com
|
13
13
|
"""
|
14
14
|
|
pyerualjetwork/visualizations.py
CHANGED
@@ -754,9 +754,9 @@ def update_history_plots_for_learner(viz_objects, depth_list, loss_list, best_ac
|
|
754
754
|
|
755
755
|
hist = viz_objects['history']
|
756
756
|
|
757
|
-
#
|
758
|
-
art1 = hist['ax'][0].plot(depth_list, loss_list, color='
|
759
|
-
hist['ax'][0].set_title('
|
757
|
+
# Loss plot
|
758
|
+
art1 = hist['ax'][0].plot(depth_list, loss_list, color='r', markersize=6, linewidth=2)
|
759
|
+
hist['ax'][0].set_title('Train Loss Over Gen')
|
760
760
|
hist['artist1'].append(art1)
|
761
761
|
|
762
762
|
# Accuracy plot
|
@@ -748,12 +748,13 @@ def update_history_plots_for_learner(viz_objects, depth_list, loss_list, best_ac
|
|
748
748
|
for i in range(len(loss_list)):
|
749
749
|
loss_list[i] = loss_list[i].get()
|
750
750
|
|
751
|
-
#
|
752
|
-
art1 = hist['ax'][0].plot(depth_list, loss_list, color='
|
753
|
-
hist['ax'][0].set_title('
|
751
|
+
# Loss plot
|
752
|
+
art1 = hist['ax'][0].plot(depth_list, loss_list, color='r', markersize=6, linewidth=2)
|
753
|
+
hist['ax'][0].set_title('Train Loss Over Gen')
|
754
754
|
hist['artist1'].append(art1)
|
755
755
|
|
756
756
|
# Accuracy plot
|
757
|
+
|
757
758
|
for i in range(len(best_acc_per_depth_list)):
|
758
759
|
best_acc_per_depth_list[i] = best_acc_per_depth_list[i].get()
|
759
760
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: pyerualjetwork
|
3
|
-
Version: 4.3.
|
3
|
+
Version: 4.3.11
|
4
4
|
Summary: PyerualJetwork is a machine learning library supported with GPU(CUDA) acceleration written in Python for professionals and researchers including with PLAN algorithm, PLANEAT algorithm (genetic optimization). Also includes data pre-process and memory manegament
|
5
5
|
Author: Hasan Can Beydili
|
6
6
|
Author-email: tchasancan@gmail.com
|
@@ -51,7 +51,7 @@ YouTube Tutorials: https://www.youtube.com/watch?v=6wMQstZ00is&list=PLNgNWpM7Hbs
|
|
51
51
|
'psutil==6.1.1'
|
52
52
|
]
|
53
53
|
|
54
|
-
matplotlib, networkx (optional).
|
54
|
+
matplotlib, networkx, seaborn (optional).
|
55
55
|
|
56
56
|
##############################
|
57
57
|
|
@@ -0,0 +1,25 @@
|
|
1
|
+
pyerualjetwork/__init__.py,sha256=5snQTX8efGK_JdtgdrZU57alCIlsFlcYror1R3HFFDQ,1280
|
2
|
+
pyerualjetwork/activation_functions.py,sha256=Ms0AGBqkJuCA42ht64MSQnO54Td_1eDGquedpoBDVbc,7642
|
3
|
+
pyerualjetwork/activation_functions_cuda.py,sha256=5y1Ti3GDfDteQDCUmODwe7tAyDAUlDTKmIikChQ8d6g,7772
|
4
|
+
pyerualjetwork/data_operations.py,sha256=Flteouu6rfSo2uHMqBHuzO02dXmbNa-I5qWmUpGTZ5Y,14760
|
5
|
+
pyerualjetwork/data_operations_cuda.py,sha256=ZcjmLXE1-HVwedextYdJZ1rgrns1OfSekzFpr1a9m6o,17625
|
6
|
+
pyerualjetwork/fitness_functions.py,sha256=9ZtCasBVjk3qQldi2pIzV6c5hTBJJyGCU0eVhJsoxPo,1145
|
7
|
+
pyerualjetwork/help.py,sha256=nQ_YbYA2RtuafhuvkreNpX0WWL1I_nzlelwCtvei0_Y,775
|
8
|
+
pyerualjetwork/loss_functions.py,sha256=6PyBI232SQRGuFnG3LDGvnv_PUdWzT2_2mUODJiejGI,618
|
9
|
+
pyerualjetwork/loss_functions_cuda.py,sha256=C93IZJcrOpT6HMK9x1O4AHJWXYTkN5WZiqdssPbvAPk,617
|
10
|
+
pyerualjetwork/memory_operations.py,sha256=I7QiZ--xSyRkFF0wcckPwZV7K9emEvyx5aJ3DiRHZFI,13468
|
11
|
+
pyerualjetwork/metrics.py,sha256=q7MkhnZDRbCjFBDDfUgrl8lBYnUT_1ro1LxeBq105pI,6077
|
12
|
+
pyerualjetwork/metrics_cuda.py,sha256=73h9GC7XwmnFCVzFEEiPQfF8CwHIz2wsCbxpZrJtYgw,5061
|
13
|
+
pyerualjetwork/model_operations.py,sha256=MCSCNYiiICRVZITobtS3ZIWmH5Q9gjyELuH32sAdgg4,12649
|
14
|
+
pyerualjetwork/model_operations_cuda.py,sha256=NT01BK5nrDYE7H1x3KnSI8gmx0QTGGB0mP_LqEb1uuU,13157
|
15
|
+
pyerualjetwork/plan.py,sha256=M8KVSXUjmSkf4bK7FnizLZjWQfI_3suHIJ-XminM2SM,22291
|
16
|
+
pyerualjetwork/plan_cuda.py,sha256=hRPPw4YsiVpfoXkNN1le1zaVHJuSySvihiFBt2eHT7c,23388
|
17
|
+
pyerualjetwork/planeat.py,sha256=FBoYirDE8R0zGtB-EFsHdQ50RijC7rnZHqDIiXx67rs,37571
|
18
|
+
pyerualjetwork/planeat_cuda.py,sha256=5suVlOfHcThhGbPlHbxAqbp5-rpbw4E2H6trvDvrmg4,37627
|
19
|
+
pyerualjetwork/ui.py,sha256=wu2BhU1k-w3Kcho5Jtq4SEKe68ftaUeRGneUOSCVDjU,575
|
20
|
+
pyerualjetwork/visualizations.py,sha256=t1BqnFUH5jKiPdFMI2kWjFg6-amrBV0wvW05aD77NQs,28288
|
21
|
+
pyerualjetwork/visualizations_cuda.py,sha256=PYRqj4QYUbuYMYcNwO8yaTPB-jK7E6kZHhTrAi0lwPU,28749
|
22
|
+
pyerualjetwork-4.3.11.dist-info/METADATA,sha256=6P6WgEt2RMPwvhAVg1cswZvq_fNB875egS_dEiabzlo,7506
|
23
|
+
pyerualjetwork-4.3.11.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
|
24
|
+
pyerualjetwork-4.3.11.dist-info/top_level.txt,sha256=BRyt62U_r3ZmJpj-wXNOoA345Bzamrj6RbaWsyW4tRg,15
|
25
|
+
pyerualjetwork-4.3.11.dist-info/RECORD,,
|
@@ -1,76 +0,0 @@
|
|
1
|
-
import cupy as cp
|
2
|
-
|
3
|
-
def pairwise_distances(X):
|
4
|
-
"""
|
5
|
-
Pairwise Euclidean distances using CuPy.
|
6
|
-
:param X: A (n_samples, n_features) CuPy array.
|
7
|
-
:return: A 2D CuPy array of distances (n_samples, n_samples).
|
8
|
-
"""
|
9
|
-
X = X.reshape(X.shape[0], -1)
|
10
|
-
r = cp.sum(X**2, axis=1, keepdims=True)
|
11
|
-
|
12
|
-
distances = cp.maximum(r - 2 * cp.dot(X, X.T) + r.T, 0.0)
|
13
|
-
|
14
|
-
return cp.sqrt(distances)
|
15
|
-
|
16
|
-
def diversity_score(population):
|
17
|
-
"""
|
18
|
-
Calculates the diversity score of a population based on the
|
19
|
-
Euclidean distances between individuals in the population.
|
20
|
-
|
21
|
-
:param population: A CuPy array of shape (n_samples, n_features).
|
22
|
-
:return: The function returns a CuPy array of diversity scores for each matrix in the population.
|
23
|
-
"""
|
24
|
-
if len(population) < 2:
|
25
|
-
return cp.zeros(len(population))
|
26
|
-
|
27
|
-
population = cp.nan_to_num(population, nan=0.0)
|
28
|
-
|
29
|
-
distances = pairwise_distances(population)
|
30
|
-
distances = cp.maximum(distances, 1e-10)
|
31
|
-
|
32
|
-
n_samples = len(population)
|
33
|
-
|
34
|
-
mask = cp.ones((n_samples, n_samples), dtype=bool)
|
35
|
-
cp.fill_diagonal(mask, 0)
|
36
|
-
|
37
|
-
avg_distances = cp.sum(distances * mask, axis=1) / (n_samples - 1)
|
38
|
-
|
39
|
-
if population.shape[1] == 0:
|
40
|
-
return cp.zeros(len(population))
|
41
|
-
|
42
|
-
max_possible_distance = cp.sqrt(population.shape[1])
|
43
|
-
max_possible_distance = float(max(max_possible_distance, 1e-10))
|
44
|
-
|
45
|
-
diversity_scores = avg_distances / max_possible_distance
|
46
|
-
|
47
|
-
return diversity_scores
|
48
|
-
|
49
|
-
|
50
|
-
def multi_head_fitness(y_true, y_pred, diversity_score, accuracy, alpha=2, beta=1.5, lambda_div=0.05):
|
51
|
-
"""
|
52
|
-
Computes a fitness score based on accuracy, margin loss, and diversity score.
|
53
|
-
|
54
|
-
:param y_true: Ground truth labels (CupPy array).
|
55
|
-
:param y_pred: Predicted labels (CupPy array).
|
56
|
-
:param diversity_score: Diversity score for a single weight matrix.
|
57
|
-
:param accuracy: Model accuracy.
|
58
|
-
:param alpha: Exponent for accuracy impact.
|
59
|
-
:param beta: Exponent for margin loss impact.
|
60
|
-
:param lambda_div: Weight for diversity score influence.
|
61
|
-
:return: Computed fitness value.
|
62
|
-
"""
|
63
|
-
incorrect = y_true != y_pred
|
64
|
-
|
65
|
-
if cp.any(incorrect):
|
66
|
-
margin_loss = cp.abs(y_true[incorrect] - y_pred[incorrect]).mean()
|
67
|
-
else:
|
68
|
-
margin_loss = 0.0
|
69
|
-
|
70
|
-
margin_loss = cp.nan_to_num(margin_loss, nan=0.0)
|
71
|
-
accuracy = max(accuracy, 1e-10)
|
72
|
-
|
73
|
-
fitness = (accuracy ** alpha) * ((1 - margin_loss) ** beta) * (1 + lambda_div * diversity_score)
|
74
|
-
fitness = cp.nan_to_num(fitness, nan=0.0, posinf=1e10, neginf=-1e10)
|
75
|
-
|
76
|
-
return fitness
|
@@ -1,24 +0,0 @@
|
|
1
|
-
pyerualjetwork/__init__.py,sha256=DRF05WRIM4ITdnKZ98dzqV5ZrU9gDodseJHR7XZRSVk,642
|
2
|
-
pyerualjetwork/activation_functions.py,sha256=bKf00lsuuLJNO-4vVp4OqBi4zJ-qZ8L3v-vl52notkY,7721
|
3
|
-
pyerualjetwork/activation_functions_cuda.py,sha256=5y1Ti3GDfDteQDCUmODwe7tAyDAUlDTKmIikChQ8d6g,7772
|
4
|
-
pyerualjetwork/data_operations.py,sha256=Flteouu6rfSo2uHMqBHuzO02dXmbNa-I5qWmUpGTZ5Y,14760
|
5
|
-
pyerualjetwork/data_operations_cuda.py,sha256=ZcjmLXE1-HVwedextYdJZ1rgrns1OfSekzFpr1a9m6o,17625
|
6
|
-
pyerualjetwork/fitness_functions.py,sha256=h-eigMH1n7vw2C53PAUGJFvUxtC_sVrFkGYH1zMcEmw,2698
|
7
|
-
pyerualjetwork/fitness_functions_cuda.py,sha256=nncxtoTjF-_UBmACeiohUot5x-sUpsZpTuobFz6Jd4s,2640
|
8
|
-
pyerualjetwork/help.py,sha256=nQ_YbYA2RtuafhuvkreNpX0WWL1I_nzlelwCtvei0_Y,775
|
9
|
-
pyerualjetwork/memory_operations.py,sha256=I7QiZ--xSyRkFF0wcckPwZV7K9emEvyx5aJ3DiRHZFI,13468
|
10
|
-
pyerualjetwork/metrics.py,sha256=q7MkhnZDRbCjFBDDfUgrl8lBYnUT_1ro1LxeBq105pI,6077
|
11
|
-
pyerualjetwork/metrics_cuda.py,sha256=73h9GC7XwmnFCVzFEEiPQfF8CwHIz2wsCbxpZrJtYgw,5061
|
12
|
-
pyerualjetwork/model_operations.py,sha256=MCSCNYiiICRVZITobtS3ZIWmH5Q9gjyELuH32sAdgg4,12649
|
13
|
-
pyerualjetwork/model_operations_cuda.py,sha256=NT01BK5nrDYE7H1x3KnSI8gmx0QTGGB0mP_LqEb1uuU,13157
|
14
|
-
pyerualjetwork/plan.py,sha256=Bxx4EYlVxVzsdtDCBoy_f_IE0LhrP7GILqyesRQ6x48,20486
|
15
|
-
pyerualjetwork/plan_cuda.py,sha256=K46cayXRowNFuLv_qkmcZIipT-nWn4yZ-UOinODD1iY,21011
|
16
|
-
pyerualjetwork/planeat.py,sha256=QyfPW5k0rvVnowtymI3LXLKp39y69M5DIyNBAeKIhRQ,37568
|
17
|
-
pyerualjetwork/planeat_cuda.py,sha256=zzsEu612Ren5K8YyvIn00RjYBG9Z6AOikiReGAPrBbg,37624
|
18
|
-
pyerualjetwork/ui.py,sha256=wu2BhU1k-w3Kcho5Jtq4SEKe68ftaUeRGneUOSCVDjU,575
|
19
|
-
pyerualjetwork/visualizations.py,sha256=6uY9U4qGOlG4aIhGdAFcTm2Z5m9Hxww4tAAG61_Vqb8,28294
|
20
|
-
pyerualjetwork/visualizations_cuda.py,sha256=o6MyZUVz040lmdI2kGlOEU2IoxvyWJPhSOKe9vkLMvY,28753
|
21
|
-
pyerualjetwork-4.3.10b2.dist-info/METADATA,sha256=Fv7c0vWkKDf6rzg4rK30PaUmT1B2MOvJZrpdqGHc5lA,7499
|
22
|
-
pyerualjetwork-4.3.10b2.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
|
23
|
-
pyerualjetwork-4.3.10b2.dist-info/top_level.txt,sha256=BRyt62U_r3ZmJpj-wXNOoA345Bzamrj6RbaWsyW4tRg,15
|
24
|
-
pyerualjetwork-4.3.10b2.dist-info/RECORD,,
|
File without changes
|
File without changes
|