pyerualjetwork 5.57a0__py3-none-any.whl → 5.57a2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyerualjetwork/__init__.py +1 -1
- pyerualjetwork/memory_ops.py +82 -0
- pyerualjetwork/nn.py +31 -11
- {pyerualjetwork-5.57a0.dist-info → pyerualjetwork-5.57a2.dist-info}/METADATA +1 -1
- {pyerualjetwork-5.57a0.dist-info → pyerualjetwork-5.57a2.dist-info}/RECORD +7 -7
- {pyerualjetwork-5.57a0.dist-info → pyerualjetwork-5.57a2.dist-info}/WHEEL +0 -0
- {pyerualjetwork-5.57a0.dist-info → pyerualjetwork-5.57a2.dist-info}/top_level.txt +0 -0
pyerualjetwork/__init__.py
CHANGED
@@ -42,7 +42,7 @@ PyerualJetwork document: https://github.com/HCB06/PyerualJetwork/blob/main/Welco
|
|
42
42
|
- Contact: tchasancan@gmail.com
|
43
43
|
"""
|
44
44
|
|
45
|
-
__version__ = "5.
|
45
|
+
__version__ = "5.57a2"
|
46
46
|
__update__ = """* Changes: https://github.com/HCB06/PyerualJetwork/blob/main/CHANGES
|
47
47
|
* PyerualJetwork Homepage: https://github.com/HCB06/PyerualJetwork/tree/main
|
48
48
|
* PyerualJetwork document: https://github.com/HCB06/PyerualJetwork/blob/main/Welcome_to_PyerualJetwork/PYERUALJETWORK_USER_MANUEL_AND_LEGAL_INFORMATION(EN).pdf
|
pyerualjetwork/memory_ops.py
CHANGED
@@ -28,6 +28,88 @@ import psutil
|
|
28
28
|
import numpy as np
|
29
29
|
import cupy as cp
|
30
30
|
import logging
|
31
|
+
from multiprocessing import shared_memory
|
32
|
+
|
33
|
+
x_train_shape = None
|
34
|
+
y_train_shape = None
|
35
|
+
x_train_dtype = None
|
36
|
+
y_train_dtype = None
|
37
|
+
x_shm_name = None
|
38
|
+
y_shm_name = None
|
39
|
+
|
40
|
+
|
41
|
+
def batcher_worker(index, batcher_fn):
|
42
|
+
"""
|
43
|
+
Shared memory ile batcher çalıştıran worker.
|
44
|
+
|
45
|
+
:param index: Batch indexi
|
46
|
+
:param batcher_fn: batcher fonksiyonu (örneğin: def batcher(x, y): ...)
|
47
|
+
:return: (x_batch, y_batch)
|
48
|
+
"""
|
49
|
+
existing_x = shared_memory.SharedMemory(name=x_shm_name)
|
50
|
+
existing_y = shared_memory.SharedMemory(name=y_shm_name)
|
51
|
+
|
52
|
+
x_np = np.ndarray(x_train_shape, dtype=x_train_dtype, buffer=existing_x.buf)
|
53
|
+
y_np = np.ndarray(y_train_shape, dtype=y_train_dtype, buffer=existing_y.buf)
|
54
|
+
|
55
|
+
result = batcher_fn(x_np, y_np)
|
56
|
+
|
57
|
+
existing_x.close()
|
58
|
+
existing_y.close()
|
59
|
+
return result
|
60
|
+
|
61
|
+
|
62
|
+
def cleanup_shared_memory(x_shm, y_shm):
|
63
|
+
"""
|
64
|
+
Shared memory nesnelerini kapatır ve belleği serbest bırakır.
|
65
|
+
"""
|
66
|
+
x_shm.close()
|
67
|
+
x_shm.unlink()
|
68
|
+
y_shm.close()
|
69
|
+
y_shm.unlink()
|
70
|
+
|
71
|
+
def evaluate_worker(params, evaluate_fn):
|
72
|
+
"""
|
73
|
+
Shared memory kullanan evaluate fonksiyonu için wrapper.
|
74
|
+
|
75
|
+
:param params: evaluate fonksiyonuna gidecek parametreler
|
76
|
+
:param evaluate_fn: evaluate fonksiyonu
|
77
|
+
:return: evaluate sonucu
|
78
|
+
"""
|
79
|
+
return evaluate_fn(*params)
|
80
|
+
|
81
|
+
|
82
|
+
def plan_fit_worker(params, plan_fit_fn):
|
83
|
+
"""
|
84
|
+
Shared memory kullanan plan_fit fonksiyonu için wrapper.
|
85
|
+
|
86
|
+
:param params: plan_fit fonksiyonuna gidecek parametreler
|
87
|
+
:param plan_fit_fn: plan_fit fonksiyonu
|
88
|
+
:return: plan_fit sonucu
|
89
|
+
"""
|
90
|
+
return plan_fit_fn(*params)
|
91
|
+
|
92
|
+
def loss_worker(params, loss_fn):
|
93
|
+
"""
|
94
|
+
Shared memory kullanan loss fonksiyonu için wrapper.
|
95
|
+
|
96
|
+
:param params: (y_true_batch, y_pred_batch)
|
97
|
+
:param loss_fn: categorical_crossentropy veya binary_crossentropy
|
98
|
+
:return: loss sonucu
|
99
|
+
"""
|
100
|
+
return loss_fn(*params)
|
101
|
+
|
102
|
+
|
103
|
+
def fitness_worker(params, fitness_fn):
|
104
|
+
"""
|
105
|
+
Shared memory kullanan fitness (örn. wals) fonksiyonu için wrapper.
|
106
|
+
|
107
|
+
:param params: (acc, loss, acc_impact, loss_impact)
|
108
|
+
:param fitness_fn: fitness fonksiyonu
|
109
|
+
:return: fitness skoru
|
110
|
+
"""
|
111
|
+
return fitness_fn(*params)
|
112
|
+
|
31
113
|
|
32
114
|
def get_available_cpu_memory():
|
33
115
|
"""
|
pyerualjetwork/nn.py
CHANGED
@@ -51,20 +51,22 @@ import numpy as np
|
|
51
51
|
import cupy as cp
|
52
52
|
import copy
|
53
53
|
import random
|
54
|
-
import
|
55
|
-
|
54
|
+
from multiprocessing import Pool, cpu_count, shared_memory
|
55
|
+
import atexit
|
56
|
+
from functools import partial
|
56
57
|
|
57
58
|
### LIBRARY IMPORTS ###
|
58
59
|
from .ui import loading_bars, initialize_loading_bar
|
59
60
|
from .cpu.activation_functions import all_activations
|
60
61
|
from .model_ops import get_acc, get_preds_softmax, get_preds
|
61
|
-
|
62
|
+
import memory_ops
|
63
|
+
from .memory_ops import optimize_labels, transfer_to_gpu, batcher_worker, cleanup_shared_memory, plan_fit_worker, fitness_worker, loss_worker, evaluate_worker
|
62
64
|
from .fitness_functions import wals
|
63
65
|
|
64
66
|
### GLOBAL VARIABLES ###
|
65
|
-
bar_format_normal = loading_bars()[0]
|
66
67
|
bar_format_learner = loading_bars()[1]
|
67
68
|
|
69
|
+
|
68
70
|
# BUILD -----
|
69
71
|
|
70
72
|
def plan_fit(
|
@@ -309,7 +311,7 @@ def learn(x_train, y_train, optimizer, gen, pop_size, fit_start=True, batch_size
|
|
309
311
|
loss_list = []
|
310
312
|
target_pop = []
|
311
313
|
|
312
|
-
progress = initialize_loading_bar(total=gen if isinstance(gen
|
314
|
+
progress = initialize_loading_bar(total=gen if isinstance(gen, int) else gen[0] + gen[1], desc="", ncols=85, bar_format=bar_format_learner)
|
313
315
|
|
314
316
|
if fit_start is False:
|
315
317
|
weight_pop, act_pop = define_genomes(input_shape=len(x_train[0]), output_shape=len(y_train[0]), neurons=neurons, activation_functions=activations, population_size=pop_size, dtype=dtype)
|
@@ -364,12 +366,30 @@ def learn(x_train, y_train, optimizer, gen, pop_size, fit_start=True, batch_size
|
|
364
366
|
|
365
367
|
if parallel_training:
|
366
368
|
|
369
|
+
x_shm = shared_memory.SharedMemory(create=True, size=x_train.nbytes)
|
370
|
+
y_shm = shared_memory.SharedMemory(create=True, size=y_train.nbytes)
|
371
|
+
|
372
|
+
# Kopyalama shared memory alanına
|
373
|
+
np.copyto(np.ndarray(x_train.shape, dtype=x_train.dtype, buffer=x_shm.buf), x_train)
|
374
|
+
np.copyto(np.ndarray(y_train.shape, dtype=y_train.dtype, buffer=y_shm.buf), y_train)
|
375
|
+
|
376
|
+
# memory_ops modülüne metadata aktar
|
377
|
+
memory_ops.x_shm_name = x_shm.name
|
378
|
+
memory_ops.y_shm_name = y_shm.name
|
379
|
+
memory_ops.x_train_shape = x_train.shape
|
380
|
+
memory_ops.y_train_shape = y_train.shape
|
381
|
+
memory_ops.x_train_dtype = x_train.dtype
|
382
|
+
memory_ops.y_train_dtype = y_train.dtype
|
383
|
+
|
384
|
+
# Bellek temizliği için register
|
385
|
+
atexit.register(lambda: cleanup_shared_memory(x_shm, y_shm))
|
386
|
+
|
367
387
|
eval_params = []
|
368
388
|
fit_params = []
|
369
389
|
|
370
390
|
with Pool(processes=thread_count) as pool:
|
371
|
-
|
372
|
-
batches = pool.
|
391
|
+
batched_worker = partial(batcher_worker, batcher_fn=batcher)
|
392
|
+
batches = pool.map(batched_worker, range(pop_size))
|
373
393
|
|
374
394
|
eval_params = [
|
375
395
|
(
|
@@ -388,7 +408,7 @@ def learn(x_train, y_train, optimizer, gen, pop_size, fit_start=True, batch_size
|
|
388
408
|
if start_this_act is not None and i == 0:
|
389
409
|
w_copy = copy.deepcopy(weight_pop[0])
|
390
410
|
|
391
|
-
W = pool.
|
411
|
+
W = pool.map(partial(plan_fit_worker, plan_fit_fn=plan_fit), fit_params)
|
392
412
|
new_weights = W
|
393
413
|
|
394
414
|
eval_params = [
|
@@ -403,14 +423,14 @@ def learn(x_train, y_train, optimizer, gen, pop_size, fit_start=True, batch_size
|
|
403
423
|
eval_params[0][8], eval_params[0][9]
|
404
424
|
)
|
405
425
|
|
406
|
-
models = pool.
|
426
|
+
models = pool.map(partial(evaluate_worker, evaluate_fn=evaluate), eval_params)
|
407
427
|
|
408
428
|
y_preds = [model[get_preds_softmax()] for model in models]
|
409
429
|
loss_params = [(eval_params[f][1], y_preds[f]) for f in range(pop_size)]
|
410
430
|
loss_func = categorical_crossentropy if loss == 'categorical_crossentropy' else binary_crossentropy
|
411
|
-
losses = pool.
|
431
|
+
losses = pool.map(partial(loss_worker, loss_fn=loss_func), loss_params)
|
412
432
|
fitness_params = [(models[f][get_acc()], losses[f], acc_impact, loss_impact) for f in range(pop_size)]
|
413
|
-
target_pop = pool.
|
433
|
+
target_pop = pool.map(partial(fitness_worker, fitness_fn=wals), fitness_params)
|
414
434
|
|
415
435
|
if cuda:
|
416
436
|
target_pop = [fit.get() for fit in target_pop]
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: pyerualjetwork
|
3
|
-
Version: 5.
|
3
|
+
Version: 5.57a2
|
4
4
|
Summary: PyereualJetwork is a GPU-accelerated + Parallel Threading Supported machine learning library in Python for professionals and researchers. It features PLAN, MLP, Deep Learning training, and ENE (Eugenic NeuroEvolution) for genetic optimization, applicable to genetic algorithms or Reinforcement Learning (RL). The library includes data pre-processing, visualizations, model saving/loading, prediction, evaluation, training, and detailed or simplified memory management.
|
5
5
|
Author: Hasan Can Beydili
|
6
6
|
Author-email: tchasancan@gmail.com
|
@@ -1,11 +1,11 @@
|
|
1
|
-
pyerualjetwork/__init__.py,sha256=
|
1
|
+
pyerualjetwork/__init__.py,sha256=__SZdVuhFDcJeHnZZyaKEnVbON1UcFV7v79WfyUVjkk,3089
|
2
2
|
pyerualjetwork/ene.py,sha256=luTvspHRTose6s3uRas40pNXyKoxU9siaHiMBNI5yoc,42136
|
3
3
|
pyerualjetwork/fitness_functions.py,sha256=D9JVCr9DFid_xXgBD4uCKxdW2k10MVDE5HZRSOK4Igg,1237
|
4
4
|
pyerualjetwork/help.py,sha256=sn9jBzXkQsTZvdgsUXUpSs_BbYYIgY3whofg6dj8peI,848
|
5
5
|
pyerualjetwork/issue_solver.py,sha256=uay_9XK6xWnLmK2P_BeyDQlyNXzg_zYffnXYd228wZk,4102
|
6
|
-
pyerualjetwork/memory_ops.py,sha256=
|
6
|
+
pyerualjetwork/memory_ops.py,sha256=lAuBphM_cNGA3GKs9icqKfVUqaBq3TaT2oTPRtZsMNA,16517
|
7
7
|
pyerualjetwork/model_ops.py,sha256=WaP1XwKqXMfZl4Yop8a1Bg0xtmLYgap9JFOWHaLr7S4,25143
|
8
|
-
pyerualjetwork/nn.py,sha256=
|
8
|
+
pyerualjetwork/nn.py,sha256=V5DJ8oQb2nZEwFyxpNXVYskt5TK0-BTSMmpVpHBFMFg,42614
|
9
9
|
pyerualjetwork/old_cpu_model_ops.py,sha256=1KNgjUeYCO_TsA5RtbNiuIiBJzq8-rL2dE6jxKqCBU0,21481
|
10
10
|
pyerualjetwork/old_cuda_model_ops.py,sha256=KAscAd8e_I8Vqdd9BJaHd6-IG6fhxFglAFxys0sqmEo,23079
|
11
11
|
pyerualjetwork/ui.py,sha256=JBTFYz5R24XwNKhA3GSW-oYAoiIBxAE3kFGXkvm5gqw,656
|
@@ -21,7 +21,7 @@ pyerualjetwork/cuda/data_ops.py,sha256=BEXh4M7BWXaTpYlVS9D2i3CGgOmL5131vy7FZyuTQ
|
|
21
21
|
pyerualjetwork/cuda/loss_functions.py,sha256=C93IZJcrOpT6HMK9x1O4AHJWXYTkN5WZiqdssPbvAPk,617
|
22
22
|
pyerualjetwork/cuda/metrics.py,sha256=PjDBoRvr6va8vRvDIJJGBO4-I4uumrk3NCM1Vz4NJTo,5054
|
23
23
|
pyerualjetwork/cuda/visualizations.py,sha256=2mHE7iqqsN3K6xtCnemS4o_YWGS0bIV2IxF4cG6Ur9k,20090
|
24
|
-
pyerualjetwork-5.
|
25
|
-
pyerualjetwork-5.
|
26
|
-
pyerualjetwork-5.
|
27
|
-
pyerualjetwork-5.
|
24
|
+
pyerualjetwork-5.57a2.dist-info/METADATA,sha256=kd3PKYtkb9OlJe-k3n-7RxYi27Cds-6zMI7wKvoHiuA,8052
|
25
|
+
pyerualjetwork-5.57a2.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
|
26
|
+
pyerualjetwork-5.57a2.dist-info/top_level.txt,sha256=BRyt62U_r3ZmJpj-wXNOoA345Bzamrj6RbaWsyW4tRg,15
|
27
|
+
pyerualjetwork-5.57a2.dist-info/RECORD,,
|
File without changes
|
File without changes
|