pyerualjetwork 5.57a0__py3-none-any.whl → 5.57a2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -42,7 +42,7 @@ PyerualJetwork document: https://github.com/HCB06/PyerualJetwork/blob/main/Welco
42
42
  - Contact: tchasancan@gmail.com
43
43
  """
44
44
 
45
- __version__ = "5.57a0"
45
+ __version__ = "5.57a2"
46
46
  __update__ = """* Changes: https://github.com/HCB06/PyerualJetwork/blob/main/CHANGES
47
47
  * PyerualJetwork Homepage: https://github.com/HCB06/PyerualJetwork/tree/main
48
48
  * PyerualJetwork document: https://github.com/HCB06/PyerualJetwork/blob/main/Welcome_to_PyerualJetwork/PYERUALJETWORK_USER_MANUEL_AND_LEGAL_INFORMATION(EN).pdf
@@ -28,6 +28,88 @@ import psutil
28
28
  import numpy as np
29
29
  import cupy as cp
30
30
  import logging
31
+ from multiprocessing import shared_memory
32
+
33
+ x_train_shape = None
34
+ y_train_shape = None
35
+ x_train_dtype = None
36
+ y_train_dtype = None
37
+ x_shm_name = None
38
+ y_shm_name = None
39
+
40
+
41
+ def batcher_worker(index, batcher_fn):
42
+ """
43
+ Shared memory ile batcher çalıştıran worker.
44
+
45
+ :param index: Batch indexi
46
+ :param batcher_fn: batcher fonksiyonu (örneğin: def batcher(x, y): ...)
47
+ :return: (x_batch, y_batch)
48
+ """
49
+ existing_x = shared_memory.SharedMemory(name=x_shm_name)
50
+ existing_y = shared_memory.SharedMemory(name=y_shm_name)
51
+
52
+ x_np = np.ndarray(x_train_shape, dtype=x_train_dtype, buffer=existing_x.buf)
53
+ y_np = np.ndarray(y_train_shape, dtype=y_train_dtype, buffer=existing_y.buf)
54
+
55
+ result = batcher_fn(x_np, y_np)
56
+
57
+ existing_x.close()
58
+ existing_y.close()
59
+ return result
60
+
61
+
62
+ def cleanup_shared_memory(x_shm, y_shm):
63
+ """
64
+ Shared memory nesnelerini kapatır ve belleği serbest bırakır.
65
+ """
66
+ x_shm.close()
67
+ x_shm.unlink()
68
+ y_shm.close()
69
+ y_shm.unlink()
70
+
71
+ def evaluate_worker(params, evaluate_fn):
72
+ """
73
+ Shared memory kullanan evaluate fonksiyonu için wrapper.
74
+
75
+ :param params: evaluate fonksiyonuna gidecek parametreler
76
+ :param evaluate_fn: evaluate fonksiyonu
77
+ :return: evaluate sonucu
78
+ """
79
+ return evaluate_fn(*params)
80
+
81
+
82
+ def plan_fit_worker(params, plan_fit_fn):
83
+ """
84
+ Shared memory kullanan plan_fit fonksiyonu için wrapper.
85
+
86
+ :param params: plan_fit fonksiyonuna gidecek parametreler
87
+ :param plan_fit_fn: plan_fit fonksiyonu
88
+ :return: plan_fit sonucu
89
+ """
90
+ return plan_fit_fn(*params)
91
+
92
+ def loss_worker(params, loss_fn):
93
+ """
94
+ Shared memory kullanan loss fonksiyonu için wrapper.
95
+
96
+ :param params: (y_true_batch, y_pred_batch)
97
+ :param loss_fn: categorical_crossentropy veya binary_crossentropy
98
+ :return: loss sonucu
99
+ """
100
+ return loss_fn(*params)
101
+
102
+
103
+ def fitness_worker(params, fitness_fn):
104
+ """
105
+ Shared memory kullanan fitness (örn. wals) fonksiyonu için wrapper.
106
+
107
+ :param params: (acc, loss, acc_impact, loss_impact)
108
+ :param fitness_fn: fitness fonksiyonu
109
+ :return: fitness skoru
110
+ """
111
+ return fitness_fn(*params)
112
+
31
113
 
32
114
  def get_available_cpu_memory():
33
115
  """
pyerualjetwork/nn.py CHANGED
@@ -51,20 +51,22 @@ import numpy as np
51
51
  import cupy as cp
52
52
  import copy
53
53
  import random
54
- import multiprocessing
55
- from multiprocessing import Pool, cpu_count
54
+ from multiprocessing import Pool, cpu_count, shared_memory
55
+ import atexit
56
+ from functools import partial
56
57
 
57
58
  ### LIBRARY IMPORTS ###
58
59
  from .ui import loading_bars, initialize_loading_bar
59
60
  from .cpu.activation_functions import all_activations
60
61
  from .model_ops import get_acc, get_preds_softmax, get_preds
61
- from .memory_ops import optimize_labels, transfer_to_gpu
62
+ import memory_ops
63
+ from .memory_ops import optimize_labels, transfer_to_gpu, batcher_worker, cleanup_shared_memory, plan_fit_worker, fitness_worker, loss_worker, evaluate_worker
62
64
  from .fitness_functions import wals
63
65
 
64
66
  ### GLOBAL VARIABLES ###
65
- bar_format_normal = loading_bars()[0]
66
67
  bar_format_learner = loading_bars()[1]
67
68
 
69
+
68
70
  # BUILD -----
69
71
 
70
72
  def plan_fit(
@@ -309,7 +311,7 @@ def learn(x_train, y_train, optimizer, gen, pop_size, fit_start=True, batch_size
309
311
  loss_list = []
310
312
  target_pop = []
311
313
 
312
- progress = initialize_loading_bar(total=gen if isinstance(gen) == int else gen[0] + gen[1], desc="", ncols=85, bar_format=bar_format_learner)
314
+ progress = initialize_loading_bar(total=gen if isinstance(gen, int) else gen[0] + gen[1], desc="", ncols=85, bar_format=bar_format_learner)
313
315
 
314
316
  if fit_start is False:
315
317
  weight_pop, act_pop = define_genomes(input_shape=len(x_train[0]), output_shape=len(y_train[0]), neurons=neurons, activation_functions=activations, population_size=pop_size, dtype=dtype)
@@ -364,12 +366,30 @@ def learn(x_train, y_train, optimizer, gen, pop_size, fit_start=True, batch_size
364
366
 
365
367
  if parallel_training:
366
368
 
369
+ x_shm = shared_memory.SharedMemory(create=True, size=x_train.nbytes)
370
+ y_shm = shared_memory.SharedMemory(create=True, size=y_train.nbytes)
371
+
372
+ # Kopyalama shared memory alanına
373
+ np.copyto(np.ndarray(x_train.shape, dtype=x_train.dtype, buffer=x_shm.buf), x_train)
374
+ np.copyto(np.ndarray(y_train.shape, dtype=y_train.dtype, buffer=y_shm.buf), y_train)
375
+
376
+ # memory_ops modülüne metadata aktar
377
+ memory_ops.x_shm_name = x_shm.name
378
+ memory_ops.y_shm_name = y_shm.name
379
+ memory_ops.x_train_shape = x_train.shape
380
+ memory_ops.y_train_shape = y_train.shape
381
+ memory_ops.x_train_dtype = x_train.dtype
382
+ memory_ops.y_train_dtype = y_train.dtype
383
+
384
+ # Bellek temizliği için register
385
+ atexit.register(lambda: cleanup_shared_memory(x_shm, y_shm))
386
+
367
387
  eval_params = []
368
388
  fit_params = []
369
389
 
370
390
  with Pool(processes=thread_count) as pool:
371
-
372
- batches = pool.starmap(batcher, [(x_train, y_train)] * pop_size)
391
+ batched_worker = partial(batcher_worker, batcher_fn=batcher)
392
+ batches = pool.map(batched_worker, range(pop_size))
373
393
 
374
394
  eval_params = [
375
395
  (
@@ -388,7 +408,7 @@ def learn(x_train, y_train, optimizer, gen, pop_size, fit_start=True, batch_size
388
408
  if start_this_act is not None and i == 0:
389
409
  w_copy = copy.deepcopy(weight_pop[0])
390
410
 
391
- W = pool.starmap(plan_fit, fit_params)
411
+ W = pool.map(partial(plan_fit_worker, plan_fit_fn=plan_fit), fit_params)
392
412
  new_weights = W
393
413
 
394
414
  eval_params = [
@@ -403,14 +423,14 @@ def learn(x_train, y_train, optimizer, gen, pop_size, fit_start=True, batch_size
403
423
  eval_params[0][8], eval_params[0][9]
404
424
  )
405
425
 
406
- models = pool.starmap(evaluate, eval_params)
426
+ models = pool.map(partial(evaluate_worker, evaluate_fn=evaluate), eval_params)
407
427
 
408
428
  y_preds = [model[get_preds_softmax()] for model in models]
409
429
  loss_params = [(eval_params[f][1], y_preds[f]) for f in range(pop_size)]
410
430
  loss_func = categorical_crossentropy if loss == 'categorical_crossentropy' else binary_crossentropy
411
- losses = pool.starmap(loss_func, loss_params)
431
+ losses = pool.map(partial(loss_worker, loss_fn=loss_func), loss_params)
412
432
  fitness_params = [(models[f][get_acc()], losses[f], acc_impact, loss_impact) for f in range(pop_size)]
413
- target_pop = pool.starmap(wals, fitness_params)
433
+ target_pop = pool.map(partial(fitness_worker, fitness_fn=wals), fitness_params)
414
434
 
415
435
  if cuda:
416
436
  target_pop = [fit.get() for fit in target_pop]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: pyerualjetwork
3
- Version: 5.57a0
3
+ Version: 5.57a2
4
4
  Summary: PyereualJetwork is a GPU-accelerated + Parallel Threading Supported machine learning library in Python for professionals and researchers. It features PLAN, MLP, Deep Learning training, and ENE (Eugenic NeuroEvolution) for genetic optimization, applicable to genetic algorithms or Reinforcement Learning (RL). The library includes data pre-processing, visualizations, model saving/loading, prediction, evaluation, training, and detailed or simplified memory management.
5
5
  Author: Hasan Can Beydili
6
6
  Author-email: tchasancan@gmail.com
@@ -1,11 +1,11 @@
1
- pyerualjetwork/__init__.py,sha256=Is2WYk7i-Z0QINy6UNTDQd5RZLX_fj7wgkbphaZlrfc,3089
1
+ pyerualjetwork/__init__.py,sha256=__SZdVuhFDcJeHnZZyaKEnVbON1UcFV7v79WfyUVjkk,3089
2
2
  pyerualjetwork/ene.py,sha256=luTvspHRTose6s3uRas40pNXyKoxU9siaHiMBNI5yoc,42136
3
3
  pyerualjetwork/fitness_functions.py,sha256=D9JVCr9DFid_xXgBD4uCKxdW2k10MVDE5HZRSOK4Igg,1237
4
4
  pyerualjetwork/help.py,sha256=sn9jBzXkQsTZvdgsUXUpSs_BbYYIgY3whofg6dj8peI,848
5
5
  pyerualjetwork/issue_solver.py,sha256=uay_9XK6xWnLmK2P_BeyDQlyNXzg_zYffnXYd228wZk,4102
6
- pyerualjetwork/memory_ops.py,sha256=TUFh9SYWCKL6N-vNdWId_EwU313TuZomQCHOrltrD-4,14280
6
+ pyerualjetwork/memory_ops.py,sha256=lAuBphM_cNGA3GKs9icqKfVUqaBq3TaT2oTPRtZsMNA,16517
7
7
  pyerualjetwork/model_ops.py,sha256=WaP1XwKqXMfZl4Yop8a1Bg0xtmLYgap9JFOWHaLr7S4,25143
8
- pyerualjetwork/nn.py,sha256=y62ton0CZGxSgPavWt8tb4XEfQ6bVBUubKw5NE7To_s,41404
8
+ pyerualjetwork/nn.py,sha256=V5DJ8oQb2nZEwFyxpNXVYskt5TK0-BTSMmpVpHBFMFg,42614
9
9
  pyerualjetwork/old_cpu_model_ops.py,sha256=1KNgjUeYCO_TsA5RtbNiuIiBJzq8-rL2dE6jxKqCBU0,21481
10
10
  pyerualjetwork/old_cuda_model_ops.py,sha256=KAscAd8e_I8Vqdd9BJaHd6-IG6fhxFglAFxys0sqmEo,23079
11
11
  pyerualjetwork/ui.py,sha256=JBTFYz5R24XwNKhA3GSW-oYAoiIBxAE3kFGXkvm5gqw,656
@@ -21,7 +21,7 @@ pyerualjetwork/cuda/data_ops.py,sha256=BEXh4M7BWXaTpYlVS9D2i3CGgOmL5131vy7FZyuTQ
21
21
  pyerualjetwork/cuda/loss_functions.py,sha256=C93IZJcrOpT6HMK9x1O4AHJWXYTkN5WZiqdssPbvAPk,617
22
22
  pyerualjetwork/cuda/metrics.py,sha256=PjDBoRvr6va8vRvDIJJGBO4-I4uumrk3NCM1Vz4NJTo,5054
23
23
  pyerualjetwork/cuda/visualizations.py,sha256=2mHE7iqqsN3K6xtCnemS4o_YWGS0bIV2IxF4cG6Ur9k,20090
24
- pyerualjetwork-5.57a0.dist-info/METADATA,sha256=ZUZ3suCf0uBfmT2UuRw3xw1M2WVfJNAUZ95TypCNOQM,8052
25
- pyerualjetwork-5.57a0.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
26
- pyerualjetwork-5.57a0.dist-info/top_level.txt,sha256=BRyt62U_r3ZmJpj-wXNOoA345Bzamrj6RbaWsyW4tRg,15
27
- pyerualjetwork-5.57a0.dist-info/RECORD,,
24
+ pyerualjetwork-5.57a2.dist-info/METADATA,sha256=kd3PKYtkb9OlJe-k3n-7RxYi27Cds-6zMI7wKvoHiuA,8052
25
+ pyerualjetwork-5.57a2.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
26
+ pyerualjetwork-5.57a2.dist-info/top_level.txt,sha256=BRyt62U_r3ZmJpj-wXNOoA345Bzamrj6RbaWsyW4tRg,15
27
+ pyerualjetwork-5.57a2.dist-info/RECORD,,