pyerualjetwork 5.57a0__py3-none-any.whl → 5.57a2__py3-none-any.whl

pyerualjetwork/__init__.py

@@ -42,7 +42,7 @@ PyerualJetwork document: https://github.com/HCB06/PyerualJetwork/blob/main/Welco
 - Contact: tchasancan@gmail.com
 """
 
-__version__ = "5.57a0"
+__version__ = "5.57a2"
 __update__ = """* Changes: https://github.com/HCB06/PyerualJetwork/blob/main/CHANGES
 * PyerualJetwork Homepage: https://github.com/HCB06/PyerualJetwork/tree/main
 * PyerualJetwork document: https://github.com/HCB06/PyerualJetwork/blob/main/Welcome_to_PyerualJetwork/PYERUALJETWORK_USER_MANUEL_AND_LEGAL_INFORMATION(EN).pdf

pyerualjetwork/memory_ops.py

@@ -28,6 +28,88 @@ import psutil
 import numpy as np
 import cupy as cp
 import logging
+from multiprocessing import shared_memory
+
+x_train_shape = None
+y_train_shape = None
+x_train_dtype = None
+y_train_dtype = None
+x_shm_name = None
+y_shm_name = None
+
+
+def batcher_worker(index, batcher_fn):
+    """
+    Shared memory ile batcher çalıştıran worker.
+
+    :param index: Batch indexi
+    :param batcher_fn: batcher fonksiyonu (örneğin: def batcher(x, y): ...)
+    :return: (x_batch, y_batch)
+    """
+    existing_x = shared_memory.SharedMemory(name=x_shm_name)
+    existing_y = shared_memory.SharedMemory(name=y_shm_name)
+
+    x_np = np.ndarray(x_train_shape, dtype=x_train_dtype, buffer=existing_x.buf)
+    y_np = np.ndarray(y_train_shape, dtype=y_train_dtype, buffer=existing_y.buf)
+
+    result = batcher_fn(x_np, y_np)
+
+    existing_x.close()
+    existing_y.close()
+    return result
+
+
+def cleanup_shared_memory(x_shm, y_shm):
+    """
+    Shared memory nesnelerini kapatır ve belleği serbest bırakır.
+    """
+    x_shm.close()
+    x_shm.unlink()
+    y_shm.close()
+    y_shm.unlink()
+
+def evaluate_worker(params, evaluate_fn):
+    """
+    Shared memory kullanan evaluate fonksiyonu için wrapper.
+
+    :param params: evaluate fonksiyonuna gidecek parametreler
+    :param evaluate_fn: evaluate fonksiyonu
+    :return: evaluate sonucu
+    """
+    return evaluate_fn(*params)
+
+
+def plan_fit_worker(params, plan_fit_fn):
+    """
+    Shared memory kullanan plan_fit fonksiyonu için wrapper.
+
+    :param params: plan_fit fonksiyonuna gidecek parametreler
+    :param plan_fit_fn: plan_fit fonksiyonu
+    :return: plan_fit sonucu
+    """
+    return plan_fit_fn(*params)
+
+def loss_worker(params, loss_fn):
+    """
+    Shared memory kullanan loss fonksiyonu için wrapper.
+
+    :param params: (y_true_batch, y_pred_batch)
+    :param loss_fn: categorical_crossentropy veya binary_crossentropy
+    :return: loss sonucu
+    """
+    return loss_fn(*params)
+
+
+def fitness_worker(params, fitness_fn):
+    """
+    Shared memory kullanan fitness (örn. wals) fonksiyonu için wrapper.
+
+    :param params: (acc, loss, acc_impact, loss_impact)
+    :param fitness_fn: fitness fonksiyonu
+    :return: fitness skoru
+    """
+    return fitness_fn(*params)
+
 
 def get_available_cpu_memory():
     """

pyerualjetwork/nn.py

@@ -51,20 +51,22 @@ import numpy as np
 import cupy as cp
 import copy
 import random
-import multiprocessing
-from multiprocessing import Pool, cpu_count
+from multiprocessing import Pool, cpu_count, shared_memory
+import atexit
+from functools import partial
 
 ### LIBRARY IMPORTS ###
 from .ui import loading_bars, initialize_loading_bar
 from .cpu.activation_functions import all_activations
 from .model_ops import get_acc, get_preds_softmax, get_preds
-from .memory_ops import optimize_labels, transfer_to_gpu
+import memory_ops
+from .memory_ops import optimize_labels, transfer_to_gpu, batcher_worker, cleanup_shared_memory, plan_fit_worker, fitness_worker, loss_worker, evaluate_worker
 from .fitness_functions import wals
 
 ### GLOBAL VARIABLES ###
-bar_format_normal = loading_bars()[0]
 bar_format_learner = loading_bars()[1]
 
+
 # BUILD -----
 
 def plan_fit(
@@ -309,7 +311,7 @@ def learn(x_train, y_train, optimizer, gen, pop_size, fit_start=True, batch_size
     loss_list = []
     target_pop = []
 
-    progress = initialize_loading_bar(total=gen if isinstance(gen) == int else gen[0] + gen[1], desc="", ncols=85, bar_format=bar_format_learner)
+    progress = initialize_loading_bar(total=gen if isinstance(gen, int) else gen[0] + gen[1], desc="", ncols=85, bar_format=bar_format_learner)
 
     if fit_start is False:
         weight_pop, act_pop = define_genomes(input_shape=len(x_train[0]), output_shape=len(y_train[0]), neurons=neurons, activation_functions=activations, population_size=pop_size, dtype=dtype)
@@ -364,12 +366,30 @@ def learn(x_train, y_train, optimizer, gen, pop_size, fit_start=True, batch_size
 
         if parallel_training:
 
+            x_shm = shared_memory.SharedMemory(create=True, size=x_train.nbytes)
+            y_shm = shared_memory.SharedMemory(create=True, size=y_train.nbytes)
+
+            # Kopyalama shared memory alanına
+            np.copyto(np.ndarray(x_train.shape, dtype=x_train.dtype, buffer=x_shm.buf), x_train)
+            np.copyto(np.ndarray(y_train.shape, dtype=y_train.dtype, buffer=y_shm.buf), y_train)
+
+            # memory_ops modülüne metadata aktar
+            memory_ops.x_shm_name = x_shm.name
+            memory_ops.y_shm_name = y_shm.name
+            memory_ops.x_train_shape = x_train.shape
+            memory_ops.y_train_shape = y_train.shape
+            memory_ops.x_train_dtype = x_train.dtype
+            memory_ops.y_train_dtype = y_train.dtype
+
+            # Bellek temizliği için register
+            atexit.register(lambda: cleanup_shared_memory(x_shm, y_shm))
+
             eval_params = []
             fit_params = []
 
             with Pool(processes=thread_count) as pool:
-
-                batches = pool.starmap(batcher, [(x_train, y_train)] * pop_size)
+                batched_worker = partial(batcher_worker, batcher_fn=batcher)
+                batches = pool.map(batched_worker, range(pop_size))
 
                 eval_params = [
                     (
@@ -388,7 +408,7 @@ def learn(x_train, y_train, optimizer, gen, pop_size, fit_start=True, batch_size
                     if start_this_act is not None and i == 0:
                         w_copy = copy.deepcopy(weight_pop[0])
 
-                    W = pool.starmap(plan_fit, fit_params)
+                    W = pool.map(partial(plan_fit_worker, plan_fit_fn=plan_fit), fit_params)
                     new_weights = W
 
                     eval_params = [
@@ -403,14 +423,14 @@ def learn(x_train, y_train, optimizer, gen, pop_size, fit_start=True, batch_size
                             eval_params[0][8], eval_params[0][9]
                         )
 
-                models = pool.starmap(evaluate, eval_params)
+                models = pool.map(partial(evaluate_worker, evaluate_fn=evaluate), eval_params)
 
                 y_preds = [model[get_preds_softmax()] for model in models]
                 loss_params = [(eval_params[f][1], y_preds[f]) for f in range(pop_size)]
                 loss_func = categorical_crossentropy if loss == 'categorical_crossentropy' else binary_crossentropy
-                losses = pool.starmap(loss_func, loss_params)
+                losses = pool.map(partial(loss_worker, loss_fn=loss_func), loss_params)
                 fitness_params = [(models[f][get_acc()], losses[f], acc_impact, loss_impact) for f in range(pop_size)]
-                target_pop = pool.starmap(wals, fitness_params)
+                target_pop = pool.map(partial(fitness_worker, fitness_fn=wals), fitness_params)
 
                 if cuda:
                     target_pop = [fit.get() for fit in target_pop]

{pyerualjetwork-5.57a0.dist-info → pyerualjetwork-5.57a2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyerualjetwork
-Version: 5.57a0
+Version: 5.57a2
 Summary: PyereualJetwork is a GPU-accelerated + Parallel Threading Supported machine learning library in Python for professionals and researchers. It features PLAN, MLP, Deep Learning training, and ENE (Eugenic NeuroEvolution) for genetic optimization, applicable to genetic algorithms or Reinforcement Learning (RL). The library includes data pre-processing, visualizations, model saving/loading, prediction, evaluation, training, and detailed or simplified memory management.
 Author: Hasan Can Beydili
 Author-email: tchasancan@gmail.com

{pyerualjetwork-5.57a0.dist-info → pyerualjetwork-5.57a2.dist-info}/RECORD

@@ -1,11 +1,11 @@
-pyerualjetwork/__init__.py,sha256=Is2WYk7i-Z0QINy6UNTDQd5RZLX_fj7wgkbphaZlrfc,3089
+pyerualjetwork/__init__.py,sha256=__SZdVuhFDcJeHnZZyaKEnVbON1UcFV7v79WfyUVjkk,3089
 pyerualjetwork/ene.py,sha256=luTvspHRTose6s3uRas40pNXyKoxU9siaHiMBNI5yoc,42136
 pyerualjetwork/fitness_functions.py,sha256=D9JVCr9DFid_xXgBD4uCKxdW2k10MVDE5HZRSOK4Igg,1237
 pyerualjetwork/help.py,sha256=sn9jBzXkQsTZvdgsUXUpSs_BbYYIgY3whofg6dj8peI,848
 pyerualjetwork/issue_solver.py,sha256=uay_9XK6xWnLmK2P_BeyDQlyNXzg_zYffnXYd228wZk,4102
-pyerualjetwork/memory_ops.py,sha256=TUFh9SYWCKL6N-vNdWId_EwU313TuZomQCHOrltrD-4,14280
+pyerualjetwork/memory_ops.py,sha256=lAuBphM_cNGA3GKs9icqKfVUqaBq3TaT2oTPRtZsMNA,16517
 pyerualjetwork/model_ops.py,sha256=WaP1XwKqXMfZl4Yop8a1Bg0xtmLYgap9JFOWHaLr7S4,25143
-pyerualjetwork/nn.py,sha256=y62ton0CZGxSgPavWt8tb4XEfQ6bVBUubKw5NE7To_s,41404
+pyerualjetwork/nn.py,sha256=V5DJ8oQb2nZEwFyxpNXVYskt5TK0-BTSMmpVpHBFMFg,42614
 pyerualjetwork/old_cpu_model_ops.py,sha256=1KNgjUeYCO_TsA5RtbNiuIiBJzq8-rL2dE6jxKqCBU0,21481
 pyerualjetwork/old_cuda_model_ops.py,sha256=KAscAd8e_I8Vqdd9BJaHd6-IG6fhxFglAFxys0sqmEo,23079
 pyerualjetwork/ui.py,sha256=JBTFYz5R24XwNKhA3GSW-oYAoiIBxAE3kFGXkvm5gqw,656
@@ -21,7 +21,7 @@ pyerualjetwork/cuda/data_ops.py,sha256=BEXh4M7BWXaTpYlVS9D2i3CGgOmL5131vy7FZyuTQ
 pyerualjetwork/cuda/loss_functions.py,sha256=C93IZJcrOpT6HMK9x1O4AHJWXYTkN5WZiqdssPbvAPk,617
 pyerualjetwork/cuda/metrics.py,sha256=PjDBoRvr6va8vRvDIJJGBO4-I4uumrk3NCM1Vz4NJTo,5054
 pyerualjetwork/cuda/visualizations.py,sha256=2mHE7iqqsN3K6xtCnemS4o_YWGS0bIV2IxF4cG6Ur9k,20090
-pyerualjetwork-5.57a0.dist-info/METADATA,sha256=ZUZ3suCf0uBfmT2UuRw3xw1M2WVfJNAUZ95TypCNOQM,8052
-pyerualjetwork-5.57a0.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-pyerualjetwork-5.57a0.dist-info/top_level.txt,sha256=BRyt62U_r3ZmJpj-wXNOoA345Bzamrj6RbaWsyW4tRg,15
-pyerualjetwork-5.57a0.dist-info/RECORD,,
+pyerualjetwork-5.57a2.dist-info/METADATA,sha256=kd3PKYtkb9OlJe-k3n-7RxYi27Cds-6zMI7wKvoHiuA,8052
+pyerualjetwork-5.57a2.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+pyerualjetwork-5.57a2.dist-info/top_level.txt,sha256=BRyt62U_r3ZmJpj-wXNOoA345Bzamrj6RbaWsyW4tRg,15
+pyerualjetwork-5.57a2.dist-info/RECORD,,