uteAI 0.1.0__py3-none-any.whl

uteAI/__init__.py

@@ -0,0 +1,8 @@
+"""
+uteAI: Advanced AI & Neural Network Library from HCMUTE
+"""
+
+__version__ = "0.1.2026"
+__author__ = "Dung Cai"
+
+from . import core

uteAI/core/__init__.py

@@ -0,0 +1,25 @@
+"""
+uteAI Core: Neural Network & High-Performance Engines
+"""
+
+from .mlp_cuda import (
+    Model,
+    Dense,
+    Dense_Dropout,
+    Activation_ReLU,
+    Activation_Sigmoid,
+    Activation_Linear,
+    Activation_SoftMax,
+    Loss_CategoricalCrossEntropy,
+    Loss_BinaryCrossEntropy,
+    Loss_MeanSquaredError,
+    Loss_MeanAbsoluteError,
+    Activation_Softmax_Loss_CategoricalCrossEntropy,
+    Optimizer_SGD,
+    Optimizer_RMSprop,
+    Optimizer_Adam,
+    Accuracy_Categorical,
+    Accuracy_Regression,
+    is_cuda_available,
+    get_gpu_device_name
+)

uteAI/core/mlp_cuda.py

@@ -0,0 +1,1051 @@
+"""
+Educational Multilayer Perceptron (MLP) Neural Network with CUDA GPU Acceleration
+==================================================================================
+Hardware-accelerated Deep Learning engine implementing standard feedforward
+neural networks from scratch with seamless CPU/GPU duality (NumPy / CuPy).
+
+Key GPU Acceleration Principles:
+    1. Parallel Matrix Multiplication (cuBLAS):
+       Forward:   Z = X @ W + b        (GEMM computed across thousands of CUDA cores)
+       Backward:  dW = X.T @ dZ,       db = sum(dZ, axis=0),  dX = dZ @ W.T
+    2. Zero-Copy Host-to-Device Pipeline:
+       Entire training sets are transferred to GPU VRAM once at the start of training.
+       Mini-batch slicing occurs directly in VRAM with zero PCIe transfer bottleneck.
+    3. Seamless CPU Fallback:
+       If an NVIDIA GPU or CuPy is not available, execution automatically and safely
+       falls back to NumPy on CPU without code modifications.
+    4. Cross-Platform Model Portability:
+       Model weights are saved as standard NumPy arrays in pickle format, ensuring that
+       checkpoints saved by MLP_CUDA can be loaded by pure-CPU code and vice-versa.
+
+Usage:
+    import MLP_CUDA as nn_model
+
+    # Automatically selects CUDA GPU if available, else CPU
+    model = nn_model.Model(device='cuda')
+    model.add(nn_model.Dense(784, 128))
+    model.add(nn_model.Activation_ReLU())
+    model.add(nn_model.Dense_Dropout(0.1))
+    model.add(nn_model.Dense(128, 64))
+    model.add(nn_model.Activation_ReLU())
+    model.add(nn_model.Dense(64, 10))
+    model.add(nn_model.Activation_Linear())
+
+    model.set(
+        loss=nn_model.Activation_Softmax_Loss_CategoricalCrossEntropy(),
+        optimizer=nn_model.Optimizer_Adam(learning_rate=0.001, decay=1e-5),
+        accuracy=nn_model.Accuracy_Categorical()
+    )
+    model.finalize()
+    model.train(X_train, y_train, validation_data=(X_val, y_val), epochs=10, batch_size=128)
+    predictions = model.predict(X_test)
+"""
+
+import math
+import pickle
+import sys
+import time
+import numpy as np
+
+# Check for CUDA (CuPy) support
+try:
+    import cupy as cp
+    _CUDA_AVAILABLE = True
+    try:
+        _DEVICE_NAME = cp.cuda.runtime.getDeviceProperties(0)['name'].decode()
+    except Exception:
+        _DEVICE_NAME = "NVIDIA CUDA GPU"
+except (ImportError, Exception):
+    cp = None
+    _CUDA_AVAILABLE = False
+    _DEVICE_NAME = "None (CPU only)"
+
+
+def is_cuda_available():
+    """Return True if NVIDIA CUDA / CuPy is available."""
+    return _CUDA_AVAILABLE
+
+
+def get_gpu_device_name():
+    """Return the name of the detected GPU device, or a fallback string."""
+    return _DEVICE_NAME
+
+
+def get_array_module(x):
+    """
+    Return cupy if x is a CuPy ndarray and CUDA is available, else numpy.
+    """
+    if _CUDA_AVAILABLE and isinstance(x, cp.ndarray):
+        return cp
+    return np
+
+
+def to_cpu(arr):
+    """Convert an array to a standard NumPy array on CPU."""
+    if _CUDA_AVAILABLE and isinstance(arr, cp.ndarray):
+        return cp.asnumpy(arr)
+    if isinstance(arr, np.ndarray):
+        return arr
+    return np.asarray(arr)
+
+
+def to_gpu(arr, dtype=None):
+    """Convert an array to a CuPy ndarray on GPU (if CUDA is available)."""
+    if not _CUDA_AVAILABLE:
+        return np.asarray(arr, dtype=dtype)
+    if isinstance(arr, cp.ndarray):
+        return arr.astype(dtype) if dtype else arr
+    return cp.asarray(arr, dtype=dtype)
+
+
+#########################################################################
+############################## LOSS CLASSES #############################
+#########################################################################
+
+class Loss:
+    """
+    Base class for loss computation: calculates batch loss, accumulated epoch loss,
+    and regularization penalties across trainable layers.
+    """
+
+    def __init__(self, xp=np):
+        self.xp = xp
+        self.accumulated_sum = 0.0
+        self.accumulated_count = 0
+        self.trainable_layers = []
+
+    def set_array_module(self, xp):
+        """Update backend numerical library (numpy or cupy)."""
+        self.xp = xp
+
+    def remember_trainable_layers(self, trainable_layers):
+        """Store references to layers subject to regularization."""
+        self.trainable_layers = trainable_layers
+
+    def calculate(self, output, y, *, include_regularization=False):
+        """Compute batch loss and accumulate running totals for epoch averaging."""
+        sample_losses = self.forward(output, y)
+        data_loss = float(self.xp.mean(sample_losses))
+
+        self.accumulated_sum += float(self.xp.sum(sample_losses))
+        self.accumulated_count += len(sample_losses)
+
+        if not include_regularization:
+            return data_loss
+        return data_loss, self.regularization_loss()
+
+    def calculate_accumulated(self, *, include_regularization=False):
+        """Compute average loss per sample across all batches in the epoch."""
+        data_loss = self.accumulated_sum / max(1, self.accumulated_count)
+        if not include_regularization:
+            return data_loss
+        return data_loss, self.regularization_loss()
+
+    def new_pass(self):
+        """Reset accumulated epoch metrics."""
+        self.accumulated_sum = 0.0
+        self.accumulated_count = 0
+
+    def regularization_loss(self):
+        """
+        Compute total L1 and L2 regularization penalty over all trainable layers:
+            L_reg = sum(lambda_1 * |W| + lambda_2 * W^2 + lambda_1 * |b| + lambda_2 * b^2)
+        """
+        reg_loss = 0.0
+        for layer in self.trainable_layers:
+            if getattr(layer, 'weights_regularizer_l1', 0) > 0:
+                reg_loss += layer.weights_regularizer_l1 * float(self.xp.sum(self.xp.abs(layer.weights)))
+            if getattr(layer, 'weights_regularizer_l2', 0) > 0:
+                reg_loss += layer.weights_regularizer_l2 * float(self.xp.sum(layer.weights ** 2))
+            if getattr(layer, 'biases_regularizer_l1', 0) > 0:
+                reg_loss += layer.biases_regularizer_l1 * float(self.xp.sum(self.xp.abs(layer.biases)))
+            if getattr(layer, 'biases_regularizer_l2', 0) > 0:
+                reg_loss += layer.biases_regularizer_l2 * float(self.xp.sum(layer.biases ** 2))
+        return reg_loss
+
+
+class Loss_CategoricalCrossEntropy(Loss):
+    """
+    Categorical Cross-Entropy Loss for multi-class classification:
+        L = -log(y_pred[true_class])
+    """
+
+    def forward(self, y_pred, y_true):
+        """Compute sample-wise loss with clipping to avoid log(0)."""
+        samples = len(y_pred)
+        y_pred_clipped = self.xp.clip(y_pred, 1e-7, 1.0 - 1e-7)
+
+        if len(y_true.shape) == 1:
+            correct_confidences = y_pred_clipped[self.xp.arange(samples), y_true]
+        elif len(y_true.shape) == 2:
+            correct_confidences = self.xp.sum(y_pred_clipped * y_true, axis=1)
+
+        negative_log_likelihoods = -self.xp.log(correct_confidences)
+        return negative_log_likelihoods
+
+    def backward(self, dvalues, y_true):
+        """Backpropagate categorical cross-entropy gradient."""
+        samples = len(dvalues)
+        labels = len(dvalues[0])
+
+        if len(y_true.shape) == 1:
+            y_true_onehot = self.xp.zeros((samples, labels), dtype=dvalues.dtype)
+            y_true_onehot[self.xp.arange(samples), y_true] = 1.0
+            y_true = y_true_onehot
+
+        self.dinputs = -y_true / self.xp.clip(dvalues, 1e-7, 1.0 - 1e-7)
+        self.dinputs = self.dinputs / samples
+
+
+class Loss_BinaryCrossEntropy(Loss):
+    """Binary Cross-Entropy Loss for binary classification tasks."""
+
+    def forward(self, y_pred, y_true):
+        """Compute sample-wise binary cross-entropy loss."""
+        y_pred_clipped = self.xp.clip(y_pred, 1e-7, 1.0 - 1e-7)
+        sample_losses = -(y_true * self.xp.log(y_pred_clipped) + (1.0 - y_true) * self.xp.log(1.0 - y_pred_clipped))
+        return self.xp.mean(sample_losses, axis=-1)
+
+    def backward(self, dvalues, y_true):
+        """Backpropagate binary cross-entropy gradient."""
+        samples = len(dvalues)
+        outputs = len(dvalues[0])
+        clipped_dvalues = self.xp.clip(dvalues, 1e-7, 1.0 - 1e-7)
+        self.dinputs = -(y_true / clipped_dvalues - (1.0 - y_true) / (1.0 - clipped_dvalues)) / outputs
+        self.dinputs = self.dinputs / samples
+
+
+class Loss_MeanSquaredError(Loss):
+    """Mean Squared Error (MSE / L2) Loss for regression."""
+
+    def forward(self, y_pred, y_true):
+        """Compute MSE loss per sample."""
+        return self.xp.mean((y_true - y_pred) ** 2, axis=-1)
+
+    def backward(self, dvalues, y_true):
+        """Backpropagate MSE gradient: dinputs = -2 * (y_true - y_pred) / outputs / samples."""
+        samples = len(dvalues)
+        outputs = len(dvalues[0])
+        self.dinputs = -2.0 * (y_true - dvalues) / outputs
+        self.dinputs = self.dinputs / samples
+
+
+class Loss_MeanAbsoluteError(Loss):
+    """Mean Absolute Error (MAE / L1) Loss for regression."""
+
+    def forward(self, y_pred, y_true):
+        """Compute MAE loss per sample."""
+        return self.xp.mean(self.xp.abs(y_true - y_pred), axis=-1)
+
+    def backward(self, dvalues, y_true):
+        """Backpropagate MAE gradient."""
+        samples = len(dvalues)
+        outputs = len(dvalues[0])
+        self.dinputs = self.xp.sign(dvalues - y_true) / outputs
+        self.dinputs = self.dinputs / samples
+
+
+class Activation_Softmax_Loss_CategoricalCrossEntropy(Loss):
+    """
+    Fused Softmax Activation and Categorical Cross-Entropy Loss:
+    Analytically simplifies the gradient to dZ = (p - y) / N, avoiding NxN Jacobian tensor
+    evaluations and preventing numerical overflow/underflow during backpropagation.
+    """
+
+    def __init__(self, xp=np):
+        super().__init__(xp=xp)
+        self.activation = Activation_SoftMax(xp=xp)
+        self.loss = Loss_CategoricalCrossEntropy(xp=xp)
+
+    def set_array_module(self, xp):
+        """Update backend numerical library."""
+        super().set_array_module(xp)
+        self.activation.set_array_module(xp)
+        self.loss.set_array_module(xp)
+
+    def forward(self, inputs, y_true):
+        """Forward pass through Softmax activation followed by CCE loss."""
+        self.activation.forward(inputs)
+        self.output = self.activation.output
+        return self.loss.forward(self.output, y_true)
+
+    def backward(self, dvalues, y_true):
+        """
+        Fused gradient: dinputs = (p - y) / N
+        Optimized for CUDA by using xp.arange() directly on GPU.
+        """
+        samples = len(dvalues)
+
+        if len(y_true.shape) == 2:
+            y_true = self.xp.argmax(y_true, axis=1)
+
+        self.dinputs = self.output.copy()
+        # Fast GPU indexing
+        self.dinputs[self.xp.arange(samples), y_true] -= 1.0
+        self.dinputs = self.dinputs / samples
+
+
+#########################################################################
+########################## ACTIVATION CLASSES ###########################
+#########################################################################
+
+class Activation_ReLU:
+    """
+    Rectified Linear Unit (ReLU):
+        forward:  f(x) = max(0, x)
+        backward: f'(x) = 1 if x > 0 else 0
+    """
+
+    def __init__(self, xp=np):
+        self.xp = xp
+
+    def set_array_module(self, xp):
+        self.xp = xp
+
+    def forward(self, inputs, training=True):
+        self.inputs = inputs
+        self.output = self.xp.maximum(0, inputs)
+
+    def backward(self, dvalues):
+        self.dinputs = dvalues.copy()
+        self.dinputs[self.inputs <= 0] = 0.0
+
+    def predictions(self, outputs):
+        return outputs
+
+
+class Activation_Sigmoid:
+    """
+    Sigmoid Activation:
+        forward:  f(x) = 1 / (1 + exp(-x))
+        backward: f'(x) = f(x) * (1 - f(x))
+    """
+
+    def __init__(self, xp=np):
+        self.xp = xp
+
+    def set_array_module(self, xp):
+        self.xp = xp
+
+    def forward(self, inputs, training=True):
+        self.inputs = inputs
+        self.output = 1.0 / (1.0 + self.xp.exp(-inputs))
+
+    def backward(self, dvalues):
+        self.dinputs = dvalues * (1.0 - self.output) * self.output
+
+    def predictions(self, outputs):
+        return (outputs > 0.5) * 1
+
+
+class Activation_Linear:
+    """Identity / Linear activation for regression or raw logit outputs."""
+
+    def __init__(self, xp=np):
+        self.xp = xp
+
+    def set_array_module(self, xp):
+        self.xp = xp
+
+    def forward(self, inputs, training=True):
+        self.inputs = inputs
+        self.output = inputs
+
+    def backward(self, dvalues):
+        self.dinputs = dvalues.copy()
+
+    def predictions(self, outputs):
+        return outputs
+
+
+class Activation_SoftMax:
+    """
+    Softmax Activation:
+        P_i = exp(z_i - max(z)) / sum(exp(z - max(z)))
+    """
+
+    def __init__(self, xp=np):
+        self.xp = xp
+
+    def set_array_module(self, xp):
+        self.xp = xp
+
+    def forward(self, inputs, training=True):
+        self.inputs = inputs
+        exp_values = self.xp.exp(inputs - self.xp.max(inputs, axis=1, keepdims=True))
+        self.output = exp_values / self.xp.sum(exp_values, axis=1, keepdims=True)
+
+    def backward(self, dvalues):
+        self.dinputs = self.xp.empty_like(dvalues)
+        for index, (single_output, single_dvalues) in enumerate(zip(self.output, dvalues)):
+            single_output = single_output.reshape(-1, 1)
+            jacobian_matrix = self.xp.diagflat(single_output) - self.xp.dot(single_output, single_output.T)
+            self.dinputs[index] = self.xp.dot(jacobian_matrix, single_dvalues)
+
+    def predictions(self, outputs):
+        return self.xp.argmax(outputs, axis=1)
+
+
+#########################################################################
+############################# DENSE LAYERS ##############################
+#########################################################################
+
+class Dense:
+    """
+    Fully-Connected (Dense) Layer with CUDA cuBLAS Matrix Acceleration:
+        Forward:   Z = X @ W + b
+        Backward:  dW = X.T @ dZ,   db = sum(dZ, axis=0),   dX = dZ @ W.T
+
+    Parameters
+    ----------
+    n_inputs : int
+        Number of input features.
+    n_neurons : int
+        Number of output neurons.
+    weight_init : str, default='gaussian'
+        Initialization scheme: 'gaussian' (0.01 * randn), 'he' (sqrt(2 / n_in)), or 'xavier'.
+    """
+
+    def __init__(self, n_inputs, n_neurons,
+                 weights_regularizer_l1=0.0, weights_regularizer_l2=0.0,
+                 biases_regularizer_l1=0.0, biases_regularizer_l2=0.0,
+                 weight_init='gaussian', xp=np):
+        self.xp = xp
+        self.n_inputs = n_inputs
+        self.n_neurons = n_neurons
+
+        # Weight initialization
+        if weight_init == 'he':
+            std = math.sqrt(2.0 / n_inputs)
+            weights_np = std * np.random.randn(n_inputs, n_neurons).astype(np.float32)
+        elif weight_init == 'xavier':
+            std = math.sqrt(2.0 / (n_inputs + n_neurons))
+            weights_np = std * np.random.randn(n_inputs, n_neurons).astype(np.float32)
+        else:
+            weights_np = (0.01 * np.random.randn(n_inputs, n_neurons)).astype(np.float32)
+
+        biases_np = np.zeros((1, n_neurons), dtype=np.float32)
+
+        if self.xp is cp and _CUDA_AVAILABLE:
+            self.weights = cp.asarray(weights_np)
+            self.biases = cp.asarray(biases_np)
+        else:
+            self.weights = weights_np
+            self.biases = biases_np
+
+        self.weights_regularizer_l1 = float(weights_regularizer_l1)
+        self.weights_regularizer_l2 = float(weights_regularizer_l2)
+        self.biases_regularizer_l1 = float(biases_regularizer_l1)
+        self.biases_regularizer_l2 = float(biases_regularizer_l2)
+
+    def set_array_module(self, xp):
+        """Migrate layer weights and biases between CPU and GPU."""
+        self.xp = xp
+        if xp is cp and _CUDA_AVAILABLE:
+            self.weights = to_gpu(self.weights)
+            self.biases = to_gpu(self.biases)
+        else:
+            self.weights = to_cpu(self.weights)
+            self.biases = to_cpu(self.biases)
+
+    def forward(self, inputs, training=True):
+        """Compute forward linear transformation on GPU/CPU."""
+        self.inputs = inputs
+        self.output = self.xp.dot(inputs, self.weights) + self.biases
+
+    def backward(self, dvalues):
+        """
+        Compute gradients with respect to weights, biases, and inputs:
+            dW = X.T @ dZ + dReg/dW
+            db = sum(dZ, axis=0) + dReg/db
+            dX = dZ @ W.T
+        """
+        self.dweights = self.xp.dot(self.inputs.T, dvalues)
+        self.dbiases = self.xp.sum(dvalues, axis=0, keepdims=True)
+
+        # L1 Weight Regularization
+        if self.weights_regularizer_l1 > 0:
+            dL1 = self.xp.ones_like(self.weights)
+            dL1[self.weights < 0] = -1.0
+            self.dweights += self.weights_regularizer_l1 * dL1
+
+        # L2 Weight Regularization
+        if self.weights_regularizer_l2 > 0:
+            self.dweights += 2.0 * self.weights_regularizer_l2 * self.weights
+
+        # L1 Bias Regularization
+        if self.biases_regularizer_l1 > 0:
+            dL1 = self.xp.ones_like(self.biases)
+            dL1[self.biases < 0] = -1.0
+            self.dbiases += self.biases_regularizer_l1 * dL1
+
+        # L2 Bias Regularization
+        if self.biases_regularizer_l2 > 0:
+            self.dbiases += 2.0 * self.biases_regularizer_l2 * self.biases
+
+        self.dinputs = self.xp.dot(dvalues, self.weights.T)
+
+    def get_parameters(self):
+        """Return layer parameters as CPU NumPy arrays for portable serialization."""
+        return to_cpu(self.weights), to_cpu(self.biases)
+
+    def set_parameters(self, weights, biases):
+        """Restore layer parameters and upload to active compute device."""
+        if self.xp is cp and _CUDA_AVAILABLE:
+            self.weights = to_gpu(weights)
+            self.biases = to_gpu(biases)
+        else:
+            self.weights = to_cpu(weights)
+            self.biases = to_cpu(biases)
+
+
+class Dense_Dropout:
+    """
+    Dropout Layer for Regularization:
+    Randomly zeroes a fraction `drop_rate` of inputs during training with inverted scaling.
+    Automatically bypassed during evaluation/inference (training=False).
+    """
+
+    def __init__(self, rate, xp=np):
+        self.xp = xp
+        self.rate = 1.0 - rate
+        self.drop_rate = rate
+
+    def set_array_module(self, xp):
+        self.xp = xp
+
+    def forward(self, inputs, training=True):
+        """Apply inverted dropout mask during training; pass-through during inference."""
+        if not training:
+            self.output = inputs.copy()
+            return
+
+        self.binary_mask = self.xp.random.binomial(1, self.rate, size=inputs.shape).astype(inputs.dtype) / self.rate
+        self.output = inputs * self.binary_mask
+
+    def backward(self, dvalues):
+        """Backpropagate gradient through active dropout mask."""
+        self.dinputs = dvalues * self.binary_mask
+
+
+#########################################################################
+########################### ACCURACY CLASSES ############################
+#########################################################################
+
+class Accuracy:
+    """Base class for computing classification or regression accuracy."""
+
+    def __init__(self, xp=np):
+        self.xp = xp
+        self.accumulated_sum = 0.0
+        self.accumulated_count = 0
+
+    def set_array_module(self, xp):
+        self.xp = xp
+
+    def calculate(self, predictions, y):
+        """Compute batch accuracy and update running totals."""
+        comparisons = self.compare(predictions, y)
+        self.accumulated_sum += float(self.xp.sum(comparisons))
+        self.accumulated_count += len(comparisons)
+        return float(self.xp.mean(comparisons))
+
+    def calculate_accumulated(self):
+        """Compute average accuracy across the entire epoch."""
+        return self.accumulated_sum / max(1, self.accumulated_count)
+
+    def new_pass(self):
+        """Reset running epoch metrics."""
+        self.accumulated_sum = 0.0
+        self.accumulated_count = 0
+
+
+class Accuracy_Categorical(Accuracy):
+    """
+    Categorical Accuracy:
+    Compares discrete class predictions with ground-truth targets.
+    """
+
+    def __init__(self, *, binary=False, xp=np):
+        super().__init__(xp=xp)
+        self.binary = binary
+
+    def init(self, y):
+        self.accumulated_sum = 0.0
+        self.accumulated_count = 0
+
+    def compare(self, predictions, y):
+        """Compare argmax predictions against integer class labels."""
+        if len(predictions.shape) == 2:
+            predictions = self.xp.argmax(predictions, axis=1)
+        if len(y.shape) == 2:
+            y = self.xp.argmax(y, axis=1)
+        return predictions == y
+
+
+class Accuracy_Regression(Accuracy):
+    """
+    Regression Accuracy:
+    Prediction is counted as correct if within std(y) / 250 tolerance.
+    """
+
+    def __init__(self, xp=np):
+        super().__init__(xp=xp)
+        self.precision = None
+
+    def init(self, y, reinit=False):
+        if self.precision is None or reinit:
+            self.precision = float(self.xp.std(y)) / 250.0
+
+    def compare(self, predictions, y):
+        return self.xp.abs(predictions - y) < self.precision
+
+
+#########################################################################
+########################## OPTIMIZER CLASSES ############################
+#########################################################################
+
+class Optimizer_SGD:
+    """Stochastic Gradient Descent (SGD) with Learning Rate Decay and Momentum."""
+
+    def __init__(self, learning_rate=1.0, decay=0.0, momentum=0.0, xp=np):
+        self.xp = xp
+        self.learning_rate = learning_rate
+        self.current_learning_rate = learning_rate
+        self.decay = decay
+        self.iterations = 0
+        self.momentum = momentum
+
+    def set_array_module(self, xp):
+        self.xp = xp
+
+    def pre_update_params(self):
+        if self.decay:
+            self.current_learning_rate = self.learning_rate * (1.0 / (1.0 + self.decay * self.iterations))
+
+    def update_params(self, layer):
+        if self.momentum:
+            if not hasattr(layer, 'weight_momentums'):
+                layer.weight_momentums = self.xp.zeros_like(layer.weights)
+                layer.bias_momentums = self.xp.zeros_like(layer.biases)
+
+            weight_updates = self.momentum * layer.weight_momentums - self.current_learning_rate * layer.dweights
+            layer.weight_momentums = weight_updates
+
+            bias_updates = self.momentum * layer.bias_momentums - self.current_learning_rate * layer.dbiases
+            layer.bias_momentums = bias_updates
+        else:
+            weight_updates = -self.current_learning_rate * layer.dweights
+            bias_updates = -self.current_learning_rate * layer.dbiases
+
+        layer.weights += weight_updates
+        layer.biases += bias_updates
+
+    def post_update_params(self):
+        self.iterations += 1
+
+
+class Optimizer_RMSprop:
+    """RMSprop Optimizer with Moving Average of Squared Gradients."""
+
+    def __init__(self, learning_rate=0.001, decay=0.0, epsilon=1e-7, rho=0.9, xp=np):
+        self.xp = xp
+        self.learning_rate = learning_rate
+        self.current_learning_rate = learning_rate
+        self.decay = decay
+        self.iterations = 0
+        self.epsilon = epsilon
+        self.rho = rho
+
+    def set_array_module(self, xp):
+        self.xp = xp
+
+    def pre_update_params(self):
+        if self.decay:
+            self.current_learning_rate = self.learning_rate * (1.0 / (1.0 + self.decay * self.iterations))
+
+    def update_params(self, layer):
+        if not hasattr(layer, 'weight_cache'):
+            layer.weight_cache = self.xp.zeros_like(layer.weights)
+            layer.bias_cache = self.xp.zeros_like(layer.biases)
+
+        layer.weight_cache = self.rho * layer.weight_cache + (1.0 - self.rho) * (layer.dweights ** 2)
+        layer.bias_cache = self.rho * layer.bias_cache + (1.0 - self.rho) * (layer.dbiases ** 2)
+
+        layer.weights += -self.current_learning_rate * layer.dweights / (self.xp.sqrt(layer.weight_cache) + self.epsilon)
+        layer.biases += -self.current_learning_rate * layer.dbiases / (self.xp.sqrt(layer.bias_cache) + self.epsilon)
+
+    def post_update_params(self):
+        self.iterations += 1
+
+
+class Optimizer_Adam:
+    """
+    Adam Optimizer (Adaptive Moment Estimation):
+    Combines first moments (momentum) and second moments (RMSprop) with bias correction.
+    Fully vectorized across GPU VRAM.
+    """
+
+    def __init__(self, learning_rate=0.001, decay=0.0, epsilon=1e-7, beta_1=0.9, beta_2=0.999, xp=np):
+        self.xp = xp
+        self.learning_rate = learning_rate
+        self.current_learning_rate = learning_rate
+        self.decay = decay
+        self.iterations = 0
+        self.epsilon = epsilon
+        self.beta_1 = beta_1
+        self.beta_2 = beta_2
+
+    def set_array_module(self, xp):
+        self.xp = xp
+
+    def pre_update_params(self):
+        if self.decay:
+            self.current_learning_rate = self.learning_rate * (1.0 / (1.0 + self.decay * self.iterations))
+
+    def update_params(self, layer):
+        if not hasattr(layer, 'weight_momentums'):
+            layer.weight_momentums = self.xp.zeros_like(layer.weights)
+            layer.weight_cache = self.xp.zeros_like(layer.weights)
+            layer.bias_momentums = self.xp.zeros_like(layer.biases)
+            layer.bias_cache = self.xp.zeros_like(layer.biases)
+
+        # Update biased first moment estimate
+        layer.weight_momentums = self.beta_1 * layer.weight_momentums + (1.0 - self.beta_1) * layer.dweights
+        layer.bias_momentums = self.beta_1 * layer.bias_momentums + (1.0 - self.beta_1) * layer.dbiases
+
+        # Update biased second raw moment estimate
+        layer.weight_cache = self.beta_2 * layer.weight_cache + (1.0 - self.beta_2) * (layer.dweights ** 2)
+        layer.bias_cache = self.beta_2 * layer.bias_cache + (1.0 - self.beta_2) * (layer.dbiases ** 2)
+
+        # Compute bias-corrected estimates
+        t = self.iterations + 1
+        weight_m_corrected = layer.weight_momentums / (1.0 - self.beta_1 ** t)
+        bias_m_corrected = layer.bias_momentums / (1.0 - self.beta_1 ** t)
+        weight_v_corrected = layer.weight_cache / (1.0 - self.beta_2 ** t)
+        bias_v_corrected = layer.bias_cache / (1.0 - self.beta_2 ** t)
+
+        # Tensor parameter updates
+        layer.weights += -self.current_learning_rate * weight_m_corrected / (self.xp.sqrt(weight_v_corrected) + self.epsilon)
+        layer.biases += -self.current_learning_rate * bias_m_corrected / (self.xp.sqrt(bias_v_corrected) + self.epsilon)
+
+    def post_update_params(self):
+        self.iterations += 1
+
+
+#########################################################################
+############################# MODEL CLASS ###############################
+#########################################################################
+
+class Model:
+    """
+    Sequential Neural Network Container with Hardware Acceleration:
+    Organizes sequential layers, forward/backward graphs, loss evaluation,
+    optimizers, and GPU/CPU device dispatch.
+
+    Parameters
+    ----------
+    device : str, default='cuda'
+        Compute target: 'cuda' (use NVIDIA GPU if available), 'cpu' (NumPy only),
+        or 'auto' (automatically pick GPU if available).
+    """
+
+    def __init__(self, device='cuda'):
+        self.layers = []
+        self.trainable_layers = []
+        self.loss = None
+        self.optimizer = None
+        self.accuracy = None
+        self.softmax_classifier_output = None
+
+        # Resolve compute backend
+        if device in ('cuda', 'auto'):
+            if _CUDA_AVAILABLE:
+                self.device = 'cuda'
+                self.xp = cp
+                print(f"[MLP_CUDA] Accelerated backend activated on: {_DEVICE_NAME}")
+            else:
+                self.device = 'cpu'
+                self.xp = np
+                print("[MLP_CUDA] Notice: CuPy/CUDA not found. Falling back to CPU (NumPy).")
+        else:
+            self.device = 'cpu'
+            self.xp = np
+            print("[MLP_CUDA] Running on CPU (NumPy backend).")
+
+    def to(self, device):
+        """Dynamically switch between 'cuda' and 'cpu' compute devices."""
+        if device == 'cuda':
+            if not _CUDA_AVAILABLE:
+                raise RuntimeError("Cannot switch to CUDA: CuPy or NVIDIA GPU is not available.")
+            self.device = 'cuda'
+            self.xp = cp
+        elif device == 'cpu':
+            self.device = 'cpu'
+            self.xp = np
+        else:
+            raise ValueError(f"Unknown device: {device}. Must be 'cuda' or 'cpu'.")
+
+        # Migrate all internal components
+        for layer in self.layers:
+            if hasattr(layer, 'set_array_module'):
+                layer.set_array_module(self.xp)
+        if self.loss and hasattr(self.loss, 'set_array_module'):
+            self.loss.set_array_module(self.xp)
+        if self.optimizer and hasattr(self.optimizer, 'set_array_module'):
+            self.optimizer.set_array_module(self.xp)
+        if self.accuracy and hasattr(self.accuracy, 'set_array_module'):
+            self.accuracy.set_array_module(self.xp)
+        return self
+
+    def add(self, layer):
+        """Append a layer to the network pipeline and configure its array backend."""
+        if hasattr(layer, 'set_array_module'):
+            layer.set_array_module(self.xp)
+        self.layers.append(layer)
+
+    def set(self, *, loss=None, optimizer=None, accuracy=None):
+        """Assign loss function, optimizer, and accuracy metric."""
+        if loss is not None:
+            if hasattr(loss, 'set_array_module'):
+                loss.set_array_module(self.xp)
+            self.loss = loss
+        if optimizer is not None:
+            if hasattr(optimizer, 'set_array_module'):
+                optimizer.set_array_module(self.xp)
+            self.optimizer = optimizer
+        if accuracy is not None:
+            if hasattr(accuracy, 'set_array_module'):
+                accuracy.set_array_module(self.xp)
+            self.accuracy = accuracy
+
+    def finalize(self):
+        """Construct forward and backward connections between layers."""
+        self.trainable_layers = []
+        num_layers = len(self.layers)
+
+        for i in range(num_layers):
+            layer = self.layers[i]
+            if hasattr(layer, 'weights'):
+                self.trainable_layers.append(layer)
+
+            if i == 0:
+                layer.prev = None
+            else:
+                layer.prev = self.layers[i - 1]
+
+            if i < num_layers - 1:
+                layer.next = self.layers[i + 1]
+            else:
+                layer.next = self.loss
+
+        if self.loss is not None:
+            self.loss.remember_trainable_layers(self.trainable_layers)
+
+        # Detect fused Softmax + Categorical Cross-Entropy combination
+        if isinstance(self.loss, Activation_Softmax_Loss_CategoricalCrossEntropy):
+            self.softmax_classifier_output = self.loss
+
+    def forward(self, X, training=True):
+        """Run feedforward propagation through all sequential layers."""
+        self.layers[0].forward(X, training=training)
+        for i in range(1, len(self.layers)):
+            self.layers[i].forward(self.layers[i - 1].output, training=training)
+        return self.layers[-1].output
+
+    def backward(self, output, y):
+        """Run backpropagation from loss function back to input layer."""
+        if self.softmax_classifier_output is not None:
+            self.softmax_classifier_output.backward(output, y)
+            dvalues = self.softmax_classifier_output.dinputs
+            for layer in reversed(self.layers[:-1]):
+                layer.backward(dvalues)
+                dvalues = layer.dinputs
+            return
+
+        self.loss.backward(output, y)
+        dvalues = self.loss.dinputs
+        for layer in reversed(self.layers):
+            layer.backward(dvalues)
+            dvalues = layer.dinputs
+
+    def to_device(self, arr, dtype=None):
+        """Convert input data to the active compute device (GPU VRAM or CPU)."""
+        if self.device == 'cuda':
+            return to_gpu(arr, dtype=dtype)
+        return np.asarray(arr, dtype=dtype)
+
+    def train(self, X, y, *, epochs=1, batch_size=None, print_every=1, validation_data=None):
+        """
+        High-Performance GPU-Accelerated Training Loop.
+        Transfers entire datasets to GPU VRAM once to maximize memory bandwidth
+        and eliminate PCIe transfer overhead during mini-batch epochs.
+        """
+        if self.accuracy is not None:
+            self.accuracy.init(y)
+
+        # 1. Zero-Copy Host-to-Device Pre-Transfer
+        transfer_start = time.time()
+        X_dev = self.to_device(X, dtype=np.float32)
+        y_dev = self.to_device(y, dtype=np.int32 if y.dtype in (np.int32, np.int64) else np.float32)
+
+        if validation_data is not None:
+            X_val, y_val = validation_data
+            X_val_dev = self.to_device(X_val, dtype=np.float32)
+            y_val_dev = self.to_device(y_val, dtype=np.int32 if y_val.dtype in (np.int32, np.int64) else np.float32)
+        else:
+            X_val_dev, y_val_dev = None, None
+
+        n_samples = len(X)
+        if batch_size is None:
+            train_steps = 1
+        else:
+            train_steps = math.ceil(n_samples / batch_size)
+
+        print(f"\n[MLP_CUDA] Training started: {n_samples:,} samples | {epochs} epochs | "
+              f"batch_size={batch_size if batch_size else n_samples} | device={self.device.upper()}")
+
+        total_train_start = time.time()
+
+        for epoch in range(1, epochs + 1):
+            epoch_start = time.time()
+            self.loss.new_pass()
+            if self.accuracy is not None:
+                self.accuracy.new_pass()
+
+            for step in range(train_steps):
+                if batch_size is None:
+                    batch_X = X_dev
+                    batch_y = y_dev
+                else:
+                    s_idx = step * batch_size
+                    e_idx = min(s_idx + batch_size, n_samples)
+                    batch_X = X_dev[s_idx:e_idx]
+                    batch_y = y_dev[s_idx:e_idx]
+
+                # Forward pass
+                output = self.forward(batch_X, training=True)
+
+                # Loss & Accuracy tracking
+                data_loss, reg_loss = self.loss.calculate(output, batch_y, include_regularization=True)
+                if self.accuracy is not None:
+                    self.accuracy.calculate(output, batch_y)
+
+                # Backward pass
+                self.backward(output, batch_y)
+
+                # Optimizer step
+                self.optimizer.pre_update_params()
+                for layer in self.trainable_layers:
+                    self.optimizer.update_params(layer)
+                self.optimizer.post_update_params()
+
+            # Epoch summary metrics
+            epoch_time = time.time() - epoch_start
+            throughput = n_samples / max(1e-5, epoch_time)
+
+            if epoch % print_every == 0 or epoch == epochs:
+                epoch_data_loss, epoch_reg_loss = self.loss.calculate_accumulated(include_regularization=True)
+                epoch_loss = epoch_data_loss + epoch_reg_loss
+                epoch_acc = self.accuracy.calculate_accumulated() if self.accuracy else 0.0
+                lr = self.optimizer.current_learning_rate
+
+                msg = (f"Epoch {epoch:3d}/{epochs} - {epoch_time:.2f}s ({throughput:,.0f} samples/s) - "
+                       f"loss: {epoch_loss:.4f} (data: {epoch_data_loss:.4f}) - acc: {epoch_acc:.4f} - lr: {lr:.6f}")
+
+                # Validation pass
+                if validation_data is not None:
+                    val_loss, val_acc = self.evaluate(X_val_dev, y_val_dev, batch_size=batch_size, is_device_array=True)
+                    msg += f" - val_loss: {val_loss:.4f} - val_acc: {val_acc:.4f}"
+
+                print(msg)
+
+        total_duration = time.time() - total_train_start
+        print(f"[MLP_CUDA] Training completed in {total_duration:.2f}s (~{n_samples * epochs / total_duration:,.0f} samples/sec overall)\n")
+
+    def evaluate(self, X_val, y_val, *, batch_size=None, is_device_array=False):
+        """
+        Evaluate model performance on validation/test data (Dropout disabled).
+        """
+        if not is_device_array:
+            X_val = self.to_device(X_val, dtype=np.float32)
+            y_val = self.to_device(y_val, dtype=np.int32 if y_val.dtype in (np.int32, np.int64) else np.float32)
+
+        self.loss.new_pass()
+        if self.accuracy is not None:
+            self.accuracy.new_pass()
+
+        n_samples = len(X_val)
+        val_steps = 1 if batch_size is None else math.ceil(n_samples / batch_size)
+
+        for step in range(val_steps):
+            if batch_size is None:
+                batch_X = X_val
+                batch_y = y_val
+            else:
+                s_idx = step * batch_size
+                e_idx = min(s_idx + batch_size, n_samples)
+                batch_X = X_val[s_idx:e_idx]
+                batch_y = y_val[s_idx:e_idx]
+
+            output = self.forward(batch_X, training=False)
+            self.loss.calculate(output, batch_y)
+            if self.accuracy is not None:
+                self.accuracy.calculate(output, batch_y)
+
+        val_loss = self.loss.calculate_accumulated()
+        val_acc = self.accuracy.calculate_accumulated() if self.accuracy is not None else 0.0
+        return float(val_loss), float(val_acc)
+
+    def predict(self, X, *, batch_size=None, return_probabilities=False):
+        """
+        Run inference with dropout disabled.
+        Returns standard NumPy arrays on CPU for seamless integration with downstream tools.
+        """
+        X_dev = self.to_device(X, dtype=np.float32)
+        n_samples = len(X_dev)
+
+        if batch_size is None:
+            output = self.forward(X_dev, training=False)
+        else:
+            steps = math.ceil(n_samples / batch_size)
+            outputs = []
+            for step in range(steps):
+                s_idx = step * batch_size
+                e_idx = min(s_idx + batch_size, n_samples)
+                batch_out = self.forward(X_dev[s_idx:e_idx], training=False)
+                outputs.append(batch_out)
+            output = self.xp.vstack(outputs)
+
+        # Apply softmax probabilities if classifier was fused
+        if self.softmax_classifier_output is not None:
+            self.softmax_classifier_output.activation.forward(output)
+            output = self.softmax_classifier_output.activation.output
+
+        if return_probabilities:
+            return to_cpu(output)
+
+        if len(output.shape) == 2 and output.shape[1] > 1:
+            preds = self.xp.argmax(output, axis=1)
+            return to_cpu(preds)
+        return to_cpu(output)
+
+    def save_parameters(self, filepath):
+        """
+        Save all layer parameters as standard CPU NumPy arrays.
+        Guarantees 100% interoperability across CPU/GPU machines and pure-NumPy libraries.
+        """
+        params = [layer.get_parameters() for layer in self.trainable_layers]
+        with open(filepath, 'wb') as f:
+            pickle.dump(params, f)
+        print(f"[MLP_CUDA] Model parameters saved to: {filepath}")
+
+    def load_parameters(self, filepath):
+        """
+        Load weights and upload them directly to the active compute device.
+        """
+        with open(filepath, 'rb') as f:
+            parameters = pickle.load(f)
+        for layer, (weights, biases) in zip(self.trainable_layers, parameters):
+            layer.set_parameters(weights, biases)
+        print(f"[MLP_CUDA] Model parameters loaded from: {filepath} into {self.device.upper()}")

uteAI/utils/__init__.py

@@ -0,0 +1 @@
+

uteai-0.1.0.dist-info/METADATA

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: uteAI
+Version: 0.1.0
+Summary: Advanced AI, Neural Network, and Edge Engine Library from HCMUTE
+Author-email: Dung Cai <dung.cai@hcmute.edu.vn>
+License: MIT
+Project-URL: Homepage, https://github.com/your-username/uteAI
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: numpy>=1.20.0

uteai-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+uteAI/__init__.py,sha256=voOfWI2PKc9mr61U_rYkcGHaf8on1KuuiU--7-QbRrk,142
+uteAI/core/__init__.py,sha256=0K5slH76qU-pMYfl0xAAsp0dXKRIt0uWcGVfGaRRm9Y,578
+uteAI/core/mlp_cuda.py,sha256=7gvJdF1K8ORnZDACmmX-sc0C38oC5cOdsmVaJ2kBmrQ,39603
+uteAI/utils/__init__.py,sha256=frcCV1k9oG9oKj3dpUqdJg1PxRT2RSN_XKdLCPjaYaY,2
+uteai-0.1.0.dist-info/METADATA,sha256=KMcZS4TAHLgbGAYuh3dPSCCACDVGaYZnJ_-tGys_zzk,569
+uteai-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+uteai-0.1.0.dist-info/top_level.txt,sha256=PgDHYEtdV49TFlqBvqQ1P-HSxIQ6WIxJt0fQnC7I51c,6
+uteai-0.1.0.dist-info/RECORD,,

uteai-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

uteai-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+uteAI